Merge pull request #1798 from mchangrh/piped-ci

Piped CI

.github/workflows/updateInvidous.yml

@@ -11,9 +11,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
-      - name: Download instance list
+      - name: Download instance lists
         run: |
-          wget https://api.invidious.io/instances.json -O ci/data.json
+          wget https://api.invidious.io/instances.json -O ci/invidious_instances.json
+          wget  https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json -O ci/piped_instances.json
       - name: Install dependencies
         run: npm ci
       - name: "Run CI"
@@ -24,7 +25,7 @@ jobs:
         # v4.2.3
         with:
           commit-message: Update Invidious List
-          author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
+          author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
           branch: ci/update_invidious_list
           title: Update Invidious List
           body: Automated Invidious list update

.gitignore

@@ -7,5 +7,6 @@ web-ext-artifacts
 dist/
 tmp/
 .DS_Store
-ci/data.json
+ci/invidious_instances.json
+ci/piped_instances.json
 test-results

ci/generateList.ts

@@ -0,0 +1,63 @@
+/*
+This file is only ran by GitHub Actions in order to populate the Invidious instances list
+
+This file should not be shipped with the extension
+*/
+
+/*
+Criteria for inclusion:
+Invidious
+- 30d uptime >= 90%
+- available for at least 80/90 days
+- must have been up for at least 90 days
+- HTTPS only
+- url includes name (this is to avoid redirects)
+
+Piped
+- 30d uptime >= 90%
+- available for at least 80/90 days
+- must have been up for at least 90 days
+- must not be a wildcard redirect to piped.video
+- must be currently up
+- must have a functioning frontend
+- must have a functioning API
+*/
+
+import { writeFile, existsSync } from "fs"
+import { join } from "path"
+import { getInvidiousList } from "./invidiousCI";
+// import { getPipedList } from "./pipedCI";
+
+const checkPath = (path: string) => existsSync(path);
+const fixArray = (arr: string[]) => [...new Set(arr)].sort()
+
+async function generateList() {
+  // import file from https://api.invidious.io/instances.json
+  const invidiousPath = join(__dirname, "invidious_instances.json");
+  // import file from https://github.com/TeamPiped/piped-uptime/raw/master/history/summary.json
+  const pipedPath = join(__dirname, "piped_instances.json");
+
+  // check if files exist
+  if (!checkPath(invidiousPath) || !checkPath(pipedPath)) {
+    console.log("Missing files")
+    process.exit(1);
+  }
+
+  // static non-invidious instances
+  const staticInstances = ["www.youtubekids.com"];
+  // invidious instances
+  const invidiousList = fixArray(getInvidiousList())
+  // piped instnaces
+  // const pipedList = fixArray(await getPipedList())
+
+  console.log([...staticInstances, ...invidiousList])
+
+  writeFile(
+    join(__dirname, "./invidiouslist.json"),
+    JSON.stringify([...staticInstances, ...invidiousList]),
+    (err) => {
+      if (err) return console.log(err);
+    }
+  );
+}
+generateList()

ci/invidiousCI.ts

@@ -1,55 +1,31 @@
-/*
-This file is only ran by GitHub Actions in order to populate the Invidious instances list
+import { InvidiousInstance, instanceMap } from "./invidiousType"
 
-This file should not be shipped with the extension
-*/
-
-import { writeFile, existsSync } from 'fs';
-import { join } from 'path';
-
-// import file from https://api.invidious.io/instances.json
-if (!existsSync(join(__dirname, "data.json"))) {
-  process.exit(1);
-}
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore
-import * as data from "../ci/data.json";
-
-type instanceMap = {
-  name: string;
-  url: string;
-  dailyRatios: {ratio: string; label: string }[];
-  thirtyDayUptime: string;
-}[]
+import * as data from "../ci/invidious_instances.json";
 
 // only https servers
 const mapped: instanceMap = data
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  .filter((i: any) => i[1]?.type === 'https')
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  .map((instance: any) => {
+  .filter((i: InvidiousInstance) => i[1]?.type === "https")
+  .map((instance: InvidiousInstance) => {
     return {
       name: instance[0],
       url: instance[1].uri,
       dailyRatios: instance[1].monitor.dailyRatios,
-      thirtyDayUptime: instance[1]?.monitor['30dRatio'].ratio,
+      thirtyDayUptime: instance[1]?.monitor["30dRatio"].ratio,
     }
-  })
+  });
 
 // reliability and sanity checks
 const reliableCheck = mapped
-  .filter((instance) => {
+  .filter(instance => {
     // 30d uptime >= 90%
-    const thirtyDayUptime = Number(instance.thirtyDayUptime) >= 90
+    const thirtyDayUptime = Number(instance.thirtyDayUptime) >= 90;
     // available for at least 80/90 days
-    const dailyRatioCheck = instance.dailyRatios.filter(status => status.label !== "black")
-    return (thirtyDayUptime && dailyRatioCheck.length >= 80)
+    const dailyRatioCheck = instance.dailyRatios.filter(status => status.label !== "black");
+    return thirtyDayUptime && dailyRatioCheck.length >= 80;
   })
   // url includes name
-  .filter(instance => instance.url.includes(instance.name))
+  .filter(instance => instance.url.includes(instance.name));
 
-// finally map to array
-const result: string[] = reliableCheck.map(instance => instance.name).sort()
-writeFile(join(__dirname, "./invidiouslist.json"), JSON.stringify(result), (err) => {
-  if (err) return console.log(err);
-})
+export function getInvidiousList(): string[] {
+  return reliableCheck.map(instance => instance.name).sort()
+}

ci/invidiousType.ts

@@ -0,0 +1,54 @@
+type ratio = {
+  ratio: string;
+  label: string;
+}
+
+export type instanceMap = {
+  name: string;
+  url: string;
+  dailyRatios: {ratio: string; label: string }[];
+  thirtyDayUptime: string;
+}[]
+
+export type InvidiousInstance = [
+  string,
+  {
+    flag: string;
+    region: string;
+    stats: null | {
+      version: string;
+      software: {
+        name: string;
+        version: string;
+        branch: string;
+      };
+      openRegistrations: boolean;
+      usage: {
+        users: {
+          total: number;
+          activeHalfyear: number;
+          activeMonth: number;
+        };
+      };
+      metadata: {
+        updatedAt: number;
+        lastChannelRefreshedAt: number;
+      };
+    };
+    cors: boolean | null;
+    api: boolean | null;
+    type: "https" | "http" | "onion" | "i2p";
+    uri: string;
+    monitor: null | {
+      monitorId: number;
+      createdAt: number;
+      statusClass: string;
+      name: string;
+      url: string | null;
+      type: "HTTP(s)";
+      dailyRatios: ratio[];
+      "90dRatio": ratio;
+      "30dRatio": ratio;
+    };
+  }
+]

ci/invidiouslist.json

@@ -1 +1 @@
-["inv.bp.projectsegfau.lt","inv.zzls.xyz","invidious.0011.lt","invidious.baczek.me","invidious.lunar.icu","invidious.privacydev.net","invidious.tiekoetter.com","iv.ggtyler.dev","iv.melmac.space","vid.puffyan.us","y.com.sb","yewtu.be","yt.artemislena.eu"]
+["www.youtubekids.com","inv.bp.projectsegfau.lt","inv.tux.pizza","inv.zzls.xyz","invidious.0011.lt","invidious.lunar.icu","invidious.privacydev.net","invidious.tiekoetter.com","iv.ggtyler.dev","iv.melmac.space","vid.priv.au","vid.puffyan.us","yewtu.be","yt.artemislena.eu"]

ci/pipedCI.ts

@@ -0,0 +1,92 @@
+import * as data from "../ci/piped_instances.json";
+
+type percent = string
+type dailyMinutesDown = Record<string, number>
+
+type PipedInstance = {
+  name: string;
+  url: string;
+  icon: string;
+  slug: string;
+  status: string;
+  uptime: percent;
+  uptimeDay: percent;
+  uptimeWeek: percent;
+  uptimeMonth: percent;
+  uptimeYear: percent;
+  time: number;
+  timeDay: number;
+  timeWeek: number;
+  timeMonth: number;
+  timeYear: number;
+  dailyMinutesDown: dailyMinutesDown
+}
+
+const percentNumber = (percent: percent) => Number(percent.replace("%", ""))
+const ninetyDaysAgo = new Date(Date.now() - 90 * 24 * 60 * 60 * 1000)
+
+function dailyMinuteFilter (dailyMinutesDown: dailyMinutesDown) {
+  let daysDown = 0
+  for (const [date, minsDown] of Object.entries(dailyMinutesDown)) {
+    if (new Date(date) >= ninetyDaysAgo && minsDown > 1000) { // if within 90 days and down for more than 1000 minutes
+      daysDown++
+    }
+  }
+  // return true f less than 10 days down
+  return daysDown < 10
+}
+
+const getHost = (url: string) => new URL(url).host
+
+const getWatchPage = async (instance: PipedInstance) =>
+  fetch(`https://${getHost(instance.url)}`, { redirect: "manual" })
+    .then(res => res.headers.get("Location"))
+    .catch(e => { console.log (e); return null })
+
+const siteOK = async (instance) => {
+  // check if entire site is redirect
+  const notRedirect = await fetch(instance.url, { redirect: "manual" })
+    .then(res => res.status == 200)
+  // only allow kavin to return piped.video
+  // if (instance.url.startsWith("https://piped.video") && instance.slug !== "kavin-rocks-official") return false
+  // check if frontend is OK
+  const watchPageStatus = await fetch(instance.frontendUrl)
+    .then(res => res.ok)
+  // test API - stream returns ok result
+  const streamStatus = await fetch(`${instance.apiUrl}/streams/BaW_jenozKc`)
+    .then(res => res.ok)
+  // get startTime of monitor
+  const age = await fetch(instance.historyUrl)
+    .then(res => res.text())
+    .then(text => { // startTime greater than 90 days ago
+      const date = text.match(/startTime: (.+)/)[1]
+      return Date.parse(date) < ninetyDaysAgo.valueOf()
+    })
+  // console.log(notRedirect, watchPageStatus, streamStatus, age, instance.frontendUrl, instance.apiUrl)
+  return notRedirect && watchPageStatus && streamStatus && age
+}
+
+const staticFilters = (data as PipedInstance[])
+  .filter(instance => {
+    const isup = instance.status === "up"
+    const monthCheck = percentNumber(instance.uptimeMonth) >= 90
+    const dailyMinuteCheck = dailyMinuteFilter(instance.dailyMinutesDown)
+    return isup && monthCheck && dailyMinuteCheck
+  })
+  .map(async instance => {
+    // get frontend url
+    const frontendUrl = await getWatchPage(instance)
+    if (!frontendUrl) return null // return false if frontend doesn't resolve
+    // get api base
+    const apiUrl = instance.url.replace("/healthcheck", "")
+    const historyUrl = `https://raw.githubusercontent.com/TeamPiped/piped-uptime/master/history/${instance.slug}.yml`
+    const pass = await siteOK({ apiUrl, historyUrl, frontendUrl, url: instance.url })
+    const frontendHost = getHost(frontendUrl)
+    return pass ? frontendHost : null
+  })
+
+export async function getPipedList(): Promise<string[]> {
+  const instances = await Promise.all(staticFilters)
+    .then(arr => arr.filter(i => i !== null))
+  return instances
+}

package.json

@@ -55,7 +55,7 @@
     "build:watch": "npm run build:watch:chrome",
     "build:watch:chrome": "webpack --env browser=chrome --config webpack/webpack.dev.js --watch",
     "build:watch:firefox": "webpack --env browser=firefox --config webpack/webpack.dev.js --watch",
-    "ci:invidious": "ts-node ci/invidiousCI.ts",
+    "ci:invidious": "ts-node ci/generateList.ts",
     "dev": "npm run build:dev && concurrently \"npm run web-run\" \"npm run build:watch\"",
     "dev:firefox": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox\" \"npm run build:watch:firefox\"",
     "dev:firefox-android": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox-android\" \"npm run build:watch:firefox\"",

src/content.ts

@@ -524,7 +524,7 @@ function createPreviewBar(): void {
             selector: ".vjs-progress-holder",
             isVisibleCheck: false
         }, {
-            // For Youtube Music
+            // For Youtube Music and YTKids
             // there are two sliders, one for volume and one for progress - both called #progressContainer
             selector: "#progress-bar>#sliderContainer>div>#sliderBar>#progressContainer",
         }, {

src/js-components/previewBar.ts

@@ -97,7 +97,8 @@ class PreviewBar {
         this.chapterTooltip = document.createElement("div");
         this.chapterTooltip.className = "ytp-tooltip-title sponsorCategoryTooltip";
 
-        const tooltipTextWrapper = document.querySelector(".ytp-tooltip-text-wrapper");
+        // global chaper tooltip or duration tooltip
+        const tooltipTextWrapper = document.querySelector(".ytp-tooltip-text-wrapper") ?? document.querySelector("#progress-bar-container.ytk-player > #hover-time-info");
         const originalTooltip = tooltipTextWrapper.querySelector(".ytp-tooltip-title:not(.sponsorCategoryTooltip)") as HTMLElement;
         if (!tooltipTextWrapper || !tooltipTextWrapper.parentElement) return;
 

src/utils.ts

@@ -299,7 +299,8 @@ export default class Utils {
             "#main-panel.ytmusic-player-page", // YouTube music
             "#player-container .video-js", // Invidious
             ".main-video-section > .video-container", // Cloudtube
-            ".shaka-video-container" // Piped
+            ".shaka-video-container", // Piped
+            "#player-container.ytk-player", // YT Kids
         ];
 
         let referenceNode = findValidElementFromSelector(selectors)

tsconfig-production.json

@@ -16,5 +16,8 @@
             "dom",
             "dom.iterable"
         ]
-    }
+    },
+    "include": [
+        "src/**/*"
+    ]
 }

tsconfig.json

@@ -16,5 +16,8 @@
             "dom",
             "dom.iterable"
         ]
-    }
+    },
+    "include": [
+        "src/**/*"
+    ]
 }