From e4e453a11c776bb4127e07252f868c3e54e3e386 Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 30 Nov 2021 15:55:26 -0500 Subject: [PATCH 1/6] add code to parse and filter --- .gitignore | 3 ++- package.json | 1 + src/ci/invidiousCI.ts | 50 +++++++++++++++++++++++++++++++++++++++ src/ci/invidiouslist.json | 1 + src/config.ts | 3 ++- 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 src/ci/invidiousCI.ts create mode 100644 src/ci/invidiouslist.json diff --git a/.gitignore b/.gitignore index 2833884c..3dcdb226 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ web-ext-artifacts .vscode/ dist/ tmp/ -.DS_Store \ No newline at end of file +.DS_Store +src/ci/data.json \ No newline at end of file diff --git a/package.json b/package.json index 2f59a888..963c1203 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "build:watch": "npm run build:watch:chrome", "build:watch:chrome": "webpack --env.browser=chrome --config webpack/webpack.dev.js --watch", "build:watch:firefox": "webpack --env.browser=firefox --config webpack/webpack.dev.js --watch", + "ci:invidious": "ts-node src/ci/invidiousCI.ts", "dev": "npm run build:dev && concurrently \"npm run web-run\" \"npm run build:watch\"", "dev:firefox": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox\" \"npm run build:watch:firefox\"", "dev:firefox-android": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox-android\" \"npm run build:watch:firefox\"", diff --git a/src/ci/invidiousCI.ts b/src/ci/invidiousCI.ts new file mode 100644 index 00000000..2c4a8089 --- /dev/null +++ b/src/ci/invidiousCI.ts @@ -0,0 +1,50 @@ +/* +This file is only ran by GitHub Actions in order to populate the Invidious instances list + +This file should not be shipped with the extension +*/ + +import { writeFile } from 'fs'; +import { join } from 'path'; + +// import file downloade from https://api.invidious.io/instances.json +import * as data from "./data.json"; + +type instanceMap = { + name: string, + url: string, + dailyRatios: {ratio: string, label: string }[], + thirtyDayUptime: string +}[] + +// only https servers +const mapped: instanceMap = data + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .filter((i: any) => i[1]?.type === 'https') + // eslint-disable-next-line @typescript-eslint/no-explicit-any + .map((instance: any) => { + return { + name: instance[0], + url: instance[1].uri, + dailyRatios: instance[1].monitor.dailyRatios, + thirtyDayUptime: instance[1]?.monitor['30dRatio'].ratio, + } + }) + +// reliability and sanity checks +const reliableCheck = mapped + .filter((instance) => { + // 30d uptime >= 90% + const thirtyDayUptime = Number(instance.thirtyDayUptime) >= 90 + // available for at least 80/90 days + const dailyRatioCheck = instance.dailyRatios.filter(status => status.label !== "black") + return (thirtyDayUptime && dailyRatioCheck.length >= 80) + }) + // url includes name + .filter(instance => instance.url.includes(instance.name)) + +// finally map to array +const result: string[] = reliableCheck.map(instance => instance.name) +writeFile(join(__dirname, "./invidiouslist.json"), JSON.stringify(result), (err) => { + if (err) return console.log(err); +}) \ No newline at end of file diff --git a/src/ci/invidiouslist.json b/src/ci/invidiouslist.json new file mode 100644 index 00000000..905605d8 --- /dev/null +++ b/src/ci/invidiouslist.json @@ -0,0 +1 @@ +["yewtu.be","invidious.snopyta.org","vid.puffyan.us","invidious.kavin.rocks","invidio.xamh.de","invidious-us.kavin.rocks","inv.riverside.rocks","vid.mint.lgbt","youtube.076.ne.jp","invidious.namazso.eu"] \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 344a7b62..912f87dc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,5 @@ import * as CompileConfig from "../config.json"; +import * as invidiousList from "../src/ci/invidiouslist.json"; import { Category, CategorySelection, CategorySkipOption, NoticeVisbilityMode, PreviewBarOption, SponsorTime, StorageChangesObject, UnEncodedSegmentTimes as UnencodedSegmentTimes } from "./types"; interface SBConfig { @@ -188,7 +189,7 @@ const Config: SBObject = { hideSkipButtonPlayerControls: false, hideDiscordLaunches: 0, hideDiscordLink: false, - invidiousInstances: ["invidious.snopyta.org"], + invidiousInstances: invidiousList, supportInvidious: false, serverAddress: CompileConfig.serverAddress, minDuration: 0, From dc47b9ffd2049e8a11e566dbf2753fbe59e17465 Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 30 Nov 2021 16:44:42 -0500 Subject: [PATCH 2/6] appease CI/ webpack move to ci/ change triggers --- .github/workflows/updateInvidous.yml | 32 ++++++++++++++++++++++++++++ .gitignore | 2 +- {src/ci => ci}/invidiousCI.ts | 9 ++++++-- {src/ci => ci}/invidiouslist.json | 0 package.json | 2 +- src/config.ts | 2 +- 6 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/updateInvidous.yml rename {src/ci => ci}/invidiousCI.ts (86%) rename {src/ci => ci}/invidiouslist.json (100%) diff --git a/.github/workflows/updateInvidous.yml b/.github/workflows/updateInvidous.yml new file mode 100644 index 00000000..f43142c7 --- /dev/null +++ b/.github/workflows/updateInvidous.yml @@ -0,0 +1,32 @@ +name: update invidious +on: + workflow_dispatch: + schedule: + - cron: '0 0 1 * *' # check every month + +jobs: + check-list: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Download instance list + run: | + wget https://api.invidious.io/instances.json -O data.json + - name: "Run CI" + run: npm run ci:invidious + - name: setup git config + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + - name: "Commit new files" + run : | + if [ $(git status --porcelain=v1 2>/dev/null | wc -l) -ge 1 ] + then + echo "pushing changes" + git add invidiouslist.json + git commit -m "[CI] New Invidious List" + git push origin main + else + echo "no changes" + fi \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3dcdb226..2eafbe10 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ web-ext-artifacts dist/ tmp/ .DS_Store -src/ci/data.json \ No newline at end of file +ci/data.json \ No newline at end of file diff --git a/src/ci/invidiousCI.ts b/ci/invidiousCI.ts similarity index 86% rename from src/ci/invidiousCI.ts rename to ci/invidiousCI.ts index 2c4a8089..05cf8f8f 100644 --- a/src/ci/invidiousCI.ts +++ b/ci/invidiousCI.ts @@ -4,10 +4,15 @@ This file is only ran by GitHub Actions in order to populate the Invidious insta This file should not be shipped with the extension */ -import { writeFile } from 'fs'; +import { writeFile, existsSync } from 'fs'; import { join } from 'path'; -// import file downloade from https://api.invidious.io/instances.json +// import file from https://api.invidious.io/instances.json +if (!existsSync('./data.json')) { + process.exit(1); +} +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore import * as data from "./data.json"; type instanceMap = { diff --git a/src/ci/invidiouslist.json b/ci/invidiouslist.json similarity index 100% rename from src/ci/invidiouslist.json rename to ci/invidiouslist.json diff --git a/package.json b/package.json index 963c1203..62153f13 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "build:watch": "npm run build:watch:chrome", "build:watch:chrome": "webpack --env.browser=chrome --config webpack/webpack.dev.js --watch", "build:watch:firefox": "webpack --env.browser=firefox --config webpack/webpack.dev.js --watch", - "ci:invidious": "ts-node src/ci/invidiousCI.ts", + "ci:invidious": "ts-node ci/invidiousCI.ts", "dev": "npm run build:dev && concurrently \"npm run web-run\" \"npm run build:watch\"", "dev:firefox": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox\" \"npm run build:watch:firefox\"", "dev:firefox-android": "npm run build:dev:firefox && concurrently \"npm run web-run:firefox-android\" \"npm run build:watch:firefox\"", diff --git a/src/config.ts b/src/config.ts index 912f87dc..814e95a6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,5 @@ import * as CompileConfig from "../config.json"; -import * as invidiousList from "../src/ci/invidiouslist.json"; +import * as invidiousList from "../ci/invidiouslist.json"; import { Category, CategorySelection, CategorySkipOption, NoticeVisbilityMode, PreviewBarOption, SponsorTime, StorageChangesObject, UnEncodedSegmentTimes as UnencodedSegmentTimes } from "./types"; interface SBConfig { From 33098ac6596dc7cd6b3eb31cf0df6e7f88d7f883 Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 1 Dec 2021 17:48:07 -0500 Subject: [PATCH 3/6] migrate and populat on reset --- src/config.ts | 7 ++++++- src/options.ts | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/config.ts b/src/config.ts index 814e95a6..5cc3ee0b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -189,7 +189,7 @@ const Config: SBObject = { hideSkipButtonPlayerControls: false, hideDiscordLaunches: 0, hideDiscordLink: false, - invidiousInstances: invidiousList, + invidiousInstances: ["invidious.snopyta.org"], // leave as default supportInvidious: false, serverAddress: CompileConfig.serverAddress, minDuration: 0, @@ -433,6 +433,11 @@ function migrateOldFormats(config: SBConfig) { if (config["previousVideoID"] !== undefined) { chrome.storage.sync.remove("previousVideoID"); } + + // populate invidiousInstances with new instances if 3p support is **DISABLED** + if (!config["supportInvidious"]) { + config["invidiousInstances"] = invidiousList + } } async function setupConfig() { diff --git a/src/options.ts b/src/options.ts index 8c5c3bca..f2d4ed4a 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,5 +1,6 @@ import Config from "./config"; import * as CompileConfig from "../config.json"; +import * as invidiousList from "../ci/invidiouslist.json"; // Make the config public for debugging purposes window.SB = Config; @@ -297,8 +298,8 @@ function invidiousInstanceAddInit(element: HTMLElement, option: string) { const resetButton = element.querySelector(".invidious-instance-reset"); resetButton.addEventListener("click", function() { if (confirm(chrome.i18n.getMessage("resetInvidiousInstanceAlert"))) { - // Set to a clone of the default - Config.config[option] = Config.defaults[option].slice(0); + // Set to CI populated list + Config.config[option] = invidiousList; } }); } From e16aae393fc73be147d863247a40a8f0f8a4e0e2 Mon Sep 17 00:00:00 2001 From: Michael C Date: Sat, 4 Dec 2021 23:16:26 -0500 Subject: [PATCH 4/6] also check against length --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 5cc3ee0b..f1e80057 100644 --- a/src/config.ts +++ b/src/config.ts @@ -435,7 +435,7 @@ function migrateOldFormats(config: SBConfig) { } // populate invidiousInstances with new instances if 3p support is **DISABLED** - if (!config["supportInvidious"]) { + if (!config["supportInvidious"] && config["invidiousInstances"].length !== invidiousList.length) { config["invidiousInstances"] = invidiousList } } From 6aa1665d7cf7b9aa01e97f999c80ae4dc5874f50 Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 28 Dec 2021 14:50:12 -0500 Subject: [PATCH 5/6] Create pull request for invidious list instead of commit --- .github/workflows/updateInvidous.yml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/updateInvidous.yml b/.github/workflows/updateInvidous.yml index f43142c7..50bb9769 100644 --- a/.github/workflows/updateInvidous.yml +++ b/.github/workflows/updateInvidous.yml @@ -15,18 +15,12 @@ jobs: wget https://api.invidious.io/instances.json -O data.json - name: "Run CI" run: npm run ci:invidious - - name: setup git config - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - name: "Commit new files" - run : | - if [ $(git status --porcelain=v1 2>/dev/null | wc -l) -ge 1 ] - then - echo "pushing changes" - git add invidiouslist.json - git commit -m "[CI] New Invidious List" - git push origin main - else - echo "no changes" - fi \ No newline at end of file + + - name: Create pull request to update list + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Update Invidious List + author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + branch: ci/update_invidious_list + title: Update Invidious List + body: Automated Invidious list update \ No newline at end of file From 7b74307013fd56232ec84318456a8c355a86723f Mon Sep 17 00:00:00 2001 From: Ajay Date: Tue, 28 Dec 2021 14:50:18 -0500 Subject: [PATCH 6/6] formatting --- src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index f1e80057..10671746 100644 --- a/src/config.ts +++ b/src/config.ts @@ -436,7 +436,7 @@ function migrateOldFormats(config: SBConfig) { // populate invidiousInstances with new instances if 3p support is **DISABLED** if (!config["supportInvidious"] && config["invidiousInstances"].length !== invidiousList.length) { - config["invidiousInstances"] = invidiousList + config["invidiousInstances"] = invidiousList; } }