From 40d522694dc4bd108dec0e2ee7af14826ca7c0f6 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 4 Sep 2020 00:05:41 +0100 Subject: [PATCH 01/10] Added has prefix implementation (no config page) --- package-lock.json | 5 +++++ package.json | 1 + src/config.ts | 2 ++ src/content.ts | 34 +++++++++++++++++++++++++++++----- src/utils.ts | 15 +++++++++++++++ 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5cd00a3d..7e806d7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10494,6 +10494,11 @@ "traverse": "0.4.x" } }, + "js-sha256": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz", + "integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", diff --git a/package.json b/package.json index 50d274e5..5d42ab1c 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "babel-loader": "^8.0.6", "babel-preset-env": "^1.7.0", "concurrently": "^5.1.0", + "js-sha256": "^0.9.0", "react": "^16.12.0", "react-dom": "^16.12.0" }, diff --git a/src/config.ts b/src/config.ts index 5d34df6b..9d830116 100644 --- a/src/config.ts +++ b/src/config.ts @@ -35,6 +35,7 @@ interface SBConfig { audioNotificationOnSkip, checkForUnlistedVideos: boolean, testingServer: boolean, + hashPrefix: boolean // What categories should be skipped categorySelections: CategorySelection[], @@ -166,6 +167,7 @@ var Config: SBObject = { audioNotificationOnSkip: false, checkForUnlistedVideos: false, testingServer: false, + hashPrefix: false, categorySelections: [{ name: "sponsor", diff --git a/src/content.ts b/src/content.ts index 7f75ada0..37999d99 100644 --- a/src/content.ts +++ b/src/content.ts @@ -618,12 +618,36 @@ function sponsorsLookup(id: string) { categories.push(categorySelection.name); } - utils.asyncRequestToServer('GET', "/api/skipSegments", { - videoID: id, - categories - }).then(async (response: FetchResponse) => { + // Check for hashPrefix setting + let getRequest; + if (Config.config.hashPrefix) { + getRequest = utils.asyncRequestToServer('GET', "/api/skipSegments/"+utils.getHash(id, 1).substr(0,4), { + categories + }); + } else { + getRequest = utils.asyncRequestToServer('GET', "/api/skipSegments", { + videoID: id, + categories + }); + } + getRequest.then(async (response: FetchResponse) => { if (response?.ok) { - let recievedSegments: SponsorTime[] = JSON.parse(response.responseText); + let getResult = JSON.parse(response.responseText); + if (Config.config.hashPrefix) { + getResult = getResult.filter((video) => { + return video.videoID = id; + }); + if (getResult.length === 1) { + getResult = getResult[0].segments; + if (getResult.length === 0) { // return if no regments found + return; + } + } else { // return if no video found + return; + } + } + + let recievedSegments: SponsorTime[] = getResult; if (!recievedSegments.length) { console.error("[SponsorBlock] Server returned malformed response: " + JSON.stringify(recievedSegments)); return; diff --git a/src/utils.ts b/src/utils.ts index 43cdc68e..d95da7dc 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,6 @@ import Config from "./config"; import { CategorySelection, SponsorTime, FetchResponse } from "./types"; +import { sha256 } from 'js-sha256'; import * as CompileConfig from "../config.json"; @@ -378,6 +379,20 @@ class Utils { isFirefox(): boolean { return typeof(browser) !== "undefined"; } + + getHash(value: string, times=5000): string { + if (times <= 0) return ""; + + for (let i = 0; i < times; i++) { + let hash = sha256.create(); + hash.update(value); + hash.hex(); + value = hash.toString(); + } + + return value; + } + } export default Utils; \ No newline at end of file From 46e6b79b207478f46ad409a57eb77bbd6d0aea2b Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 4 Sep 2020 00:39:26 +0100 Subject: [PATCH 02/10] Added config option for query by hash prefix --- public/_locales/en/messages.json | 6 ++++++ public/options/options.html | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/public/_locales/en/messages.json b/public/_locales/en/messages.json index 3ae46303..842e5d83 100644 --- a/public/_locales/en/messages.json +++ b/public/_locales/en/messages.json @@ -202,6 +202,12 @@ "whatViewTracking": { "message": "This feature tracks which segments you have skipped to let users know how much their submission has helped others and used as a metric along with upvotes to ensure that spam doesn't get into the database. The extension sends a message to the server each time you skip a segment. Hopefully most people don't change this setting so that the view numbers are accurate. :)" }, + "enableQueryByHashPrefix": { + "message": "Query By Hash Prefix" + }, + "whatQueryByHashPrefix": { + "message": "Instead of requesting segments from the server using the videoID, the first 4 characters of the hash of the videoID are sent. This server will send back data for all videos with similar hashes." + }, "showNotice": { "message": "Show Notice Again" }, diff --git a/public/options/options.html b/public/options/options.html index 5a5e6caa..92ac686d 100644 --- a/public/options/options.html +++ b/public/options/options.html @@ -302,6 +302,23 @@
__MSG_whatViewTracking__
+
+
+ +
+ + +
+
+ +
__MSG_whatQueryByHashPrefix__
+
+

From 1a48f556fa82858514d90ac256e9a4e63895ea15 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 4 Sep 2020 00:39:41 +0100 Subject: [PATCH 03/10] typo in comment --- src/content.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content.ts b/src/content.ts index 37999d99..205d64c8 100644 --- a/src/content.ts +++ b/src/content.ts @@ -639,7 +639,7 @@ function sponsorsLookup(id: string) { }); if (getResult.length === 1) { getResult = getResult[0].segments; - if (getResult.length === 0) { // return if no regments found + if (getResult.length === 0) { // return if no segments found return; } } else { // return if no video found From 35651d2a50833725f083dd7e7f83f3ac3979cf16 Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 4 Sep 2020 11:54:30 +0100 Subject: [PATCH 04/10] Update src/content.ts Co-authored-by: Ajay Ramachandran --- src/content.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content.ts b/src/content.ts index 205d64c8..061a0159 100644 --- a/src/content.ts +++ b/src/content.ts @@ -621,7 +621,7 @@ function sponsorsLookup(id: string) { // Check for hashPrefix setting let getRequest; if (Config.config.hashPrefix) { - getRequest = utils.asyncRequestToServer('GET', "/api/skipSegments/"+utils.getHash(id, 1).substr(0,4), { + getRequest = utils.asyncRequestToServer('GET', "/api/skipSegments/" + utils.getHash(id, 1).substr(0,4), { categories }); } else { From b6206fabbcd182a96a4221972ffadfc153b86aba Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 4 Sep 2020 11:55:42 +0100 Subject: [PATCH 05/10] Update src/content.ts Co-authored-by: Ajay Ramachandran --- src/content.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content.ts b/src/content.ts index 061a0159..499dea18 100644 --- a/src/content.ts +++ b/src/content.ts @@ -637,7 +637,7 @@ function sponsorsLookup(id: string) { getResult = getResult.filter((video) => { return video.videoID = id; }); - if (getResult.length === 1) { + if (getResult.length > 0) { getResult = getResult[0].segments; if (getResult.length === 0) { // return if no segments found return; From d8a48ed9bca889dec5c8ea04d47d7e322eda871d Mon Sep 17 00:00:00 2001 From: Joe Dowd Date: Fri, 4 Sep 2020 11:56:07 +0100 Subject: [PATCH 06/10] Update src/content.ts Co-authored-by: Ajay Ramachandran --- src/content.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/content.ts b/src/content.ts index 499dea18..7661edc1 100644 --- a/src/content.ts +++ b/src/content.ts @@ -634,9 +634,7 @@ function sponsorsLookup(id: string) { if (response?.ok) { let getResult = JSON.parse(response.responseText); if (Config.config.hashPrefix) { - getResult = getResult.filter((video) => { - return video.videoID = id; - }); + getResult = getResult.filter((video) => video.videoID === id); if (getResult.length > 0) { getResult = getResult[0].segments; if (getResult.length === 0) { // return if no segments found From 65f7f9605f6a777b57b17d2a596a1797ec698278 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Sep 2020 12:14:38 -0400 Subject: [PATCH 07/10] Fix broken shadow ban check --- src/content.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/content.ts b/src/content.ts index 7661edc1..f757d671 100644 --- a/src/content.ts +++ b/src/content.ts @@ -633,6 +633,8 @@ function sponsorsLookup(id: string) { getRequest.then(async (response: FetchResponse) => { if (response?.ok) { let getResult = JSON.parse(response.responseText); + console.log(getResult); + alert(getResult.length); if (Config.config.hashPrefix) { getResult = getResult.filter((video) => video.videoID === id); if (getResult.length > 0) { From e0c6b687d3f5500e01d544914501e6b655c0c613 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Sep 2020 12:17:09 -0400 Subject: [PATCH 08/10] Revert "Fix broken shadow ban check" This reverts commit 65f7f9605f6a777b57b17d2a596a1797ec698278. --- src/content.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/content.ts b/src/content.ts index f757d671..7661edc1 100644 --- a/src/content.ts +++ b/src/content.ts @@ -633,8 +633,6 @@ function sponsorsLookup(id: string) { getRequest.then(async (response: FetchResponse) => { if (response?.ok) { let getResult = JSON.parse(response.responseText); - console.log(getResult); - alert(getResult.length); if (Config.config.hashPrefix) { getResult = getResult.filter((video) => video.videoID === id); if (getResult.length > 0) { From d0061985ca6315adbf12c8a20aee454ff8e14aa9 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Sep 2020 12:39:00 -0400 Subject: [PATCH 09/10] Separate refetch to a new option --- public/_locales/en/messages.json | 6 ++++ public/options/options.html | 17 +++++++++++ src/config.ts | 5 ++-- src/content.ts | 51 +++++++++++++++++++------------- 4 files changed, 56 insertions(+), 23 deletions(-) diff --git a/public/_locales/en/messages.json b/public/_locales/en/messages.json index 842e5d83..19a999b1 100644 --- a/public/_locales/en/messages.json +++ b/public/_locales/en/messages.json @@ -208,6 +208,12 @@ "whatQueryByHashPrefix": { "message": "Instead of requesting segments from the server using the videoID, the first 4 characters of the hash of the videoID are sent. This server will send back data for all videos with similar hashes." }, + "enableRefetchWhenNotFound": { + "message": "Refetch Segments On New Videos" + }, + "whatRefetchWhenNotFound": { + "message": "If the video is new, and there are no segments found, it will keep refetching every few minutes while you watch." + }, "showNotice": { "message": "Show Notice Again" }, diff --git a/public/options/options.html b/public/options/options.html index 92ac686d..04620a9e 100644 --- a/public/options/options.html +++ b/public/options/options.html @@ -319,6 +319,23 @@
__MSG_whatQueryByHashPrefix__
+
+
+ +
+ + +
+
+ +
__MSG_whatRefetchWhenNotFound__
+
+

diff --git a/src/config.ts b/src/config.ts index 9d830116..264fbe0d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,7 +6,6 @@ const utils = new Utils(); interface SBConfig { userID: string, - // sponsorTimes: SBMap, segmentTimes: SBMap, defaultCategory: string, whitelistedChannels: string[], @@ -35,7 +34,8 @@ interface SBConfig { audioNotificationOnSkip, checkForUnlistedVideos: boolean, testingServer: boolean, - hashPrefix: boolean + hashPrefix: boolean, + refetchWhenNotFound: boolean, // What categories should be skipped categorySelections: CategorySelection[], @@ -168,6 +168,7 @@ var Config: SBObject = { checkForUnlistedVideos: false, testingServer: false, hashPrefix: false, + refetchWhenNotFound: true, categorySelections: [{ name: "sponsor", diff --git a/src/content.ts b/src/content.ts index 7661edc1..8787a2b7 100644 --- a/src/content.ts +++ b/src/content.ts @@ -632,20 +632,23 @@ function sponsorsLookup(id: string) { } getRequest.then(async (response: FetchResponse) => { if (response?.ok) { - let getResult = JSON.parse(response.responseText); + let result = JSON.parse(response.responseText); if (Config.config.hashPrefix) { - getResult = getResult.filter((video) => video.videoID === id); - if (getResult.length > 0) { - getResult = getResult[0].segments; - if (getResult.length === 0) { // return if no segments found + result = result.filter((video) => video.videoID === id); + if (result.length > 0) { + result = result[0].segments; + if (result.length === 0) { // return if no segments found + retryFetch(id); return; } } else { // return if no video found + console.log("refetching") + retryFetch(id); return; } } - let recievedSegments: SponsorTime[] = getResult; + let recievedSegments: SponsorTime[] = result; if (!recievedSegments.length) { console.error("[SponsorBlock] Server returned malformed response: " + JSON.stringify(recievedSegments)); return; @@ -689,32 +692,38 @@ function sponsorsLookup(id: string) { sponsorLookupRetries = 0; } else if (response?.status === 404) { - sponsorDataFound = false; - - //check if this video was uploaded recently - utils.wait(() => !!videoInfo).then(() => { - let dateUploaded = videoInfo?.microformat?.playerMicroformatRenderer?.uploadDate; - - //if less than 3 days old - if (Date.now() - new Date(dateUploaded).getTime() < 259200000) { - //TODO lower when server becomes better - setTimeout(() => sponsorsLookup(id), 180000); - } - }); - - sponsorLookupRetries = 0; + retryFetch(id); } else if (sponsorLookupRetries < 90 && !recheckStarted) { recheckStarted = true; //TODO lower when server becomes better (back to 1 second) //some error occurred, try again in a second - setTimeout(() => sponsorsLookup(id), 10000); + setTimeout(() => sponsorsLookup(id), 10000 + Math.random() * 30000); sponsorLookupRetries++; } }); } +function retryFetch(id: string): void { + if (!Config.config.refetchWhenNotFound) return; + + sponsorDataFound = false; + + //check if this video was uploaded recently + utils.wait(() => !!videoInfo).then(() => { + let dateUploaded = videoInfo?.microformat?.playerMicroformatRenderer?.uploadDate; + + //if less than 3 days old + if (Date.now() - new Date(dateUploaded).getTime() < 259200000) { + //TODO lower when server becomes better + setTimeout(() => sponsorsLookup(id), 120000); + } + }); + + sponsorLookupRetries = 0; +} + /** * Only should be used when it is okay to skip a sponsor when in the middle of it * From 633ae3b16f6b01a0ac32f387eb702a8be70675b9 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Sep 2020 12:39:48 -0400 Subject: [PATCH 10/10] Remove log --- src/content.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/content.ts b/src/content.ts index 8787a2b7..a529d727 100644 --- a/src/content.ts +++ b/src/content.ts @@ -642,7 +642,6 @@ function sponsorsLookup(id: string) { return; } } else { // return if no video found - console.log("refetching") retryFetch(id); return; }