From 1bda331b0ce4351ed650251cee06bdc8d538ddfd Mon Sep 17 00:00:00 2001 From: Michael C Date: Mon, 20 Feb 2023 22:21:53 -0500 Subject: [PATCH] add new CWS user parsing method --- src/routes/getTotalStats.ts | 67 +++++++++++++++++++------------------ src/utils/getCWSUsers.ts | 13 +++++++ test/cases/userCounter.ts | 10 ++++++ 3 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 src/utils/getCWSUsers.ts diff --git a/src/routes/getTotalStats.ts b/src/routes/getTotalStats.ts index 187a59d..4a89b8f 100644 --- a/src/routes/getTotalStats.ts +++ b/src/routes/getTotalStats.ts @@ -3,6 +3,7 @@ import { config } from "../config"; import { Request, Response } from "express"; import axios from "axios"; import { Logger } from "../utils/logger"; +import { getCWSUsers } from "../utils/getCWSUsers"; // A cache of the number of chrome web store users let chromeUsersCache = 0; @@ -26,10 +27,7 @@ let lastFetch: DBStatsData = { minutesSaved: 0 }; -updateExtensionUsers(); - export async function getTotalStats(req: Request, res: Response): Promise { - const row = await getStats(!!req.query.countContributingUsers); lastFetch = row; @@ -51,7 +49,7 @@ export async function getTotalStats(req: Request, res: Response): Promise if (now - lastUserCountCheck > 5000000) { lastUserCountCheck = now; - updateExtensionUsers(); + await updateExtensionUsers(); } } } @@ -68,41 +66,46 @@ function getStats(countContributingUsers: boolean): Promise { } -function updateExtensionUsers() { +async function updateExtensionUsers() { if (config.userCounterURL) { axios.get(`${config.userCounterURL}/api/v1/userCount`) - .then(res => { - apiUsersCache = Math.max(apiUsersCache, res.data.userCount); - }) - .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`)); + .then(res => apiUsersCache = Math.max(apiUsersCache, res.data.userCount)) + .catch( /* istanbul ignore next */ () => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`)); } const mozillaAddonsUrl = "https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/"; const chromeExtensionUrl = "https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone"; + const chromeExtId = "mnjggcdmjocbbbhaepdhchncahnbgone"; - axios.get(mozillaAddonsUrl) - .then(res => { - firefoxUsersCache = res.data.average_daily_users; - axios.get(chromeExtensionUrl) - .then(res => { - const body = res.data; - // 2021-01-05 - // [...]= 0) { - const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen); - const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", ""); - chromeUsersCache = parseInt(userDownloadsStr); - } - else { - lastUserCountCheck = 0; - } - }) - .catch(() => Logger.debug(`Failing to connect to ${chromeExtensionUrl}`)); - }) - .catch(() => { + firefoxUsersCache = await axios.get(mozillaAddonsUrl) + .then(res => res.data.average_daily_users ) + .catch( /* istanbul ignore next */ () => { Logger.debug(`Failing to connect to ${mozillaAddonsUrl}`); + return 0; }); + chromeUsersCache = await getCWSUsers(chromeExtId) ?? await getChromeUsers(chromeExtensionUrl); } + +function getChromeUsers(chromeExtensionUrl: string): Promise { + return axios.get(chromeExtensionUrl) + .then(res => { + const body = res.data; + // 2021-01-05 + // [...]= 0) { + const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen); + const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", ""); + return parseInt(userDownloadsStr); + } else { + lastUserCountCheck = 0; + } + }) + .catch(/* istanbul ignore next */ () => { + Logger.debug(`Failing to connect to ${chromeExtensionUrl}`); + return 0; + }); +} \ No newline at end of file diff --git a/src/utils/getCWSUsers.ts b/src/utils/getCWSUsers.ts new file mode 100644 index 0000000..b71821d --- /dev/null +++ b/src/utils/getCWSUsers.ts @@ -0,0 +1,13 @@ +import axios from "axios"; +import { Logger } from "../utils/logger"; + +export const getCWSUsers = (extID: string) => + axios.post(`https://chrome.google.com/webstore/ajax/detail?pv=20210820&id=${extID}`) + .then((res) => res.data.split("\n")[2]) + .then((data) => JSON.parse(data)) + .then((data) => (data[1][1][0][23]).replaceAll(/,|\+/,"")) + .then((data) => parseInt(data)) + .catch((err) => { + Logger.error(`Error getting chrome users - ${err}`); + return undefined; + }); \ No newline at end of file diff --git a/test/cases/userCounter.ts b/test/cases/userCounter.ts index 8215ce1..1730d26 100644 --- a/test/cases/userCounter.ts +++ b/test/cases/userCounter.ts @@ -2,6 +2,7 @@ import axios from "axios"; import assert from "assert"; import { config } from "../../src/config"; import { getHash } from "../../src/utils/getHash"; +import { client } from "../utils/httpClient"; describe("userCounter", () => { it("Should return 200", function (done) { @@ -20,4 +21,13 @@ describe("userCounter", () => { }) .catch(err => done(err)); }); + it("Should not incremeent counter on OPTIONS", function (done) { + /* cannot spy test */ + if (!config.userCounterURL) this.skip(); // skip if no userCounterURL is set + //const spy = sinon.spy(UserCounter); + client({ method: "OPTIONS", url: "/api/status" }) + .then(() => client({ method: "GET", url: "/api/status" })); + //assert.strictEqual(spy.callCount, 1); + done(); + }); }); \ No newline at end of file