add new CWS user parsing method

src/routes/getTotalStats.ts

@@ -3,6 +3,7 @@ import { config } from "../config";
 import { Request, Response } from "express";
 import axios from "axios";
 import { Logger } from "../utils/logger";
+import { getCWSUsers } from "../utils/getCWSUsers";
 
 // A cache of the number of chrome web store users
 let chromeUsersCache = 0;
@@ -26,10 +27,7 @@ let lastFetch: DBStatsData = {
     minutesSaved: 0
 };
 
-updateExtensionUsers();
-
 export async function getTotalStats(req: Request, res: Response): Promise<void> {
-
     const row = await getStats(!!req.query.countContributingUsers);
     lastFetch = row;
 
@@ -51,7 +49,7 @@ export async function getTotalStats(req: Request, res: Response): Promise<void>
         if (now - lastUserCountCheck > 5000000) {
             lastUserCountCheck = now;
 
-            updateExtensionUsers();
+            await updateExtensionUsers();
         }
     }
 }
@@ -68,41 +66,46 @@ function getStats(countContributingUsers: boolean): Promise<DBStatsData> {
 }
 
 
-function updateExtensionUsers() {
+async function updateExtensionUsers() {
     if (config.userCounterURL) {
         axios.get(`${config.userCounterURL}/api/v1/userCount`)
-            .then(res => {
+            .then(res => apiUsersCache = Math.max(apiUsersCache, res.data.userCount))
-                apiUsersCache = Math.max(apiUsersCache, res.data.userCount);
+            .catch( /* istanbul ignore next */ () => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
-            })
-            .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
     }
 
     const mozillaAddonsUrl = "https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/";
     const chromeExtensionUrl = "https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone";
+    const chromeExtId = "mnjggcdmjocbbbhaepdhchncahnbgone";
 
-    axios.get(mozillaAddonsUrl)
+    firefoxUsersCache = await axios.get(mozillaAddonsUrl)
-        .then(res => {
+        .then(res => res.data.average_daily_users )
-            firefoxUsersCache = res.data.average_daily_users;
+        .catch( /* istanbul ignore next */ () => {
-            axios.get(chromeExtensionUrl)
-                .then(res => {
-                    const body = res.data;
-                    // 2021-01-05
-                    // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
-                    const matchingString = '"UserDownloads:';
-                    const matchingStringLen = matchingString.length;
-                    const userDownloadsStartIndex = body.indexOf(matchingString);
-                    if (userDownloadsStartIndex >= 0) {
-                        const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
-                        const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
-                        chromeUsersCache = parseInt(userDownloadsStr);
-                    }
-                    else {
-                        lastUserCountCheck = 0;
-                    }
-                })
-                .catch(() => Logger.debug(`Failing to connect to ${chromeExtensionUrl}`));
-        })
-        .catch(() => {
             Logger.debug(`Failing to connect to ${mozillaAddonsUrl}`);
+            return 0;
+        });
+    chromeUsersCache = await getCWSUsers(chromeExtId) ?? await getChromeUsers(chromeExtensionUrl);
+}
+
+function getChromeUsers(chromeExtensionUrl: string): Promise<number> {
+    return axios.get(chromeExtensionUrl)
+        .then(res => {
+            const body = res.data;
+            // 2021-01-05
+            // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
+            const matchingString = '"UserDownloads:';
+            const matchingStringLen = matchingString.length;
+            const userDownloadsStartIndex = body.indexOf(matchingString);
+            /* istanbul ignore else */
+            if (userDownloadsStartIndex >= 0) {
+                const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
+                const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
+                return parseInt(userDownloadsStr);
+            } else {
+                lastUserCountCheck = 0;
+            }
+        })
+        .catch(/* istanbul ignore next */ () => {
+            Logger.debug(`Failing to connect to ${chromeExtensionUrl}`);
+            return 0;
         });
 }

src/utils/getCWSUsers.ts

@@ -0,0 +1,13 @@
+import axios from "axios";
+import { Logger } from "../utils/logger";
+
+export const getCWSUsers = (extID: string) =>
+    axios.post(`https://chrome.google.com/webstore/ajax/detail?pv=20210820&id=${extID}`)
+        .then((res) => res.data.split("\n")[2])
+        .then((data) => JSON.parse(data))
+        .then((data) => (data[1][1][0][23]).replaceAll(/,|\+/,""))
+        .then((data) => parseInt(data))
+        .catch((err) => {
+            Logger.error(`Error getting chrome users - ${err}`);
+            return undefined;
+        });

test/cases/userCounter.ts

@@ -2,6 +2,7 @@ import axios from "axios";
 import assert from "assert";
 import { config } from "../../src/config";
 import { getHash } from "../../src/utils/getHash";
+import { client } from "../utils/httpClient";
 
 describe("userCounter", () => {
     it("Should return 200", function (done) {
@@ -20,4 +21,13 @@ describe("userCounter", () => {
             })
             .catch(err => done(err));
     });
+    it("Should not incremeent counter on OPTIONS", function (done) {
+        /* cannot spy test */
+        if (!config.userCounterURL) this.skip(); // skip if no userCounterURL is set
+        //const spy = sinon.spy(UserCounter);
+        client({ method: "OPTIONS", url: "/api/status" })
+            .then(() => client({ method: "GET", url: "/api/status" }));
+        //assert.strictEqual(spy.callCount, 1);
+        done();
+    });
 });