Merge pull request #541 from mchangrh/etagTest

add etag and other tests
2025-12-15 16:07:03 +03:00 · 2023-02-22 01:38:41 -05:00
parent f4286b15a1 436e75e3e6
commit d76ee7cd22
42 changed files with 1536 additions and 1070 deletions
--- a/src/middleware/userCounter.ts
+++ b/src/middleware/userCounter.ts
@@ -14,7 +14,7 @@ export function userCounter(req: Request, res: Response, next: NextFunction): vo
                method: "post",
                url: `${config.userCounterURL}/api/v1/addIP?hashedIP=${getIP(req)}`,
                httpAgent
-            }).catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
+            }).catch(() => /* instanbul skip next */ Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
        }
    }

--- a/src/routes/getSegmentInfo.ts
+++ b/src/routes/getSegmentInfo.ts
@@ -34,11 +34,11 @@ async function handleGetSegmentInfo(req: Request, res: Response): Promise<DBSegm
    // deduplicate with set
    UUIDs = [ ...new Set(UUIDs)];
    // if more than 10 entries, slice
-    if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
-    if (!Array.isArray(UUIDs) || !UUIDs) {
+    if (!Array.isArray(UUIDs) || !UUIDs?.length) {
        res.status(400).send("UUIDs parameter does not match format requirements.");
        return;
    }
+    if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
    const DBSegments = await getSegmentsByUUID(UUIDs);
    // all uuids failed lookup
    if (!DBSegments?.length) {
--- a/src/routes/getSkipSegmentsByHash.ts
+++ b/src/routes/getSkipSegmentsByHash.ts
@@ -25,13 +25,13 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
    try {
        await getEtag("skipSegmentsHash", hashPrefix, service)
            .then(etag => res.set("ETag", etag))
-            .catch(() => null);
+            .catch(/* istanbul ignore next */ () => null);
        const output = Object.entries(segments).map(([videoID, data]) => ({
            videoID,
            segments: data.segments,
        }));
        return res.status(output.length === 0 ? 404 : 200).json(output);
-    } catch(e) {
+    } catch (e) /* istanbul ignore next */ {
        Logger.error(`skip segments by hash error: ${e}`);

        return res.status(500).send("Internal server error");
--- a/src/routes/getTopCategoryUsers.ts
+++ b/src/routes/getTopCategoryUsers.ts
@@ -2,10 +2,12 @@ import { db } from "../databases/databases";
 import { createMemoryCache } from "../utils/createMemoryCache";
 import { config } from "../config";
 import { Request, Response } from "express";
+import { validateCategories } from "../utils/parseParams";

 const MILLISECONDS_IN_MINUTE = 60000;
 // eslint-disable-next-line @typescript-eslint/no-misused-promises
 const getTopCategoryUsersWithCache = createMemoryCache(generateTopCategoryUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
+/* istanbul ignore next */
 const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;

 interface DBSegment {
@@ -38,7 +40,6 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
        }
    }

-
    return {
        userNames,
        viewCounts,
@@ -51,7 +52,7 @@ export async function getTopCategoryUsers(req: Request, res: Response): Promise<
    const sortType = parseInt(req.query.sortType as string);
    const category = req.query.category as string;

-    if (sortType == undefined || !config.categoryList.includes(category) ) {
+    if (sortType == undefined || !validateCategories([category]) ) {
        //invalid request
        return res.sendStatus(400);
    }
--- a/src/routes/getTotalStats.ts
+++ b/src/routes/getTotalStats.ts
@@ -3,6 +3,7 @@ import { config } from "../config";
 import { Request, Response } from "express";
 import axios from "axios";
 import { Logger } from "../utils/logger";
+import { getCWSUsers } from "../utils/getCWSUsers";

 // A cache of the number of chrome web store users
 let chromeUsersCache = 0;
@@ -29,30 +30,30 @@ let lastFetch: DBStatsData = {
 updateExtensionUsers();

 export async function getTotalStats(req: Request, res: Response): Promise<void> {
-
-    const row = await getStats(!!req.query.countContributingUsers);
+    const countContributingUsers = Boolean(req.query?.countContributingUsers == "true");
+    const row = await getStats(countContributingUsers);
    lastFetch = row;

-    if (row !== undefined) {
-        const extensionUsers = chromeUsersCache + firefoxUsersCache;
+    /* istanbul ignore if */
+    if (!row) res.sendStatus(500);
+    const extensionUsers = chromeUsersCache + firefoxUsersCache;

-        //send this result
-        res.send({
-            userCount: row.userCount,
-            activeUsers: extensionUsers,
-            apiUsers: Math.max(apiUsersCache, extensionUsers),
-            viewCount: row.viewCount,
-            totalSubmissions: row.totalSubmissions,
-            minutesSaved: row.minutesSaved,
-        });
+    //send this result
+    res.send({
+        userCount: row.userCount ?? 0,
+        activeUsers: extensionUsers,
+        apiUsers: Math.max(apiUsersCache, extensionUsers),
+        viewCount: row.viewCount,
+        totalSubmissions: row.totalSubmissions,
+        minutesSaved: row.minutesSaved,
+    });

-        // Check if the cache should be updated (every ~14 hours)
-        const now = Date.now();
-        if (now - lastUserCountCheck > 5000000) {
-            lastUserCountCheck = now;
+    // Check if the cache should be updated (every ~14 hours)
+    const now = Date.now();
+    if (now - lastUserCountCheck > 5000000) {
+        lastUserCountCheck = now;

-            updateExtensionUsers();
-        }
+        updateExtensionUsers();
    }
 }

@@ -67,42 +68,53 @@ function getStats(countContributingUsers: boolean): Promise<DBStatsData> {
    }
 }

-
 function updateExtensionUsers() {
+    /* istanbul ignore else */
    if (config.userCounterURL) {
        axios.get(`${config.userCounterURL}/api/v1/userCount`)
-            .then(res => {
-                apiUsersCache = Math.max(apiUsersCache, res.data.userCount);
-            })
-            .catch(() => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
+            .then(res => apiUsersCache = Math.max(apiUsersCache, res.data.userCount))
+            .catch( /* istanbul ignore next */ () => Logger.debug(`Failing to connect to user counter at: ${config.userCounterURL}`));
    }

    const mozillaAddonsUrl = "https://addons.mozilla.org/api/v3/addons/addon/sponsorblock/";
    const chromeExtensionUrl = "https://chrome.google.com/webstore/detail/sponsorblock-for-youtube/mnjggcdmjocbbbhaepdhchncahnbgone";
+    const chromeExtId = "mnjggcdmjocbbbhaepdhchncahnbgone";

    axios.get(mozillaAddonsUrl)
-        .then(res => {
-            firefoxUsersCache = res.data.average_daily_users;
-            axios.get(chromeExtensionUrl)
-                .then(res => {
-                    const body = res.data;
-                    // 2021-01-05
-                    // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
-                    const matchingString = '"UserDownloads:';
-                    const matchingStringLen = matchingString.length;
-                    const userDownloadsStartIndex = body.indexOf(matchingString);
-                    if (userDownloadsStartIndex >= 0) {
-                        const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
-                        const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
-                        chromeUsersCache = parseInt(userDownloadsStr);
-                    }
-                    else {
-                        lastUserCountCheck = 0;
-                    }
-                })
-                .catch(() => Logger.debug(`Failing to connect to ${chromeExtensionUrl}`));
-        })
-        .catch(() => {
+        .then(res => firefoxUsersCache = res.data.average_daily_users )
+        .catch( /* istanbul ignore next */ () => {
            Logger.debug(`Failing to connect to ${mozillaAddonsUrl}`);
+            return 0;
        });
+    getCWSUsers(chromeExtId)
+        .then(res => chromeUsersCache = res)
+        .catch(/* istanbul ignore next */ () =>
+            getChromeUsers(chromeExtensionUrl)
+                .then(res => chromeUsersCache = res)
+        );
 }
+
+/* istanbul ignore next */
+function getChromeUsers(chromeExtensionUrl: string): Promise<number> {
+    return axios.get(chromeExtensionUrl)
+        .then(res => {
+            const body = res.data;
+            // 2021-01-05
+            // [...]<span><meta itemprop="interactionCount" content="UserDownloads:100.000+"/><meta itemprop="opera[...]
+            const matchingString = '"UserDownloads:';
+            const matchingStringLen = matchingString.length;
+            const userDownloadsStartIndex = body.indexOf(matchingString);
+            /* istanbul ignore else */
+            if (userDownloadsStartIndex >= 0) {
+                const closingQuoteIndex = body.indexOf('"', userDownloadsStartIndex + matchingStringLen);
+                const userDownloadsStr = body.substr(userDownloadsStartIndex + matchingStringLen, closingQuoteIndex - userDownloadsStartIndex).replace(",", "").replace(".", "");
+                return parseInt(userDownloadsStr);
+            } else {
+                lastUserCountCheck = 0;
+            }
+        })
+        .catch(/* istanbul ignore next */ () => {
+            Logger.debug(`Failing to connect to ${chromeExtensionUrl}`);
+            return 0;
+        });
+}
--- a/src/routes/postSkipSegments.ts
+++ b/src/routes/postSkipSegments.ts
@@ -120,7 +120,7 @@ async function sendWebhooks(apiVideoDetails: videoDetails, userID: string, video
 //   false for a pass - it was confusing and lead to this bug - any use of this function in
 //   the future could have the same problem.
 async function autoModerateSubmission(apiVideoDetails: videoDetails,
-    submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
+    submission: { videoID: VideoID; userID: HashedUserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
    // get duration from API
    const apiDuration = apiVideoDetails.duration;
    // if API fail or returns 0, get duration from client
@@ -156,7 +156,7 @@ async function autoModerateSubmission(apiVideoDetails: videoDetails,
    return false;
 }

-async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
+async function checkUserActiveWarning(userID: HashedUserID): Promise<CheckResult> {
    const MILLISECONDS_IN_HOUR = 3600000;
    const now = Date.now();
    const warnings = (await db.prepare("all",
@@ -337,10 +337,10 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
    return CHECK_PASS;
 }

-async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
+async function checkByAutoModerator(videoID: VideoID, userID: HashedUserID, segments: IncomingSegment[], service: Service, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
    // Auto moderator check
    if (service == Service.YouTube) {
-        const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { userID, videoID, segments, service, videoDuration });
+        const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { videoID, userID, segments, service, videoDuration });
        if (autoModerateResult) {
            return {
                pass: false,
@@ -492,7 +492,10 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
    let { videoID, userID: paramUserID, service, videoDuration, videoDurationParam, segments, userAgent } = preprocessInput(req);

    //hash the userID
-    const userID = await getHashCache(paramUserID || "");
+    if (!paramUserID) {
+        return res.status(400).send("No userID provided");
+    }
+    const userID: HashedUserID = await getHashCache(paramUserID);

    const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent, service);
    if (!invalidCheckResult.pass) {
--- a/src/utils/createMemoryCache.ts
+++ b/src/utils/createMemoryCache.ts
@@ -1,4 +1,5 @@
 export function createMemoryCache(memoryFn: (...args: any[]) => void, cacheTimeMs: number): any {
+    /* istanbul ignore if */
    if (isNaN(cacheTimeMs)) cacheTimeMs = 0;

    // holds the promise results
--- a/src/utils/getCWSUsers.ts
+++ b/src/utils/getCWSUsers.ts
@@ -0,0 +1,13 @@
+import axios from "axios";
+import { Logger } from "../utils/logger";
+
+export const getCWSUsers = (extID: string): Promise<number | undefined> =>
+    axios.post(`https://chrome.google.com/webstore/ajax/detail?pv=20210820&id=${extID}`)
+        .then(res => res.data.split("\n")[2])
+        .then(data => JSON.parse(data))
+        .then(data => (data[1][1][0][23]).replaceAll(/,|\+/g,""))
+        .then(data => parseInt(data))
+        .catch((err) => {
+            Logger.error(`Error getting chrome users - ${err}`);
+            return 0;
+        });
--- a/src/utils/getHashCache.ts
+++ b/src/utils/getHashCache.ts
@@ -28,7 +28,7 @@ async function getFromRedis<T extends string>(key: HashedValue): Promise<T & Has
                Logger.debug(`Got data from redis: ${reply}`);
                return reply as T & HashedValue;
            }
-        } catch (err) {
+        } catch (err) /* istanbul ignore next */ {
            Logger.error(err as string);
        }
    }
@@ -37,7 +37,7 @@ async function getFromRedis<T extends string>(key: HashedValue): Promise<T & Has
    const data = getHash(key, cachedHashTimes);

    if (!config.redis?.disableHashCache) {
-        redis.set(redisKey, data).catch((err) => Logger.error(err));
+        redis.set(redisKey, data).catch(/* istanbul ignore next */ (err) => Logger.error(err));
    }

    return data as T & HashedValue;
--- a/src/utils/innerTubeAPI.ts
+++ b/src/utils/innerTubeAPI.ts
@@ -70,15 +70,16 @@ export async function getPlayerData (videoID: string, ignoreCache = false): Prom
    }
    try {
        const data = await getFromITube(videoID)
-            .catch(err => {
+            .catch(/* istanbul ignore next */ err => {
                Logger.warn(`InnerTube API Error for ${videoID}: ${err}`);
                return Promise.reject(err);
            });
        DiskCache.set(cacheKey, data)
            .then(() => Logger.debug(`InnerTube API: video information cache set for: ${videoID}`))
-            .catch((err: any) => Logger.warn(err));
+            .catch(/* istanbul ignore next */ (err: any) => Logger.warn(err));
        return data;
    } catch (err) {
+        /* istanbul ignore next */
        return Promise.reject(err);
    }
 }
--- a/src/utils/isUserTempVIP.ts
+++ b/src/utils/isUserTempVIP.ts
@@ -11,7 +11,7 @@ export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID
    try {
        const reply = await redis.get(tempVIPKey(hashedUserID));
        return reply && reply == channelID;
-    } catch (e) {
+    } catch (e) /* istanbul ignore next */ {
        Logger.error(e as string);
        return false;
    }
--- a/src/utils/logger.ts
+++ b/src/utils/logger.ts
@@ -45,6 +45,7 @@ class Logger {
    };

    constructor() {
+        /* istanbul ignore if */
        if (config.mode === "development") {
            this._settings.INFO = true;
            this._settings.DEBUG = true;
@@ -73,9 +74,11 @@ class Logger {

        let color = colors.Bright;
        if (level === LogLevel.ERROR) color = colors.FgRed;
+        /* istanbul ignore if */
        if (level === LogLevel.WARN) color = colors.FgYellow;

        let levelStr = level.toString();
+        /* istanbul ignore if */
        if (levelStr.length === 4) {
            levelStr += " ";  // ensure logs are aligned
        }
--- a/src/utils/parseParams.ts
+++ b/src/utils/parseParams.ts
@@ -73,3 +73,6 @@ export const parseActionTypes = (req: Request, fallback: ActionType[]): ActionTy

 export const parseRequiredSegments = (req: Request): SegmentUUID[] | undefined =>
    syntaxErrorWrapper(getRequiredSegments, req, []); // never fall back
+
+export const validateCategories = (categories: string[]): boolean =>
+    categories.every((category: string) => config.categoryList.includes(category));
--- a/src/utils/redis.ts
+++ b/src/utils/redis.ts
@@ -135,17 +135,21 @@ if (config.redis?.enabled) {
            .then((reply) => resolve(reply))
            .catch((err) => reject(err))
    );
+    /* istanbul ignore next */
    client.on("error", function(error) {
        lastClientFail = Date.now();
        Logger.error(`Redis Error: ${error}`);
    });
+    /* istanbul ignore next */
    client.on("reconnect", () => {
        Logger.info("Redis: trying to reconnect");
    });
+    /* istanbul ignore next */
    readClient?.on("error", function(error) {
        lastReadFail = Date.now();
        Logger.error(`Redis Read-Only Error: ${error}`);
    });
+    /* istanbul ignore next */
    readClient?.on("reconnect", () => {
        Logger.info("Redis Read-Only: trying to reconnect");
    });