Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into getUserInfo/param

src/config.ts

@@ -77,7 +77,8 @@ addDefaults(config, {
             name: "vipUsers"
         }]
     },
-    diskCache: null
+    diskCache: null,
+    crons: null
 });
 
 // Add defaults

src/cronjob/downvoteSegmentArchiveJob.ts

@@ -0,0 +1,67 @@
+import { CronJob } from "cron";
+
+import { config as serverConfig } from "../config";
+import { Logger } from "../utils/logger";
+import { db } from "../databases/databases";
+import { DBSegment } from "../types/segments.model";
+
+const jobConfig = serverConfig?.crons?.downvoteSegmentArchive;
+
+export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise<number> => {
+  const timeNow = runTime || new Date().getTime();
+  const threshold = dayLimit * 86400000;
+
+  Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`);
+  try {
+    // insert into archive sponsorTime
+    await db.prepare(
+      'run',
+      `INSERT INTO "archivedSponsorTimes" 
+        SELECT * 
+        FROM "sponsorTimes" 
+        WHERE "votes" < ? AND (? - "timeSubmitted") > ?`,
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when insert segment in archivedSponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  // remove from sponsorTime
+  try {
+    await db.prepare(
+      'run', 
+      'DELETE FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?',
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when deleting segment in sponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  Logger.info('DownvoteSegmentArchiveJob finished');
+  return 0;
+};
+
+const DownvoteSegmentArchiveJob = new CronJob(
+  jobConfig?.schedule || "0 0 * * * 0",
+  () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold)
+);
+
+if (serverConfig?.crons?.enabled && jobConfig && !jobConfig.schedule) {
+  Logger.error("Invalid cron schedule for downvoteSegmentArchive");
+}
+
+export default DownvoteSegmentArchiveJob;

src/cronjob/index.ts

@@ -0,0 +1,13 @@
+import { Logger } from "../utils/logger";
+import { config } from "../config";
+import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob";
+
+export function startAllCrons (): void {
+  if (config?.crons?.enabled) {
+    Logger.info("Crons started");
+
+    DownvoteSegmentArchiveJob.start();
+  } else {
+    Logger.info("Crons dissabled");
+  }
+}

src/index.ts

@@ -2,13 +2,17 @@ import {config} from "./config";
 import {initDb} from './databases/databases';
 import {createServer} from "./app";
 import {Logger} from "./utils/logger";
+import {startAllCrons} from "./cronjob";
 
 async function init() {
     await initDb();
 
     createServer(() => {
         Logger.info("Server started on port " + config.port + ".");
+
+        // ignite cron job after server created
+        startAllCrons();
     });
 }
- 
+
 init();

src/routes/getLockCategories.ts

@@ -13,11 +13,15 @@ export async function getLockCategories(req: Request, res: Response): Promise<Re
 
     try {
         // Get existing lock categories markers
-        const lockedCategories = await db.prepare('all', 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category}[];
-        if (lockedCategories.length === 0 || !lockedCategories[0]) return res.sendStatus(404);
-        // map to array in JS becaues of SQL incompatibilities
-        const categories = Object.values(lockedCategories).map((entry) => entry.category);
+        const row = await db.prepare('all', 'SELECT "category", "reason" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string}[];
+        // map categories to array in JS becaues of SQL incompatibilities
+        const categories = row.map(item => item.category);
+        if (categories.length === 0 || !categories[0]) return res.sendStatus(404);
+        // Get longest lock reason
+        const reason = row.map(item => item.reason)
+            .reduce((a,b) => (a.length > b.length) ? a : b); 
         return res.send({
+            reason,
             categories
         });
     } catch (err) {

src/routes/getLockCategoriesByHash.ts

@@ -7,13 +7,15 @@ import { Category, VideoID, VideoIDHash } from "../types/segments.model";
 interface LockResultByHash {
     videoID: VideoID,
     hash: VideoIDHash,
+    reason: string,
     categories: Category[]
 }
 
 interface DBLock {
     videoID: VideoID,
     hash: VideoIDHash,
-    category: Category
+    category: Category,
+    reason: string,
 }
 
 const mergeLocks = (source: DBLock[]) => {
@@ -22,12 +24,15 @@ const mergeLocks = (source: DBLock[]) => {
         // videoID already exists
         const destMatch = dest.find(s => s.videoID === obj.videoID);
         if (destMatch) {
+            // override longer reason
+            if (obj.reason.length > destMatch.reason.length) destMatch.reason = obj.reason;
             // push to categories
             destMatch.categories.push(obj.category);
         } else {
             dest.push({
                 videoID: obj.videoID,
                 hash: obj.hash,
+                reason: obj.reason,
                 categories: [obj.category]
             });
         }
@@ -45,7 +50,7 @@ export async function getLockCategoriesByHash(req: Request, res: Response): Prom
 
     try {
         // Get existing lock categories markers
-        const lockedRows = await db.prepare('all', 'SELECT "videoID", "hashedVideoID" as "hash", "category" from "lockCategories" where "hashedVideoID" LIKE ?', [hashPrefix + '%']) as DBLock[];
+        const lockedRows = await db.prepare('all', 'SELECT "videoID", "hashedVideoID" as "hash", "category", "reason" from "lockCategories" where "hashedVideoID" LIKE ?', [hashPrefix + '%']) as DBLock[];
         if (lockedRows.length === 0 || !lockedRows[0]) return res.sendStatus(404);
         // merge all locks
         return res.send(mergeLocks(lockedRows));

src/routes/getSkipSegments.ts

@@ -275,6 +275,10 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise<DBSeg
  */
 async function handleGetSegments(req: Request, res: Response): Promise<Segment[] | false> {
     const videoID = req.query.videoID as VideoID;
+    if (!videoID) {
+        res.status(400).send("videoID not specified");
+        return false;
+    }
     // Default to sponsor
     // If using params instead of JSON, only one category can be pulled
     const categories: Category[] = req.query.categories

src/routes/getSkipSegmentsByHash.ts

@@ -5,7 +5,7 @@ import { ActionType, Category, SegmentUUID, Service, VideoIDHash } from '../type
 
 export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
     let hashPrefix = req.params.prefix as VideoIDHash;
-    if (!hashPrefixTester(req.params.prefix)) {
+    if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) {
         return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
     }
     hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;

src/routes/postSkipSegments.ts

@@ -338,11 +338,13 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
     });
 
     const invalidFields = [];
+    const errors = [];
     if (typeof videoID !== 'string') {
       invalidFields.push('videoID');
     }
     if (typeof userID !== 'string' || userID.length < 30) {
       invalidFields.push('userID');
+      if (userID.length < 30) errors.push(`userID must be at least 30 characters long`);
     }
     if (!Array.isArray(segments) || segments.length < 1) {
       invalidFields.push('segments');
@@ -350,8 +352,9 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
 
     if (invalidFields.length !== 0) {
       // invalid request
-      const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, '');
-      return res.status(400).send(`No valid ${fields} field(s) provided`);
+      const formattedFields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, '');
+      const formattedErrors = errors.reduce((p, c, i) => p + (i !== 0 ? '. ' : ' ') + c, '');
+      return res.status(400).send(`No valid ${formattedFields} field(s) provided.${formattedErrors}`);
     }
 
     //hash the userID

src/routes/postWarning.ts

@@ -6,6 +6,8 @@ import {getHash} from '../utils/getHash';
 import { HashedUserID, UserID } from '../types/user.model';
 
 export async function postWarning(req: Request, res: Response): Promise<Response> {
+    // exit early if no body passed in
+    if (!req.body.userID && !req.body.issuerUserID) return res.status(400).json({"message": "Missing parameters"});
     // Collect user input data
     const issuerUserID: HashedUserID = getHash(<UserID> req.body.issuerUserID);
     const userID: UserID = req.body.userID;

src/routes/setUsername.ts

@@ -62,7 +62,7 @@ export async function setUsername(req: Request, res: Response): Promise<Response
         const locked = adminUserIDInput === undefined ? 0 : 1;
         let oldUserName = '';
 
-        if (row?.userName?.length > 0) {
+        if (row?.userName !== undefined) {
             //already exists, update this row
             oldUserName = row.userName;
             await db.prepare('run', `UPDATE "userNames" SET "userName" = ?, "locked" = ? WHERE "userID" = ?`, [userName, locked, userID]);

src/routes/voteOnSponsorTime.ts

@@ -258,6 +258,10 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
         //invalid request
         return res.sendStatus(400);
     }
+    if (paramUserID.length < 30 && config.mode !== "test") {
+        // Ignore this vote, invalid
+        return res.sendStatus(200);
+    }
 
     //hash the userID
     const nonAnonUserID = getHash(paramUserID);
@@ -428,7 +432,7 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
             if (isVIP && incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
                 // Unide and Lock this submission
                 await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 1, hidden = 0 WHERE "UUID" = ?', [UUID]);
-            } else if (isVIP && incrementAmount < 0 && voteTypeEnum === voteTypes.normal) {
+            } else if (isVIP && incrementAmount <= 0 && voteTypeEnum === voteTypes.normal) {
                  // Unlock if a VIP downvotes it
                  await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
             }

src/types/config.model.ts

@@ -44,6 +44,7 @@ export interface SBSConfig {
     postgres?: PoolConfig;
     dumpDatabase?: DumpDatabase;
     diskCache: CacheOptions;
+    crons: CronJobOptions;
 }
 
 export interface WebhookConfig {
@@ -81,3 +82,17 @@ export interface DumpDatabaseTable {
     name: string;
     order?: string;
 }
+
+export interface CronJobDefault {
+    schedule: string;
+}
+
+export interface CronJobOptions {
+    enabled: boolean;
+    downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron;
+}
+
+export interface DownvoteSegmentArchiveCron {
+    voteThreshold: number;
+    timeThresholdInDays: number;
+}

src/utils/redisKeys.ts

@@ -14,5 +14,5 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
 } 
 
 export function reputationKey(userID: UserID): string {
-    return "reputation.user." + userID;
+    return "reputation.user.v2." + userID;
 }

src/utils/reputation.ts

@@ -7,12 +7,14 @@ interface ReputationDBResult {
     totalSubmissions: number,
     downvotedSubmissions: number,
     nonSelfDownvotedSubmissions: number,
-    upvotedSum: number,
+    votedSum: number,
     lockedSum: number,
+    semiOldUpvotedSubmissions: number,
     oldUpvotedSubmissions: number
 }
 
 export async function getReputation(userID: UserID): Promise<number> {
+    const weekAgo = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago
     const pastDate = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago
     // 1596240000000 is August 1st 2020, a little after auto upvote was disabled
     const fetchFromDB = () => db.prepare("get", 
@@ -23,10 +25,11 @@ export async function getReputation(userID: UserID): Promise<number> {
 					 	WHERE b."userID" = ?
                             AND b."votes" > 0 AND b."category" = "a"."category" AND b."videoID" = "a"."videoID" LIMIT 1)
 					THEN 1 ELSE 0 END) AS "nonSelfDownvotedSubmissions",
-                SUM(CASE WHEN "votes" > 0 AND "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "upvotedSum",
+                SUM(CASE WHEN "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "votedSum",
                 SUM(locked) AS "lockedSum",
+                SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "semiOldUpvotedSubmissions",
                 SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions"
-            FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, pastDate, userID]) as Promise<ReputationDBResult>;
+            FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, weekAgo, pastDate, userID]) as Promise<ReputationDBResult>;
 
     const result = await QueryCacher.get(fetchFromDB, reputationKey(userID));
     
@@ -45,11 +48,19 @@ export async function getReputation(userID: UserID): Promise<number> {
         return convertRange(Math.min(nonSelfDownvoteRatio, 0.4), 0.05, 0.4, -0.5, -2.5);
     }
 
-    if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) {
+    if (result.votedSum < 5) {
         return 0;
     }
 
-    return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 20);
+    if (result.oldUpvotedSubmissions < 3) {
+        if (result.semiOldUpvotedSubmissions > 3) {
+            return convertRange(Math.min(result.votedSum, 150), 5, 150, 0, 2) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 5);
+        } else {
+            return 0;
+        }
+    }
+
+    return convertRange(Math.min(result.votedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 20);
 }
 
 function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number {