Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into getLockCategories/reason

src/config.ts

@@ -77,7 +77,8 @@ addDefaults(config, {
             name: "vipUsers"
         }]
     },
-    diskCache: null
+    diskCache: null,
+    crons: null
 });
 
 // Add defaults

src/cronjob/downvoteSegmentArchiveJob.ts

@@ -0,0 +1,63 @@
+import { CronJob } from "cron";
+
+import { config as serverConfig } from "../config";
+import { Logger } from "../utils/logger";
+import { db } from "../databases/databases";
+import { DBSegment } from "../types/segments.model";
+
+const jobConfig = serverConfig?.crons?.downvoteSegmentArchive;
+
+export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise<number> => {
+  const timeNow = runTime || new Date().getTime();
+  const threshold = dayLimit * 86400000;
+
+  Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`);
+  try {
+    // insert into archive sponsorTime
+    await db.prepare(
+      'run',
+      `INSERT INTO "archivedSponsorTimes" 
+        SELECT * 
+        FROM "sponsorTimes" 
+        WHERE "votes" < ? AND (? - "timeSubmitted") > ?`,
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when insert segment in archivedSponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  // remove from sponsorTime
+  try {
+    await db.prepare(
+      'run', 
+      'DELETE FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?',
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when deleting segment in sponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  Logger.info('DownvoteSegmentArchiveJob finished');
+  return 0;
+};
+
+const DownvoteSegmentArchiveJob = new CronJob(
+  jobConfig?.schedule || new Date(1),
+  () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold)
+);
+
+export default DownvoteSegmentArchiveJob;

src/cronjob/index.ts

@@ -0,0 +1,13 @@
+import { Logger } from "../utils/logger";
+import { config } from "../config";
+import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob";
+
+export function startAllCrons (): void {
+  if (config?.crons?.enabled) {
+    Logger.info("Crons started");
+
+    DownvoteSegmentArchiveJob.start();
+  } else {
+    Logger.info("Crons dissabled");
+  }
+}

src/index.ts

@@ -2,13 +2,17 @@ import {config} from "./config";
 import {initDb} from './databases/databases';
 import {createServer} from "./app";
 import {Logger} from "./utils/logger";
+import {startAllCrons} from "./cronjob";
 
 async function init() {
     await initDb();
 
     createServer(() => {
         Logger.info("Server started on port " + config.port + ".");
+
+        // ignite cron job after server created
+        startAllCrons();
     });
 }
- 
+
 init();

src/routes/getSkipSegments.ts

@@ -275,6 +275,10 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise<DBSeg
  */
 async function handleGetSegments(req: Request, res: Response): Promise<Segment[] | false> {
     const videoID = req.query.videoID as VideoID;
+    if (!videoID) {
+        res.status(400).send("videoID not specified");
+        return false;
+    }
     // Default to sponsor
     // If using params instead of JSON, only one category can be pulled
     const categories: Category[] = req.query.categories

src/routes/getSkipSegmentsByHash.ts

@@ -5,7 +5,7 @@ import { ActionType, Category, SegmentUUID, Service, VideoIDHash } from '../type
 
 export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
     let hashPrefix = req.params.prefix as VideoIDHash;
-    if (!hashPrefixTester(req.params.prefix)) {
+    if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) {
         return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
     }
     hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;

src/routes/postSkipSegments.ts

@@ -16,6 +16,7 @@ import { getCategoryActionType } from '../utils/categoryInfo';
 import { QueryCacher } from '../utils/queryCacher';
 import { getReputation } from '../utils/reputation';
 import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
+import { UserID } from '../types/user.model';
 
 async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
     const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -164,7 +165,7 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
 //   false for a pass - it was confusing and lead to this bug - any use of this function in
 //   the future could have the same problem.
 async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, 
-            submission: { videoID: any; userID: any; segments: any }) {
+            submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[] }) {
     if (apiVideoInfo) {
         const {err, data} = apiVideoInfo;
         if (err) return false;
@@ -234,7 +235,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
                     const startTime = parseFloat(segments[i].segment[0]);
                     const endTime = parseFloat(segments[i].segment[1]);
 
-                    const UUID = getSubmissionUUID(submission.videoID, segments[i].category, submission.userID, startTime, endTime);
+                    const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime);
                     // Send to Discord
                     // Note, if this is too spammy. Consider sending all the segments as one Webhook
                     sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
@@ -519,7 +520,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
             //this can just be a hash of the data
             //it's better than generating an actual UUID like what was used before
             //also better for duplication checking
-            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
+            const UUID = getSubmissionUUID(videoID, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
             const hashedVideoID = getHash(videoID, 1);
 
             const startingLocked = isVIP ? 1 : 0;

src/routes/postWarning.ts

@@ -6,6 +6,8 @@ import {getHash} from '../utils/getHash';
 import { HashedUserID, UserID } from '../types/user.model';
 
 export async function postWarning(req: Request, res: Response): Promise<Response> {
+    // exit early if no body passed in
+    if (!req.body.userID && !req.body.issuerUserID) return res.status(400).json({"message": "Missing parameters"});
     // Collect user input data
     const issuerUserID: HashedUserID = getHash(<UserID> req.body.issuerUserID);
     const userID: UserID = req.body.userID;

src/types/config.model.ts

@@ -44,6 +44,7 @@ export interface SBSConfig {
     postgres?: PoolConfig;
     dumpDatabase?: DumpDatabase;
     diskCache: CacheOptions;
+    crons: CronJobOptions;
 }
 
 export interface WebhookConfig {
@@ -81,3 +82,17 @@ export interface DumpDatabaseTable {
     name: string;
     order?: string;
 }
+
+export interface CronJobDefault {
+    schedule: string;
+}
+
+export interface CronJobOptions {
+    enabled: boolean;
+    downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron;
+}
+
+export interface DownvoteSegmentArchiveCron {
+    voteThreshold: number;
+    timeThresholdInDays: number;
+}

src/utils/getSubmissionUUID.ts

@@ -1,6 +1,8 @@
 import {getHash} from './getHash';
 import { HashedValue } from '../types/hash.model';
+import { ActionType, Category, VideoID } from '../types/segments.model';
+import { UserID } from '../types/user.model';
 
-export function getSubmissionUUID(videoID: string, category: string, userID: string, startTime: number, endTime: number): HashedValue{
-    return getHash('v2-categories' + videoID + startTime + endTime + category + userID, 1);
+export function getSubmissionUUID(videoID: VideoID, actionType: ActionType, userID: UserID, startTime: number, endTime: number): HashedValue{
+    return `3${getHash('v3' + videoID + startTime + endTime + userID, 1)}` as HashedValue;
 }

src/utils/redisKeys.ts

@@ -14,5 +14,5 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
 } 
 
 export function reputationKey(userID: UserID): string {
-    return "reputation.user." + userID;
+    return "reputation.user.v2." + userID;
 }

src/utils/reputation.ts

@@ -7,12 +7,14 @@ interface ReputationDBResult {
     totalSubmissions: number,
     downvotedSubmissions: number,
     nonSelfDownvotedSubmissions: number,
-    upvotedSum: number,
+    votedSum: number,
     lockedSum: number,
+    semiOldUpvotedSubmissions: number,
     oldUpvotedSubmissions: number
 }
 
 export async function getReputation(userID: UserID): Promise<number> {
+    const weekAgo = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago
     const pastDate = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago
     // 1596240000000 is August 1st 2020, a little after auto upvote was disabled
     const fetchFromDB = () => db.prepare("get", 
@@ -23,10 +25,11 @@ export async function getReputation(userID: UserID): Promise<number> {
 					 	WHERE b."userID" = ?
                             AND b."votes" > 0 AND b."category" = "a"."category" AND b."videoID" = "a"."videoID" LIMIT 1)
 					THEN 1 ELSE 0 END) AS "nonSelfDownvotedSubmissions",
-                SUM(CASE WHEN "votes" > 0 AND "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "upvotedSum",
+                SUM(CASE WHEN "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "votedSum",
                 SUM(locked) AS "lockedSum",
+                SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "semiOldUpvotedSubmissions",
                 SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions"
-            FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, pastDate, userID]) as Promise<ReputationDBResult>;
+            FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, weekAgo, pastDate, userID]) as Promise<ReputationDBResult>;
 
     const result = await QueryCacher.get(fetchFromDB, reputationKey(userID));
     
@@ -45,11 +48,19 @@ export async function getReputation(userID: UserID): Promise<number> {
         return convertRange(Math.min(nonSelfDownvoteRatio, 0.4), 0.05, 0.4, -0.5, -2.5);
     }
 
-    if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) {
+    if (result.votedSum < 5) {
         return 0;
     }
 
-    return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 20);
+    if (result.oldUpvotedSubmissions < 3) {
+        if (result.semiOldUpvotedSubmissions > 3) {
+            return convertRange(Math.min(result.votedSum, 150), 5, 150, 0, 2) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 5);
+        } else {
+            return 0;
+        }
+    }
+
+    return convertRange(Math.min(result.votedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 20);
 }
 
 function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number {