Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer

2025-12-08 20:47:02 +03:00 · 2021-07-12 11:05:14 -04:00
parent a23ec160c0 9445a06871
commit 3d0b9edb9c
9 changed files with 321 additions and 2 deletions
--- a/DatabaseSchema.md
+++ b/DatabaseSchema.md
@@ -10,6 +10,7 @@
 [shadowBannedUsers](#shadowBannedUsers)  
 [unlistedVideos](#unlistedVideos)  
 [config](#config)  
+[archivedSponsorTimes](#archivedSponsorTimes)

 ### vipUsers
 | Name | Type | |
@@ -35,6 +36,7 @@
 | timeSubmitted | INTEGER | not null |
 | views | INTEGER | not null |
 | category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
 | service | TEXT | not null, default 'Youtube' |
 | videoDuration | INTEGER | not null, default '0' |
 | hidden | INTEGER | not null, default '0' |
@@ -140,7 +142,28 @@
 | key | TEXT | not null, unique |
 | value | TEXT | not null |

+### archivedSponsorTimes

+| Name | Type | |
+| -- | :--: | -- |
+| videoID | TEXT | not null |
+| startTime | REAL | not null |
+| endTime | REAL | not null |
+| votes | INTEGER | not null |
+| locked | INTEGER | not null, default '0' |
+| incorrectVotes | INTEGER | not null, default 1 |
+| UUID | TEXT | not null, unique |
+| userID | TEXT | not null |
+| timeSubmitted | INTEGER | not null |
+| views | INTEGER | not null |
+| category | TEXT | not null, default 'sponsor' |
+| actionType | TEXT | not null, default 'skip' |
+| service | TEXT | not null, default 'Youtube' |
+| videoDuration | INTEGER | not null, default '0' |
+| hidden | INTEGER | not null, default '0' |
+| reputation | REAL | not null, default '0' |
+| shadowHidden | INTEGER | not null |
+| hashedVideoID | TEXT | not null, default '', sha256 |

 # Private 

--- a/databases/_upgrade_sponsorTimes_21.sql
+++ b/databases/_upgrade_sponsorTimes_21.sql
@@ -0,0 +1,26 @@
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" (
+	"videoID" TEXT NOT NULL,
+	"startTime" REAL NOT NULL,
+	"endTime" REAL NOT NULL,
+	"votes" INTEGER NOT NULL,
+	"locked" INTEGER NOT NULL DEFAULT '0',
+	"incorrectVotes" INTEGER NOT NULL DEFAULT 1,
+	"UUID" TEXT NOT NULL UNIQUE,
+	"userID" TEXT NOT NULL,
+	"timeSubmitted" INTEGER NOT NULL,
+	"views" INTEGER NOT NULL,
+	"category" TEXT NOT NULL DEFAULT 'sponsor',
+	"service" TEXT NOT NULL DEFAULT 'Youtube',
+	"actionType" TEXT NOT NULL DEFAULT 'skip',
+	"videoDuration" INTEGER NOT NULL DEFAULT '0',
+	"hidden" INTEGER NOT NULL DEFAULT '0',
+	"reputation" REAL NOT NULL DEFAULT '0',
+	"shadowHidden" INTEGER NOT NULL,
+	"hashedVideoID" TEXT NOT NULL DEFAULT ''
+);
+
+UPDATE "config" SET value = 21 WHERE key = 'version';
+
+COMMIT;
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
    "@ajayyy/lru-diskcache": "^1.1.9",
    "@types/request": "^2.48.6",
    "better-sqlite3": "^7.4.1",
+    "cron": "^1.8.2",
    "express": "^4.17.1",
    "express-promise-router": "^4.1.0",
    "express-rate-limit": "^5.3.0",
@@ -28,6 +29,7 @@
  },
  "devDependencies": {
    "@types/better-sqlite3": "^5.4.3",
+    "@types/cron": "^1.7.3",
    "@types/express": "^4.17.13",
    "@types/express-rate-limit": "^5.1.3",
    "@types/mocha": "^8.2.3",
--- a/src/config.ts
+++ b/src/config.ts
@@ -77,7 +77,8 @@ addDefaults(config, {
            name: "vipUsers"
        }]
    },
-    diskCache: null
+    diskCache: null,
+    crons: null
 });

 // Add defaults
--- a/src/cronjob/downvoteSegmentArchiveJob.ts
+++ b/src/cronjob/downvoteSegmentArchiveJob.ts
@@ -0,0 +1,63 @@
+import { CronJob } from "cron";
+
+import { config as serverConfig } from "../config";
+import { Logger } from "../utils/logger";
+import { db } from "../databases/databases";
+import { DBSegment } from "../types/segments.model";
+
+const jobConfig = serverConfig?.crons?.downvoteSegmentArchive;
+
+export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise<number> => {
+  const timeNow = runTime || new Date().getTime();
+  const threshold = dayLimit * 86400000;
+
+  Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`);
+  try {
+    // insert into archive sponsorTime
+    await db.prepare(
+      'run',
+      `INSERT INTO "archivedSponsorTimes" 
+        SELECT * 
+        FROM "sponsorTimes" 
+        WHERE "votes" < ? AND (? - "timeSubmitted") > ?`,
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when insert segment in archivedSponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  // remove from sponsorTime
+  try {
+    await db.prepare(
+      'run', 
+      'DELETE FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?',
+      [
+        voteLimit,
+        timeNow,
+        threshold
+      ]
+    ) as DBSegment[];
+
+  } catch (err) {
+    Logger.error('Execption when deleting segment in sponsorTimes');
+    Logger.error(err);
+    return 1;
+  }
+
+  Logger.info('DownvoteSegmentArchiveJob finished');
+  return 0;
+};
+
+const DownvoteSegmentArchiveJob = new CronJob(
+  jobConfig?.schedule || new Date(1),
+  () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold)
+);
+
+export default DownvoteSegmentArchiveJob;
--- a/src/cronjob/index.ts
+++ b/src/cronjob/index.ts
@@ -0,0 +1,13 @@
+import { Logger } from "../utils/logger";
+import { config } from "../config";
+import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob";
+
+export function startAllCrons (): void {
+  if (config?.crons?.enabled) {
+    Logger.info("Crons started");
+
+    DownvoteSegmentArchiveJob.start();
+  } else {
+    Logger.info("Crons dissabled");
+  }
+}
--- a/src/index.ts
+++ b/src/index.ts
@@ -2,13 +2,17 @@ import {config} from "./config";
 import {initDb} from './databases/databases';
 import {createServer} from "./app";
 import {Logger} from "./utils/logger";
+import {startAllCrons} from "./cronjob";

 async function init() {
    await initDb();

    createServer(() => {
        Logger.info("Server started on port " + config.port + ".");
+
+        // ignite cron job after server created
+        startAllCrons();
    });
 }
- 
+
 init();
--- a/src/types/config.model.ts
+++ b/src/types/config.model.ts
@@ -44,6 +44,7 @@ export interface SBSConfig {
    postgres?: PoolConfig;
    dumpDatabase?: DumpDatabase;
    diskCache: CacheOptions;
+    crons: CronJobOptions;
 }

 export interface WebhookConfig {
@@ -81,3 +82,17 @@ export interface DumpDatabaseTable {
    name: string;
    order?: string;
 }
+
+export interface CronJobDefault {
+    schedule: string;
+}
+
+export interface CronJobOptions {
+    enabled: boolean;
+    downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron;
+}
+
+export interface DownvoteSegmentArchiveCron {
+    voteThreshold: number;
+    timeThresholdInDays: number;
+}
--- a/test/cases/downvoteSegmentArchiveJob.ts
+++ b/test/cases/downvoteSegmentArchiveJob.ts
@@ -0,0 +1,172 @@
+import assert from 'assert';
+
+import { db } from '../../src/databases/databases';
+import { getHash } from '../../src/utils/getHash';
+import { archiveDownvoteSegment } from '../../src/cronjob/downvoteSegmentArchiveJob';
+import { DBSegment } from '../../src/types/segments.model';
+
+const records = [
+    ['testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 100, 0, 0, getHash('testtesttest', 1)],
+    ['testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 120, 0, 0, getHash('testtesttest2', 1)],
+    ['testtesttest', 12, 14, 2, 0, '1-uuid-0-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'mute', 'ytb', 100, 0, 0, getHash('testtesttest', 1)],
+    ['testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 101, 0, 0, getHash('testtesttest', 1)],
+    ['testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 140, 0, 0, getHash('testtesttest,test', 1)],
+
+    ['test3', 1, 11, 2, 0, '1-uuid-4', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 200, 0, 0, getHash('test3', 1)],
+    ['test3', 7, 22, -3, 0, '1-uuid-5', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 300, 0, 0, getHash('test3', 1)],
+
+    ['multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 400, 0, 0, getHash('multiple', 1)],
+    ['multiple', 20, 33, -4, 0, '1-uuid-7', 'testman', new Date('October 1, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 500, 0, 0, getHash('multiple', 1)],
+
+    ['locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 230, 0, 0, getHash('locked', 1)],
+    ['locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 190, 0, 0, getHash('locked', 1)],
+
+    ['onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 190, 1, 0, getHash('onlyHiddenSegments', 1)],
+
+    ['requiredSegmentVid-raw', 60, 70, 2, 0, 'requiredSegmentVid-raw-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)],
+    ['requiredSegmentVid-raw', 60, 70, -1, 0, 'requiredSegmentVid-raw-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)],
+    ['requiredSegmentVid-raw', 80, 90, -2, 0, 'requiredSegmentVid-raw-3', 'testman', new Date('November 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)],
+    ['requiredSegmentVid-raw', 80, 90, 2, 0, 'requiredSegmentVid-raw-4', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)]
+];
+
+describe('downvoteSegmentArchiveJob', () => {
+    beforeEach(async () => {
+        const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "actionType", "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
+
+        for (let i = 0; i < records.length; i += 1) {
+            await db.prepare('run', query, records[i]);
+        }
+
+        return;
+    });
+
+    it('Should update the database version when starting the application', async () => {
+        const version = (await db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version'])).value;
+        assert.ok(version >= 21, "version should be greater or equal to 21");
+    });
+
+    afterEach(async () => {
+        await db.prepare('run', 'DELETE FROM "sponsorTimes"');
+        await db.prepare('run', 'DELETE FROM "archivedSponsorTimes"');
+    });
+
+    const getArchivedSegment = (): Promise<DBSegment[]> => {
+        return db.prepare('all', 'SELECT * FROM "archivedSponsorTimes"');
+    };
+
+    const getSegmentsInMainTable = (dayLimit: number, voteLimit: number, now: number): Promise<DBSegment[]> => {
+        return db.prepare(
+            'all', 
+            'SELECT * FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?',
+            [
+                voteLimit,
+                now,
+                dayLimit * 86400000,
+            ]
+        );
+    };
+
+    const countSegmentInMainTable = (): Promise<number> => {
+        return db.prepare(
+            'get', 
+            'SELECT COUNT(*) as count FROM "sponsorTimes"'
+        ).then(res => res.count);
+    };
+
+    it('Should archive all records match', async () => {
+        const dayLimit = 20;
+        const voteLimit = 0;
+        const time = new Date('December 17, 2022').getTime();
+        const res = await archiveDownvoteSegment(dayLimit, voteLimit, time);
+        assert.strictEqual(res, 0, 'Expection in archiveDownvoteSegment');
+
+        // check segments in archived table
+        const archivedSegment = await getArchivedSegment();
+        assert.strictEqual(archivedSegment.length, 4, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 4`);
+
+        // check segments not in main table
+        const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time);
+        assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`);
+
+        // check number segments remain in main table
+        assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table');
+    });
+
+    it('Should archive records with vote < -1 match', async () => {
+        const dayLimit = 20;
+        const voteLimit = -1;
+        const time = new Date('December 17, 2022').getTime();
+        const res = await archiveDownvoteSegment(dayLimit, voteLimit, time);
+        assert.strictEqual(res, 0, '');
+
+        // check segments in archived table
+        const archivedSegment = await getArchivedSegment();
+        assert.strictEqual(archivedSegment.length, 3, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 3`);
+
+        // check segments not in main table
+        const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time);
+        assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`);
+
+        // check number segments remain in main table
+        assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table');
+    });
+
+    it('Should archive records with vote < -2 and day < 30 match', async () => {
+        const dayLimit = 30;
+        const voteLimit = -2;
+        const time = new Date('December 17, 2021').getTime();
+        const res = await archiveDownvoteSegment(dayLimit, voteLimit, time);
+        assert.strictEqual(res, 0, '');
+
+        // check segments in archived table
+        const archivedSegment = await getArchivedSegment();
+        assert.strictEqual(archivedSegment.length, 1, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 1`);
+
+        assert.strictEqual(archivedSegment[0].votes, -4, `Incorrect segment vote in archiveTable: ${archivedSegment[0].votes} instead of -4`);
+
+        // check segments not in main table
+        const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time);
+        assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`);
+
+        // check number segments remain in main table
+        assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table');
+    });
+
+    it('Should archive records with vote < -2 and day < 300 match', async () => {
+        const dayLimit = 300;
+        const voteLimit = -2;
+        const time = new Date('December 17, 2022').getTime();
+        const res = await archiveDownvoteSegment(dayLimit, voteLimit, time);
+        assert.strictEqual(res, 0, '');
+
+        // check segments in archived table
+        const archivedSegment = await getArchivedSegment();
+        assert.strictEqual(archivedSegment.length, 2, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 2`);
+
+        // check segments not in main table
+        const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time);
+        assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`);
+
+        // check number segments remain in main table
+        assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table');
+    });
+
+    it('Should not archive any', async () => {
+        const dayLimit = 300;
+        const voteLimit = -2;
+        const time = new Date('December 17, 2021').getTime();
+        const res = await archiveDownvoteSegment(dayLimit, voteLimit, time);
+        assert.strictEqual(res, 0, '');
+
+        // check segments in archived table
+        const archivedSegment = await getArchivedSegment();
+        assert.strictEqual(archivedSegment.length, 0, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 0`);
+
+        // check segments not in main table
+        const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time);
+        assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`);
+
+        // check number segments remain in main table
+        assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table');
+    });
+});