diff --git a/DatabaseSchema.md b/DatabaseSchema.md index 485c4c3..26f2319 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -10,6 +10,7 @@ [shadowBannedUsers](#shadowBannedUsers) [unlistedVideos](#unlistedVideos) [config](#config) +[archivedSponsorTimes](#archivedSponsorTimes) ### vipUsers | Name | Type | | @@ -35,6 +36,7 @@ | timeSubmitted | INTEGER | not null | | views | INTEGER | not null | | category | TEXT | not null, default 'sponsor' | +| actionType | TEXT | not null, default 'skip' | | service | TEXT | not null, default 'Youtube' | | videoDuration | INTEGER | not null, default '0' | | hidden | INTEGER | not null, default '0' | @@ -140,7 +142,28 @@ | key | TEXT | not null, unique | | value | TEXT | not null | +### archivedSponsorTimes +| Name | Type | | +| -- | :--: | -- | +| videoID | TEXT | not null | +| startTime | REAL | not null | +| endTime | REAL | not null | +| votes | INTEGER | not null | +| locked | INTEGER | not null, default '0' | +| incorrectVotes | INTEGER | not null, default 1 | +| UUID | TEXT | not null, unique | +| userID | TEXT | not null | +| timeSubmitted | INTEGER | not null | +| views | INTEGER | not null | +| category | TEXT | not null, default 'sponsor' | +| actionType | TEXT | not null, default 'skip' | +| service | TEXT | not null, default 'Youtube' | +| videoDuration | INTEGER | not null, default '0' | +| hidden | INTEGER | not null, default '0' | +| reputation | REAL | not null, default '0' | +| shadowHidden | INTEGER | not null | +| hashedVideoID | TEXT | not null, default '', sha256 | # Private diff --git a/databases/_upgrade_sponsorTimes_21.sql b/databases/_upgrade_sponsorTimes_21.sql new file mode 100644 index 0000000..4e850bd --- /dev/null +++ b/databases/_upgrade_sponsorTimes_21.sql @@ -0,0 +1,26 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL DEFAULT '0', + "incorrectVotes" INTEGER NOT NULL DEFAULT 1, + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'Youtube', + "actionType" TEXT NOT NULL DEFAULT 'skip', + "videoDuration" INTEGER NOT NULL DEFAULT '0', + "hidden" INTEGER NOT NULL DEFAULT '0', + "reputation" REAL NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL DEFAULT '' +); + +UPDATE "config" SET value = 21 WHERE key = 'version'; + +COMMIT; diff --git a/package.json b/package.json index 56386f6..fcba30c 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "@ajayyy/lru-diskcache": "^1.1.9", "@types/request": "^2.48.6", "better-sqlite3": "^7.4.1", + "cron": "^1.8.2", "express": "^4.17.1", "express-promise-router": "^4.1.0", "express-rate-limit": "^5.3.0", @@ -28,6 +29,7 @@ }, "devDependencies": { "@types/better-sqlite3": "^5.4.3", + "@types/cron": "^1.7.3", "@types/express": "^4.17.13", "@types/express-rate-limit": "^5.1.3", "@types/mocha": "^8.2.3", diff --git a/src/config.ts b/src/config.ts index 1e900b9..9002df7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,7 +77,8 @@ addDefaults(config, { name: "vipUsers" }] }, - diskCache: null + diskCache: null, + crons: null }); // Add defaults diff --git a/src/cronjob/downvoteSegmentArchiveJob.ts b/src/cronjob/downvoteSegmentArchiveJob.ts new file mode 100644 index 0000000..f5e8e03 --- /dev/null +++ b/src/cronjob/downvoteSegmentArchiveJob.ts @@ -0,0 +1,63 @@ +import { CronJob } from "cron"; + +import { config as serverConfig } from "../config"; +import { Logger } from "../utils/logger"; +import { db } from "../databases/databases"; +import { DBSegment } from "../types/segments.model"; + +const jobConfig = serverConfig?.crons?.downvoteSegmentArchive; + +export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise => { + const timeNow = runTime || new Date().getTime(); + const threshold = dayLimit * 86400000; + + Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`); + try { + // insert into archive sponsorTime + await db.prepare( + 'run', + `INSERT INTO "archivedSponsorTimes" + SELECT * + FROM "sponsorTimes" + WHERE "votes" < ? AND (? - "timeSubmitted") > ?`, + [ + voteLimit, + timeNow, + threshold + ] + ) as DBSegment[]; + + } catch (err) { + Logger.error('Execption when insert segment in archivedSponsorTimes'); + Logger.error(err); + return 1; + } + + // remove from sponsorTime + try { + await db.prepare( + 'run', + 'DELETE FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?', + [ + voteLimit, + timeNow, + threshold + ] + ) as DBSegment[]; + + } catch (err) { + Logger.error('Execption when deleting segment in sponsorTimes'); + Logger.error(err); + return 1; + } + + Logger.info('DownvoteSegmentArchiveJob finished'); + return 0; +}; + +const DownvoteSegmentArchiveJob = new CronJob( + jobConfig?.schedule || new Date(1), + () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold) +); + +export default DownvoteSegmentArchiveJob; diff --git a/src/cronjob/index.ts b/src/cronjob/index.ts new file mode 100644 index 0000000..46aa80c --- /dev/null +++ b/src/cronjob/index.ts @@ -0,0 +1,13 @@ +import { Logger } from "../utils/logger"; +import { config } from "../config"; +import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob"; + +export function startAllCrons (): void { + if (config?.crons?.enabled) { + Logger.info("Crons started"); + + DownvoteSegmentArchiveJob.start(); + } else { + Logger.info("Crons dissabled"); + } +} diff --git a/src/index.ts b/src/index.ts index 13576a7..babb1a9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,13 +2,17 @@ import {config} from "./config"; import {initDb} from './databases/databases'; import {createServer} from "./app"; import {Logger} from "./utils/logger"; +import {startAllCrons} from "./cronjob"; async function init() { await initDb(); createServer(() => { Logger.info("Server started on port " + config.port + "."); + + // ignite cron job after server created + startAllCrons(); }); } - + init(); \ No newline at end of file diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 1dabdbe..cb5daa2 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -44,6 +44,7 @@ export interface SBSConfig { postgres?: PoolConfig; dumpDatabase?: DumpDatabase; diskCache: CacheOptions; + crons: CronJobOptions; } export interface WebhookConfig { @@ -81,3 +82,17 @@ export interface DumpDatabaseTable { name: string; order?: string; } + +export interface CronJobDefault { + schedule: string; +} + +export interface CronJobOptions { + enabled: boolean; + downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron; +} + +export interface DownvoteSegmentArchiveCron { + voteThreshold: number; + timeThresholdInDays: number; +} \ No newline at end of file diff --git a/test/cases/downvoteSegmentArchiveJob.ts b/test/cases/downvoteSegmentArchiveJob.ts new file mode 100644 index 0000000..6700272 --- /dev/null +++ b/test/cases/downvoteSegmentArchiveJob.ts @@ -0,0 +1,172 @@ +import assert from 'assert'; + +import { db } from '../../src/databases/databases'; +import { getHash } from '../../src/utils/getHash'; +import { archiveDownvoteSegment } from '../../src/cronjob/downvoteSegmentArchiveJob'; +import { DBSegment } from '../../src/types/segments.model'; + +const records = [ + ['testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 100, 0, 0, getHash('testtesttest', 1)], + ['testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 120, 0, 0, getHash('testtesttest2', 1)], + ['testtesttest', 12, 14, 2, 0, '1-uuid-0-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'mute', 'ytb', 100, 0, 0, getHash('testtesttest', 1)], + ['testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 101, 0, 0, getHash('testtesttest', 1)], + ['testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 140, 0, 0, getHash('testtesttest,test', 1)], + + ['test3', 1, 11, 2, 0, '1-uuid-4', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 200, 0, 0, getHash('test3', 1)], + ['test3', 7, 22, -3, 0, '1-uuid-5', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 300, 0, 0, getHash('test3', 1)], + + ['multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 400, 0, 0, getHash('multiple', 1)], + ['multiple', 20, 33, -4, 0, '1-uuid-7', 'testman', new Date('October 1, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 500, 0, 0, getHash('multiple', 1)], + + ['locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 230, 0, 0, getHash('locked', 1)], + ['locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 190, 0, 0, getHash('locked', 1)], + + ['onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 190, 1, 0, getHash('onlyHiddenSegments', 1)], + + ['requiredSegmentVid-raw', 60, 70, 2, 0, 'requiredSegmentVid-raw-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)], + ['requiredSegmentVid-raw', 60, 70, -1, 0, 'requiredSegmentVid-raw-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)], + ['requiredSegmentVid-raw', 80, 90, -2, 0, 'requiredSegmentVid-raw-3', 'testman', new Date('November 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)], + ['requiredSegmentVid-raw', 80, 90, 2, 0, 'requiredSegmentVid-raw-4', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)] +]; + +describe('downvoteSegmentArchiveJob', () => { + beforeEach(async () => { + const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "actionType", "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + + for (let i = 0; i < records.length; i += 1) { + await db.prepare('run', query, records[i]); + } + + return; + }); + + it('Should update the database version when starting the application', async () => { + const version = (await db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version'])).value; + assert.ok(version >= 21, "version should be greater or equal to 21"); + }); + + afterEach(async () => { + await db.prepare('run', 'DELETE FROM "sponsorTimes"'); + await db.prepare('run', 'DELETE FROM "archivedSponsorTimes"'); + }); + + const getArchivedSegment = (): Promise => { + return db.prepare('all', 'SELECT * FROM "archivedSponsorTimes"'); + }; + + const getSegmentsInMainTable = (dayLimit: number, voteLimit: number, now: number): Promise => { + return db.prepare( + 'all', + 'SELECT * FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?', + [ + voteLimit, + now, + dayLimit * 86400000, + ] + ); + }; + + const countSegmentInMainTable = (): Promise => { + return db.prepare( + 'get', + 'SELECT COUNT(*) as count FROM "sponsorTimes"' + ).then(res => res.count); + }; + + it('Should archive all records match', async () => { + const dayLimit = 20; + const voteLimit = 0; + const time = new Date('December 17, 2022').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, 'Expection in archiveDownvoteSegment'); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 4, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 4`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should archive records with vote < -1 match', async () => { + const dayLimit = 20; + const voteLimit = -1; + const time = new Date('December 17, 2022').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 3, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 3`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should archive records with vote < -2 and day < 30 match', async () => { + const dayLimit = 30; + const voteLimit = -2; + const time = new Date('December 17, 2021').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 1, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 1`); + + assert.strictEqual(archivedSegment[0].votes, -4, `Incorrect segment vote in archiveTable: ${archivedSegment[0].votes} instead of -4`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should archive records with vote < -2 and day < 300 match', async () => { + const dayLimit = 300; + const voteLimit = -2; + const time = new Date('December 17, 2022').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 2, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 2`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should not archive any', async () => { + const dayLimit = 300; + const voteLimit = -2; + const time = new Date('December 17, 2021').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 0, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 0`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); +});