From 6b5156468ca1ca60f97396f2e79a5cfbd9ebc9c4 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Fri, 9 Jul 2021 11:46:04 +0700 Subject: [PATCH 1/3] Add archive downvote segment cron --- DatabaseSchema.md | 23 +++++++++ databases/_upgrade_sponsorTimes_21.sql | 26 ++++++++++ package.json | 2 + src/config.ts | 3 +- src/cronjob/downvoteSegmentArchiveJob.ts | 63 ++++++++++++++++++++++++ src/cronjob/index.ts | 13 +++++ src/index.ts | 6 ++- src/types/config.model.ts | 15 ++++++ 8 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_21.sql create mode 100644 src/cronjob/downvoteSegmentArchiveJob.ts create mode 100644 src/cronjob/index.ts diff --git a/DatabaseSchema.md b/DatabaseSchema.md index 485c4c3..26f2319 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -10,6 +10,7 @@ [shadowBannedUsers](#shadowBannedUsers) [unlistedVideos](#unlistedVideos) [config](#config) +[archivedSponsorTimes](#archivedSponsorTimes) ### vipUsers | Name | Type | | @@ -35,6 +36,7 @@ | timeSubmitted | INTEGER | not null | | views | INTEGER | not null | | category | TEXT | not null, default 'sponsor' | +| actionType | TEXT | not null, default 'skip' | | service | TEXT | not null, default 'Youtube' | | videoDuration | INTEGER | not null, default '0' | | hidden | INTEGER | not null, default '0' | @@ -140,7 +142,28 @@ | key | TEXT | not null, unique | | value | TEXT | not null | +### archivedSponsorTimes +| Name | Type | | +| -- | :--: | -- | +| videoID | TEXT | not null | +| startTime | REAL | not null | +| endTime | REAL | not null | +| votes | INTEGER | not null | +| locked | INTEGER | not null, default '0' | +| incorrectVotes | INTEGER | not null, default 1 | +| UUID | TEXT | not null, unique | +| userID | TEXT | not null | +| timeSubmitted | INTEGER | not null | +| views | INTEGER | not null | +| category | TEXT | not null, default 'sponsor' | +| actionType | TEXT | not null, default 'skip' | +| service | TEXT | not null, default 'Youtube' | +| videoDuration | INTEGER | not null, default '0' | +| hidden | INTEGER | not null, default '0' | +| reputation | REAL | not null, default '0' | +| shadowHidden | INTEGER | not null | +| hashedVideoID | TEXT | not null, default '', sha256 | # Private diff --git a/databases/_upgrade_sponsorTimes_21.sql b/databases/_upgrade_sponsorTimes_21.sql new file mode 100644 index 0000000..4e850bd --- /dev/null +++ b/databases/_upgrade_sponsorTimes_21.sql @@ -0,0 +1,26 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "archivedSponsorTimes" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL DEFAULT '0', + "incorrectVotes" INTEGER NOT NULL DEFAULT 1, + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'Youtube', + "actionType" TEXT NOT NULL DEFAULT 'skip', + "videoDuration" INTEGER NOT NULL DEFAULT '0', + "hidden" INTEGER NOT NULL DEFAULT '0', + "reputation" REAL NOT NULL DEFAULT '0', + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL DEFAULT '' +); + +UPDATE "config" SET value = 21 WHERE key = 'version'; + +COMMIT; diff --git a/package.json b/package.json index 56386f6..fcba30c 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "@ajayyy/lru-diskcache": "^1.1.9", "@types/request": "^2.48.6", "better-sqlite3": "^7.4.1", + "cron": "^1.8.2", "express": "^4.17.1", "express-promise-router": "^4.1.0", "express-rate-limit": "^5.3.0", @@ -28,6 +29,7 @@ }, "devDependencies": { "@types/better-sqlite3": "^5.4.3", + "@types/cron": "^1.7.3", "@types/express": "^4.17.13", "@types/express-rate-limit": "^5.1.3", "@types/mocha": "^8.2.3", diff --git a/src/config.ts b/src/config.ts index 1e900b9..9002df7 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,7 +77,8 @@ addDefaults(config, { name: "vipUsers" }] }, - diskCache: null + diskCache: null, + crons: null }); // Add defaults diff --git a/src/cronjob/downvoteSegmentArchiveJob.ts b/src/cronjob/downvoteSegmentArchiveJob.ts new file mode 100644 index 0000000..c2f8f4e --- /dev/null +++ b/src/cronjob/downvoteSegmentArchiveJob.ts @@ -0,0 +1,63 @@ +import { CronJob } from "cron"; + +import { config as serverConfig } from "../config"; +import { Logger } from "../utils/logger"; +import { db } from "../databases/databases"; +import { DBSegment } from "../types/segments.model"; + +const jobConfig = serverConfig?.crons?.downvoteSegmentArchive; + +export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number, runTime?: number): Promise => { + const timeNow = runTime || new Date().getTime(); + const threshold = dayLimit * 86400000; + + Logger.info(`DownvoteSegmentArchiveJob starts at ${timeNow}`); + try { + // insert into archive sponsorTime + await db.prepare( + 'run', + `INSERT INTO "archivedSponsorTimes" + SELECT * + FROM "sponsorTimes" + WHERE votes < ? AND (? - timeSubmitted) > ?`, + [ + voteLimit, + timeNow, + threshold + ] + ) as DBSegment[]; + + } catch (err) { + Logger.error('Execption when insert segment in archivedSponsorTimes'); + Logger.error(err); + return 1; + } + + // remove from sponsorTime + try { + await db.prepare( + 'run', + 'DELETE FROM "sponsorTimes" WHERE votes < ? AND (? - timeSubmitted) > ?', + [ + voteLimit, + timeNow, + threshold + ] + ) as DBSegment[]; + + } catch (err) { + Logger.error('Execption when deleting segment in sponsorTimes'); + Logger.error(err); + return 1; + } + + Logger.info('DownvoteSegmentArchiveJob finished'); + return 0; +}; + +const DownvoteSegmentArchiveJob = new CronJob( + jobConfig?.schedule || new Date(1), + () => archiveDownvoteSegment(jobConfig?.timeThresholdInDays, jobConfig?.voteThreshold) +); + +export default DownvoteSegmentArchiveJob; diff --git a/src/cronjob/index.ts b/src/cronjob/index.ts new file mode 100644 index 0000000..46aa80c --- /dev/null +++ b/src/cronjob/index.ts @@ -0,0 +1,13 @@ +import { Logger } from "../utils/logger"; +import { config } from "../config"; +import DownvoteSegmentArchiveJob from "./downvoteSegmentArchiveJob"; + +export function startAllCrons (): void { + if (config?.crons?.enabled) { + Logger.info("Crons started"); + + DownvoteSegmentArchiveJob.start(); + } else { + Logger.info("Crons dissabled"); + } +} diff --git a/src/index.ts b/src/index.ts index 13576a7..babb1a9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,13 +2,17 @@ import {config} from "./config"; import {initDb} from './databases/databases'; import {createServer} from "./app"; import {Logger} from "./utils/logger"; +import {startAllCrons} from "./cronjob"; async function init() { await initDb(); createServer(() => { Logger.info("Server started on port " + config.port + "."); + + // ignite cron job after server created + startAllCrons(); }); } - + init(); \ No newline at end of file diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 1dabdbe..cb5daa2 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -44,6 +44,7 @@ export interface SBSConfig { postgres?: PoolConfig; dumpDatabase?: DumpDatabase; diskCache: CacheOptions; + crons: CronJobOptions; } export interface WebhookConfig { @@ -81,3 +82,17 @@ export interface DumpDatabaseTable { name: string; order?: string; } + +export interface CronJobDefault { + schedule: string; +} + +export interface CronJobOptions { + enabled: boolean; + downvoteSegmentArchive: CronJobDefault & DownvoteSegmentArchiveCron; +} + +export interface DownvoteSegmentArchiveCron { + voteThreshold: number; + timeThresholdInDays: number; +} \ No newline at end of file From bbb1db014cb023af1ad7e208adeb7d172cc5089c Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Fri, 9 Jul 2021 11:46:38 +0700 Subject: [PATCH 2/3] Add archive downvote function unit test --- test/cases/downvoteSegmentArchiveJob.ts | 167 ++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 test/cases/downvoteSegmentArchiveJob.ts diff --git a/test/cases/downvoteSegmentArchiveJob.ts b/test/cases/downvoteSegmentArchiveJob.ts new file mode 100644 index 0000000..c750526 --- /dev/null +++ b/test/cases/downvoteSegmentArchiveJob.ts @@ -0,0 +1,167 @@ +import assert from 'assert'; + +import { db } from '../../src/databases/databases'; +import { getHash } from '../../src/utils/getHash'; +import { archiveDownvoteSegment } from '../../src/cronjob/downvoteSegmentArchiveJob'; +import { DBSegment } from '../../src/types/segments.model'; + +const records = [ + ['testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 100, 0, 0, getHash('testtesttest', 1)], + ['testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 120, 0, 0, getHash('testtesttest2', 1)], + ['testtesttest', 12, 14, 2, 0, '1-uuid-0-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'mute', 'ytb', 100, 0, 0, getHash('testtesttest', 1)], + ['testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 101, 0, 0, getHash('testtesttest', 1)], + ['testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 140, 0, 0, getHash('testtesttest,test', 1)], + + ['test3', 1, 11, 2, 0, '1-uuid-4', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 200, 0, 0, getHash('test3', 1)], + ['test3', 7, 22, -3, 0, '1-uuid-5', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 300, 0, 0, getHash('test3', 1)], + + ['multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 400, 0, 0, getHash('multiple', 1)], + ['multiple', 20, 33, -4, 0, '1-uuid-7', 'testman', new Date('October 1, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 500, 0, 0, getHash('multiple', 1)], + + ['locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 230, 0, 0, getHash('locked', 1)], + ['locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', new Date('December 17, 2021').getTime(), 50, 'intro', 'skip', 'ytb', 190, 0, 0, getHash('locked', 1)], + + ['onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 190, 1, 0, getHash('onlyHiddenSegments', 1)], + + ['requiredSegmentVid-raw', 60, 70, 2, 0, 'requiredSegmentVid-raw-1', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)], + ['requiredSegmentVid-raw', 60, 70, -1, 0, 'requiredSegmentVid-raw-2', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)], + ['requiredSegmentVid-raw', 80, 90, -2, 0, 'requiredSegmentVid-raw-3', 'testman', new Date('November 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)], + ['requiredSegmentVid-raw', 80, 90, 2, 0, 'requiredSegmentVid-raw-4', 'testman', new Date('December 17, 2021').getTime(), 50, 'sponsor', 'skip', 'ytb', 0, 0, 0, getHash('requiredSegmentVid-raw', 1)] +]; + +describe('downvoteSegmentArchiveJob', () => { + beforeEach(async () => { + const query = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "actionType", "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'; + + for (let i = 0; i < records.length; i += 1) { + await db.prepare('run', query, records[i]); + } + + return; + }); + + afterEach(async () => { + await db.prepare('run', 'DELETE FROM "sponsorTimes"'); + await db.prepare('run', 'DELETE FROM "archivedSponsorTimes"'); + }); + + const getArchivedSegment = (): Promise => { + return db.prepare('all', 'SELECT * FROM "archivedSponsorTimes"'); + }; + + const getSegmentsInMainTable = (dayLimit: number, voteLimit: number, now: number): Promise => { + return db.prepare( + 'all', + 'SELECT * FROM "sponsorTimes" WHERE votes < ? AND (? - timeSubmitted) > ?', + [ + voteLimit, + now, + dayLimit * 86400000, + ] + ); + }; + + const countSegmentInMainTable = (): Promise => { + return db.prepare( + 'get', + 'SELECT COUNT(*) as count FROM "sponsorTimes"' + ).then(res => res.count); + }; + + it('Should archive all records match', async () => { + const dayLimit = 20; + const voteLimit = 0; + const time = new Date('December 17, 2022').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, 'Expection in archiveDownvoteSegment'); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 4, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 4`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should archive records with vote < -1 match', async () => { + const dayLimit = 20; + const voteLimit = -1; + const time = new Date('December 17, 2022').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 3, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 3`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should archive records with vote < -2 and day < 30 match', async () => { + const dayLimit = 30; + const voteLimit = -2; + const time = new Date('December 17, 2021').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 1, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 1`); + + assert.strictEqual(archivedSegment[0].votes, -4, `Incorrect segment vote in archiveTable: ${archivedSegment[0].votes} instead of -4`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should archive records with vote < -2 and day < 300 match', async () => { + const dayLimit = 300; + const voteLimit = -2; + const time = new Date('December 17, 2022').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 2, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 2`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); + + it('Should not archive any', async () => { + const dayLimit = 300; + const voteLimit = -2; + const time = new Date('December 17, 2021').getTime(); + const res = await archiveDownvoteSegment(dayLimit, voteLimit, time); + assert.strictEqual(res, 0, ''); + + // check segments in archived table + const archivedSegment = await getArchivedSegment(); + assert.strictEqual(archivedSegment.length, 0, `Incorrect segment in archiveTable: ${archivedSegment.length} instead of 0`); + + // check segments not in main table + const segments = await getSegmentsInMainTable(dayLimit, voteLimit, time); + assert.strictEqual(segments.length, 0, `Incorrect segment in main table: ${segments.length} instead of 0`); + + // check number segments remain in main table + assert.strictEqual(await countSegmentInMainTable(), records.length - archivedSegment.length ,'Incorrect segment remain in main table'); + }); +}); From d9a66a58949e001ae66405cb4b861dfb68e26c7a Mon Sep 17 00:00:00 2001 From: Michael C Date: Sat, 10 Jul 2021 00:26:18 -0400 Subject: [PATCH 3/3] quote & check for db version --- src/cronjob/downvoteSegmentArchiveJob.ts | 4 ++-- test/cases/downvoteSegmentArchiveJob.ts | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/cronjob/downvoteSegmentArchiveJob.ts b/src/cronjob/downvoteSegmentArchiveJob.ts index c2f8f4e..f5e8e03 100644 --- a/src/cronjob/downvoteSegmentArchiveJob.ts +++ b/src/cronjob/downvoteSegmentArchiveJob.ts @@ -19,7 +19,7 @@ export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number `INSERT INTO "archivedSponsorTimes" SELECT * FROM "sponsorTimes" - WHERE votes < ? AND (? - timeSubmitted) > ?`, + WHERE "votes" < ? AND (? - "timeSubmitted") > ?`, [ voteLimit, timeNow, @@ -37,7 +37,7 @@ export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number try { await db.prepare( 'run', - 'DELETE FROM "sponsorTimes" WHERE votes < ? AND (? - timeSubmitted) > ?', + 'DELETE FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?', [ voteLimit, timeNow, diff --git a/test/cases/downvoteSegmentArchiveJob.ts b/test/cases/downvoteSegmentArchiveJob.ts index c750526..6700272 100644 --- a/test/cases/downvoteSegmentArchiveJob.ts +++ b/test/cases/downvoteSegmentArchiveJob.ts @@ -40,6 +40,11 @@ describe('downvoteSegmentArchiveJob', () => { return; }); + it('Should update the database version when starting the application', async () => { + const version = (await db.prepare('get', 'SELECT key, value FROM config where key = ?', ['version'])).value; + assert.ok(version >= 21, "version should be greater or equal to 21"); + }); + afterEach(async () => { await db.prepare('run', 'DELETE FROM "sponsorTimes"'); await db.prepare('run', 'DELETE FROM "archivedSponsorTimes"'); @@ -52,7 +57,7 @@ describe('downvoteSegmentArchiveJob', () => { const getSegmentsInMainTable = (dayLimit: number, voteLimit: number, now: number): Promise => { return db.prepare( 'all', - 'SELECT * FROM "sponsorTimes" WHERE votes < ? AND (? - timeSubmitted) > ?', + 'SELECT * FROM "sponsorTimes" WHERE "votes" < ? AND (? - "timeSubmitted") > ?', [ voteLimit, now,