From 7f074554c4b305c3b5e2613d4fa945bd3b744515 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Thu, 1 Jul 2021 10:30:45 +0700 Subject: [PATCH 1/5] Fix schema md header link --- DatabaseSchema.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/DatabaseSchema.md b/DatabaseSchema.md index 3d792d2..abb0539 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -1,15 +1,15 @@ # SponsorTimesDB -[vipUsers](###vipUsers) -[sponsorTimes](###sponsorTimes) -[userNames](###userNames) -[userNameLogs](###userNameLogs) -[categoryVotes](###categoryVotes) -[lockCategories](###lockCategories) -[warnings](###warnings) -[shadowBannedUsers](###shadowBannedUsers) -[unlistedVideos](###unlistedVideos) -[config](###config) +[vipUsers](#vipUsers) +[sponsorTimes](#sponsorTimes) +[userNames](#userNames) +[userNameLogs](#userNameLogs) +[categoryVotes](#categoryVotes) +[lockCategories](#lockCategories) +[warnings](#warnings) +[shadowBannedUsers](#shadowBannedUsers) +[unlistedVideos](#unlistedVideos) +[config](#config) ### vipUsers | Name | Type | | @@ -142,10 +142,10 @@ # Private -[vote](###vote) -[categoryVotes](###categoryVotes) -[sponsorTimes](###sponsorTimes) -[config](###config) +[vote](#vote) +[categoryVotes](#categoryVotes) +[sponsorTimes](#sponsorTimes) +[config](#config) ### vote From 402ea3597127e7464a168c018f5f9caef7f75a04 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Thu, 1 Jul 2021 10:33:47 +0700 Subject: [PATCH 2/5] Add postSkipSegments return lastes warning reason --- package.json | 1 - src/routes/postSkipSegments.ts | 90 ++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 44 deletions(-) diff --git a/package.json b/package.json index f8b69b9..620ec82 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ "express": "^4.17.1", "express-rate-limit": "^5.1.3", "http": "0.0.0", - "iso8601-duration": "^1.2.0", "node-fetch": "^2.6.0", "pg": "^8.5.1", "redis": "^3.1.1", diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index fffb51a..5262175 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -4,7 +4,6 @@ import {db, privateDB} from '../databases/databases'; import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi'; import {getSubmissionUUID} from '../utils/getSubmissionUUID'; import fetch from 'node-fetch'; -import isoDurations, { end } from 'iso8601-duration'; import {getHash} from '../utils/getHash'; import {getIP} from '../utils/getIP'; import {getFormattedTime} from '../utils/getFormattedTime'; @@ -272,6 +271,34 @@ async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promi } } +async function checkUserActiveWarning(userID: string): Promise<{ pass: boolean; errorMessage: string; }> { + const MILLISECONDS_IN_HOUR = 3600000; + const now = Date.now(); + const warnings = await db.prepare('all', + `SELECT "reason" + FROM warnings + WHERE "userID" = ? AND "issueTime" > ? AND enabled = 1 + ORDER BY "issueTime" DESC + LIMIT ?`, + [ + userID, + Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR)), + config.maxNumberOfActiveWarnings + ], + ) as {reason: string}[] + + if (warnings?.length >= config.maxNumberOfActiveWarnings) { + const defaultMessage = 'Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?'; + + return { + pass: false, + errorMessage: warnings[0]?.reason?.length > 0 ? warnings[0].reason : defaultMessage + }; + } + + return {pass: true, errorMessage: ''}; +} + function proxySubmission(req: Request) { fetch(config.proxySubmission + '/api/skipSegments?userID=' + req.query.userID + '&videoID=' + req.query.videoID, { method: 'POST', @@ -319,26 +346,18 @@ export async function postSkipSegments(req: Request, res: Response) { } if (invalidFields.length !== 0) { - // invalid request - const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, ''); - res.status(400).send(`No valid ${fields} field(s) provided`); - return; + // invalid request + const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, ''); + res.status(400).send(`No valid ${fields} field(s) provided`); + return; } //hash the userID userID = getHash(userID); - //hash the ip 5000 times so no one can get it from the database - const hashedIP = getHash(getIP(req) + config.globalSalt); - - const MILLISECONDS_IN_HOUR = 3600000; - const now = Date.now(); - const warningsCount = (await db.prepare('get', `SELECT count(*) as count FROM warnings WHERE "userID" = ? AND "issueTime" > ? AND enabled = 1`, - [userID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))], - )).count; - - if (warningsCount >= config.maxNumberOfActiveWarnings) { - return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?'); + const warningResult: {pass: boolean, errorMessage: string} = await checkUserActiveWarning(userID); + if (!warningResult.pass) { + return res.status(403).send(warningResult.errorMessage); } let lockedCategoryList = (await db.prepare('all', 'SELECT category from "lockCategories" where "videoID" = ?', [videoID])).map((list: any) => { @@ -350,9 +369,15 @@ export async function postSkipSegments(req: Request, res: Response) { const decreaseVotes = 0; - const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 - AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as - {videoDuration: VideoDuration, UUID: SegmentUUID}[]; + const previousSubmissions = await db.prepare('all', + `SELECT "videoDuration", "UUID" + FROM "sponsorTimes" + WHERE "videoID" = ? AND "service" = ? AND + "hidden" = 0 AND "shadowHidden" = 0 AND + "votes" >= 0 AND "videoDuration" != 0`, + [videoID, service] + ) as {videoDuration: VideoDuration, UUID: SegmentUUID}[]; + // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden const videoDurationChanged = (videoDuration: number) => videoDuration != 0 && previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); @@ -452,37 +477,16 @@ export async function postSkipSegments(req: Request, res: Response) { const UUIDs = []; const newSegments = []; + //hash the ip 5000 times so no one can get it from the database + const hashedIP = getHash(getIP(req) + config.globalSalt); + try { //get current time const timeSubmitted = Date.now(); const yesterday = timeSubmitted - 86400000; - // Disable IP ratelimiting for now - if (false) { - //check to see if this ip has submitted too many sponsors today - const rateLimitCheckRow = await privateDB.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "hashedIP" = ? AND "videoID" = ? AND "timeSubmitted" > ?`, [hashedIP, videoID, yesterday]); - if (rateLimitCheckRow.count >= 10) { - //too many sponsors for the same video from the same ip address - res.sendStatus(429); - - return; - } - } - - // Disable max submissions for now - if (false) { - //check to see if the user has already submitted sponsors for this video - const duplicateCheckRow = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ?`, [userID, videoID]); - - if (duplicateCheckRow.count >= 16) { - //too many sponsors for the same video from the same user - res.sendStatus(429); - - return; - } - } //check to see if this user is shadowbanned const shadowBanRow = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); From 09e50d432e178a83b04f6bb52fcf54d286bd6ce9 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Thu, 1 Jul 2021 11:31:35 +0700 Subject: [PATCH 3/5] Update postSegment test with reason --- test/cases/postSkipSegments.ts | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 2c9a3d8..3c6e555 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -23,22 +23,25 @@ describe('postSkipSegments', () => { const warnUser01Hash = getHash("warn-user01"); const warnUser02Hash = getHash("warn-user02"); const warnUser03Hash = getHash("warn-user03"); + const reason01 = 'Reason01'; + const reason02 = ''; + const reason03 = 'Reason03'; const MILLISECONDS_IN_HOUR = 3600000; const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires; - const insertWarningQuery = 'INSERT INTO warnings ("userID", "issueTime", "issuerUserID", "enabled") VALUES(?, ?, ?, ?)'; - db.prepare("run", insertWarningQuery, [warnUser01Hash, now, warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser01Hash, (now - 1000), warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser01Hash, (now - 2000), warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser01Hash, (now - 3601000), warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser02Hash, now, warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser02Hash, now, warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser02Hash, (now - (warningExpireTime + 1000)), warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser02Hash, (now - (warningExpireTime + 2000)), warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser03Hash, now, warnVip01Hash, 0]); - db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 1000), warnVip01Hash, 0]); - db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 2000), warnVip01Hash, 1]); - db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 3601000), warnVip01Hash, 1]); + const insertWarningQuery = 'INSERT INTO warnings ("userID", "issueTime", "issuerUserID", "enabled", "reason") VALUES(?, ?, ?, ?, ?)'; + db.prepare("run", insertWarningQuery, [warnUser01Hash, now, warnVip01Hash, 1, reason01]); + db.prepare("run", insertWarningQuery, [warnUser01Hash, (now - 1000), warnVip01Hash, 1, reason01]); + db.prepare("run", insertWarningQuery, [warnUser01Hash, (now - 2000), warnVip01Hash, 1, reason01]); + db.prepare("run", insertWarningQuery, [warnUser01Hash, (now - 3601000), warnVip01Hash, 1, reason01]); + db.prepare("run", insertWarningQuery, [warnUser02Hash, now, warnVip01Hash, 1, reason02]); + db.prepare("run", insertWarningQuery, [warnUser02Hash, now, warnVip01Hash, 1, reason02]); + db.prepare("run", insertWarningQuery, [warnUser02Hash, (now - (warningExpireTime + 1000)), warnVip01Hash, 1, reason02]); + db.prepare("run", insertWarningQuery, [warnUser02Hash, (now - (warningExpireTime + 2000)), warnVip01Hash, 1, reason02]); + db.prepare("run", insertWarningQuery, [warnUser03Hash, now, warnVip01Hash, 0, reason03]); + db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 1000), warnVip01Hash, 0, reason03]); + db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 2000), warnVip01Hash, 1, reason03]); + db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 3601000), warnVip01Hash, 1, reason03]); const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; db.prepare("run", insertVipUserQuery, [getHash("VIPUserSubmission")]); @@ -619,7 +622,12 @@ describe('postSkipSegments', () => { }) .then(async res => { if (res.status === 403) { - done(); // success + const errorMessage = await res.text(); + if (errorMessage === 'Reason01') { + done(); // success + } else { + done("Status code was 403 but message was: " + errorMessage); + } } else { done("Status code was " + res.status); } From 24480fd18c75b42aab1716df8203cc839f43e435 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Thu, 1 Jul 2021 11:38:29 +0700 Subject: [PATCH 4/5] Revear auto clear unused code --- src/routes/postSkipSegments.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 5262175..d9b8869 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -486,7 +486,31 @@ export async function postSkipSegments(req: Request, res: Response) { const yesterday = timeSubmitted - 86400000; + // Disable IP ratelimiting for now + if (false) { + //check to see if this ip has submitted too many sponsors today + const rateLimitCheckRow = await privateDB.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "hashedIP" = ? AND "videoID" = ? AND "timeSubmitted" > ?`, [hashedIP, videoID, yesterday]); + if (rateLimitCheckRow.count >= 10) { + //too many sponsors for the same video from the same ip address + res.sendStatus(429); + + return; + } + } + + // Disable max submissions for now + if (false) { + //check to see if the user has already submitted sponsors for this video + const duplicateCheckRow = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ?`, [userID, videoID]); + + if (duplicateCheckRow.count >= 16) { + //too many sponsors for the same video from the same user + res.sendStatus(429); + + return; + } + } //check to see if this user is shadowbanned const shadowBanRow = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); From a7315eaee040891776f01da3d206183a4e93be03 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 3 Jul 2021 22:45:13 -0400 Subject: [PATCH 5/5] Add case for default warning message --- test/cases/postSkipSegments.ts | 39 +++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 3c6e555..eb67fcd 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -23,9 +23,11 @@ describe('postSkipSegments', () => { const warnUser01Hash = getHash("warn-user01"); const warnUser02Hash = getHash("warn-user02"); const warnUser03Hash = getHash("warn-user03"); + const warnUser04Hash = getHash("warn-user04"); const reason01 = 'Reason01'; const reason02 = ''; const reason03 = 'Reason03'; + const reason04 = ''; const MILLISECONDS_IN_HOUR = 3600000; const warningExpireTime = MILLISECONDS_IN_HOUR * config.hoursAfterWarningExpires; @@ -42,6 +44,10 @@ describe('postSkipSegments', () => { db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 1000), warnVip01Hash, 0, reason03]); db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 2000), warnVip01Hash, 1, reason03]); db.prepare("run", insertWarningQuery, [warnUser03Hash, (now - 3601000), warnVip01Hash, 1, reason03]); + db.prepare("run", insertWarningQuery, [warnUser04Hash, now, warnVip01Hash, 0, reason04]); + db.prepare("run", insertWarningQuery, [warnUser04Hash, (now - 1000), warnVip01Hash, 0, reason04]); + db.prepare("run", insertWarningQuery, [warnUser04Hash, (now - 2000), warnVip01Hash, 1, reason04]); + db.prepare("run", insertWarningQuery, [warnUser04Hash, (now - 3601000), warnVip01Hash, 1, reason04]); const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; db.prepare("run", insertVipUserQuery, [getHash("VIPUserSubmission")]); @@ -604,7 +610,7 @@ describe('postSkipSegments', () => { .catch(err => done("Couldn't call endpoint")); }); - it('Should be rejected if user has to many active warnings', (done: Done) => { + it('Should be rejected with custom message if user has to many active warnings', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", { method: 'POST', @@ -701,6 +707,37 @@ describe('postSkipSegments', () => { .catch(err => done(true)); }); + it('Should be rejected with default message if user has to many active warnings', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes", { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + userID: "warn-user01", + videoID: "dQw4w9WgXcF", + segments: [{ + segment: [0, 10], + category: "sponsor", + }], + }), + }) + .then(async res => { + if (res.status === 403) { + const errorMessage = await res.text(); + if (errorMessage !== '') { + done(); // success + } else { + done("Status code was 403 but message was: " + errorMessage); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + it('Should return 400 for missing params (JSON method) 1', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", {