Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 03:26:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

add service to table only have videoID

Browse Source
This commit is contained in:
Haidang666
2021-09-30 13:56:55 +07:00
parent 99d72b92e4
commit 356974b478
18 changed files with 188 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
DatabaseSchema.md
View File

@@ -37,7 +37,7 @@
| views | INTEGER | not null | | views | INTEGER | not null |
| category | TEXT | not null, default 'sponsor' | | category | TEXT | not null, default 'sponsor' |
| actionType | TEXT | not null, default 'skip' | | actionType | TEXT | not null, default 'skip' |
| service | TEXT | not null, default 'Youtube' | | service | TEXT | not null, default 'YouTube' |
| videoDuration | INTEGER | not null, default '0' | | videoDuration | INTEGER | not null, default '0' |
| hidden | INTEGER | not null, default '0' | | hidden | INTEGER | not null, default '0' |
| reputation | REAL | not null, default '0' | | reputation | REAL | not null, default '0' |
@@ -96,10 +96,11 @@
| category | TEXT | not null | | category | TEXT | not null |
| hashedVideoID | TEXT | not null, default '' | | hashedVideoID | TEXT | not null, default '' |
| reason | TEXT | not null, default '' | | reason | TEXT | not null, default '' |
| service | TEXT | not null, default 'YouTube' |
| index | field | | index | field |
| -- | :--: | | -- | :--: |
| noSegments_videoID | videoID | | lockCategories_videoID | videoID, service, category |
### warnings ### warnings
@@ -135,6 +136,7 @@
| views | TEXT | not null | | views | TEXT | not null |
| channelID | TEXT | not null | | channelID | TEXT | not null |
| timeSubmitted | INTEGER | not null | | timeSubmitted | INTEGER | not null |
| service | TEXT | not null, default 'YouTube' |
### config ### config
@@ -159,7 +161,7 @@
| views | INTEGER | not null | | views | INTEGER | not null |
| category | TEXT | not null, default 'sponsor' | | category | TEXT | not null, default 'sponsor' |
| actionType | TEXT | not null, default 'skip' | | actionType | TEXT | not null, default 'skip' |
| service | TEXT | not null, default 'Youtube' | | service | TEXT | not null, default 'YouTube' |
| videoDuration | INTEGER | not null, default '0' | | videoDuration | INTEGER | not null, default '0' |
| hidden | INTEGER | not null, default '0' | | hidden | INTEGER | not null, default '0' |
| reputation | REAL | not null, default '0' | | reputation | REAL | not null, default '0' |
@@ -208,11 +210,12 @@
| videoID | TEXT | not null | | videoID | TEXT | not null |
| hashedIP | TEXT | not null | | hashedIP | TEXT | not null |
| timeSubmitted | INTEGER | not null | | timeSubmitted | INTEGER | not null |
| service | TEXT | not null, default 'YouTube' |
| index | field | | index | field |
| -- | :--: | | -- | :--: |
| sponsorTimes_hashedIP | hashedIP | | sponsorTimes_hashedIP | hashedIP |
| privateDB_sponsorTimes_videoID | videoID | | privateDB_sponsorTimes_videoID_v2 | videoID, service |
### config ### config

4
databases/_private_indexes.sql
View File

@@ -5,9 +5,9 @@ CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP"
("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST) ("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default; TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID" CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID_v2"
ON public."sponsorTimes" USING btree ON public."sponsorTimes" USING btree
("videoID" ASC NULLS LAST) ("videoID" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST)
; ;
-- votes -- votes

4
databases/_sponsorTimes_indexes.sql
View File

@@ -53,9 +53,9 @@ CREATE INDEX IF NOT EXISTS "warnings_issueTime"
-- lockCategories -- lockCategories
CREATE INDEX IF NOT EXISTS "noSegments_videoID" CREATE INDEX IF NOT EXISTS "lockCategories_videoID"
ON public."lockCategories" USING btree ON public."lockCategories" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default; TABLESPACE pg_default;
-- categoryVotes -- categoryVotes

10
databases/_upgrade_private_3.sql Normal file
View File

@@ -0,0 +1,10 @@
BEGIN TRANSACTION;
ALTER TABLE "sponsorTimes" ADD "service" TEXT NOT NULL default 'YouTube';
-- UPDATE "sponsorTimes" SET "service" = "YouTube";
DROP INDEX IF EXISTS "privateDB_sponsorTimes_videoID";
UPDATE "config" SET value = 3 WHERE key = 'version';
COMMIT;

21
databases/_upgrade_sponsorTimes_24.sql Normal file
View File

@@ -0,0 +1,21 @@
BEGIN TRANSACTION;
ALTER TABLE "lockCategories" ADD "service" TEXT NOT NULL default 'YouTube';
UPDATE "lockCategories"
SET "service" = "sponsorTimes"."service"
FROM "sponsorTimes"
WHERE "lockCategories"."videoID" = "sponsorTimes"."videoID";
ALTER TABLE "unlistedVideos" ADD "service" TEXT NOT NULL default 'YouTube';
UPDATE "unlistedVideos"
SET "service" = "sponsorTimes"."service"
FROM "sponsorTimes"
WHERE "unlistedVideos"."videoID" = "sponsorTimes"."videoID";
DROP INDEX IF EXISTS "noSegments_videoID";
UPDATE "config" SET value = 24 WHERE key = 'version';
COMMIT;

4
src/routes/addUnlistedVideo.ts
View File

@@ -1,5 +1,6 @@
import { Request, Response } from "express"; import { Request, Response } from "express";
import { db } from "../databases/databases"; import { db } from "../databases/databases";
import { getService } from "../utils/getService";
import { Logger } from "../utils/logger"; import { Logger } from "../utils/logger";
/** /**
@@ -14,6 +15,7 @@ export function addUnlistedVideo(req: Request, res: Response): Response {
const year = req.body.year || 0; const year = req.body.year || 0;
const views = req.body.views || 0; const views = req.body.views || 0;
const channelID = req.body.channelID || "Unknown"; const channelID = req.body.channelID || "Unknown";
const service = getService(req.body.service);
if (videoID === undefined || typeof(videoID) !== "string" || videoID.length !== 11) { if (videoID === undefined || typeof(videoID) !== "string" || videoID.length !== 11) {
return res.status(400).send("Invalid parameters"); return res.status(400).send("Invalid parameters");
@@ -21,7 +23,7 @@ export function addUnlistedVideo(req: Request, res: Response): Response {
try { try {
const timeSubmitted = Date.now(); const timeSubmitted = Date.now();
db.prepare("run", `INSERT INTO "unlistedVideos" ("videoID", "year", "views", "channelID", "timeSubmitted") values (?, ?, ?, ?, ?)`, [videoID, year, views, channelID, timeSubmitted]); db.prepare("run", `INSERT INTO "unlistedVideos" ("videoID", "year", "views", "channelID", "timeSubmitted", "service") values (?, ?, ?, ?, ?, ?)`, [videoID, year, views, channelID, timeSubmitted, service]);
} catch (err) { } catch (err) {
Logger.error(err as string); Logger.error(err as string);
return res.sendStatus(500); return res.sendStatus(500);

23
src/routes/deleteLockCategories.ts
View File

@@ -2,14 +2,16 @@ import { Request, Response } from "express";
import { isUserVIP } from "../utils/isUserVIP"; import { isUserVIP } from "../utils/isUserVIP";
import { getHash } from "../utils/getHash"; import { getHash } from "../utils/getHash";
import { db } from "../databases/databases"; import { db } from "../databases/databases";
import { Category, VideoID } from "../types/segments.model"; import { Category, Service, VideoID } from "../types/segments.model";
import { UserID } from "../types/user.model"; import { UserID } from "../types/user.model";
import { getService } from "../utils/getService";
export async function deleteLockCategoriesEndpoint(req: Request, res: Response): Promise<Response> { export async function deleteLockCategoriesEndpoint(req: Request, res: Response): Promise<Response> {
// Collect user input data // Collect user input data
const videoID = req.body.videoID as VideoID; const videoID = req.body.videoID as VideoID;
const userID = req.body.userID as UserID; const userID = req.body.userID as UserID;
const categories = req.body.categories as Category[]; const categories = req.body.categories as Category[];
const service = getService(req.body.service);
// Check input data is valid // Check input data is valid
if (!videoID if (!videoID
@@ -33,7 +35,7 @@ export async function deleteLockCategoriesEndpoint(req: Request, res: Response):
}); });
} }
await deleteLockCategories(videoID, categories); await deleteLockCategories(videoID, categories, service);
return res.status(200).json({ message: `Removed lock categories entrys for video ${videoID}` }); return res.status(200).json({ message: `Removed lock categories entrys for video ${videoID}` });
} }
@@ -42,13 +44,20 @@ export async function deleteLockCategoriesEndpoint(req: Request, res: Response):
* *
* @param videoID * @param videoID
* @param categories If null, will remove all * @param categories If null, will remove all
* @param service
*/ */
export async function deleteLockCategories(videoID: VideoID, categories: Category[]): Promise<void> { export async function deleteLockCategories(videoID: VideoID, categories: Category[], service: Service): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', [videoID])).filter((entry: any) => { const entries = (
return categories === null || categories.indexOf(entry.category) !== -1; await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ? AND "service" = ?', [videoID, service]))
}); .filter((entry: any) => {
return categories === null || categories.indexOf(entry.category) !== -1;
});
for (const entry of entries) { for (const entry of entries) {
await db.prepare("run", 'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]); await db.prepare(
"run",
'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?',
[videoID, service, entry.category]
);
} }
} }

6
src/routes/getLockCategories.ts
View File

@@ -2,18 +2,20 @@ import { db } from "../databases/databases";
import { Logger } from "../utils/logger"; import { Logger } from "../utils/logger";
import { Request, Response } from "express"; import { Request, Response } from "express";
import { Category, VideoID } from "../types/segments.model"; import { Category, VideoID } from "../types/segments.model";
import { getService } from "../utils/getService";
export async function getLockCategories(req: Request, res: Response): Promise<Response> { export async function getLockCategories(req: Request, res: Response): Promise<Response> {
const videoID = req.query.videoID as VideoID; const videoID = req.query.videoID as VideoID;
const service = getService(req.query.service as string);
if (videoID == undefined) { if (videoID == undefined) {
//invalid request //invalid request
return res.sendStatus(400); return res.sendStatus(400);
} }
console.log(service);
try { try {
// Get existing lock categories markers // Get existing lock categories markers
const row = await db.prepare("all", 'SELECT "category", "reason" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string}[]; const row = await db.prepare("all", 'SELECT "category", "reason" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]) as {category: Category, reason: string}[];
// map categories to array in JS becaues of SQL incompatibilities // map categories to array in JS becaues of SQL incompatibilities
const categories = row.map(item => item.category); const categories = row.map(item => item.category);
if (categories.length === 0 || !categories[0]) return res.sendStatus(404); if (categories.length === 0 || !categories[0]) return res.sendStatus(404);

5
src/routes/getSkipSegments.ts
View File

@@ -27,8 +27,9 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
if (cache.shadowHiddenSegmentIPs[videoID] === undefined) cache.shadowHiddenSegmentIPs[videoID] = {}; if (cache.shadowHiddenSegmentIPs[videoID] === undefined) cache.shadowHiddenSegmentIPs[videoID] = {};
if (cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] === undefined) { if (cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] === undefined) {
cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ?', const service = getService(req?.query?.service as string);
[videoID, segment.timeSubmitted]) as { hashedIP: HashedIP }[]; cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ? AND "service" = ?',
[videoID, segment.timeSubmitted, service]) as { hashedIP: HashedIP }[];
} }
//if this isn't their ip, don't send it to them //if this isn't their ip, don't send it to them

14
src/routes/postLockCategories.ts
View File

@@ -4,6 +4,7 @@ import { isUserVIP } from "../utils/isUserVIP";
import { db } from "../databases/databases"; import { db } from "../databases/databases";
import { Request, Response } from "express"; import { Request, Response } from "express";
import { VideoIDHash } from "../types/segments.model"; import { VideoIDHash } from "../types/segments.model";
import { getService } from "../utils/getService";
export async function postLockCategories(req: Request, res: Response): Promise<string[]> { export async function postLockCategories(req: Request, res: Response): Promise<string[]> {
// Collect user input data // Collect user input data
@@ -11,6 +12,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
let userID = req.body.userID; let userID = req.body.userID;
const categories = req.body.categories; const categories = req.body.categories;
const reason: string = req.body.reason ?? ""; const reason: string = req.body.reason ?? "";
const service = getService(req.body.service);
// Check input data is valid // Check input data is valid
if (!videoID if (!videoID
@@ -37,7 +39,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
} }
// Get existing lock categories markers // Get existing lock categories markers
let noCategoryList = await db.prepare("all", 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]); let noCategoryList = await db.prepare("all", 'SELECT "category" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
if (!noCategoryList || noCategoryList.length === 0) { if (!noCategoryList || noCategoryList.length === 0) {
noCategoryList = []; noCategoryList = [];
} else { } else {
@@ -65,9 +67,9 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
// create database entry // create database entry
for (const category of categoriesToMark) { for (const category of categoriesToMark) {
try { try {
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason") VALUES(?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason]); await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason, service]);
} catch (err) { } catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`); Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}' (${service})`);
Logger.error(err as string); Logger.error(err as string);
res.status(500).json({ res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.", message: "Internal Server Error: Could not write marker to the database.",
@@ -85,10 +87,10 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
for (const category of overlapCategories) { for (const category of overlapCategories) {
try { try {
await db.prepare("run", await db.prepare("run",
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ?', 'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ? AND "service" = ?',
[reason, userID, videoID, category]); [reason, userID, videoID, category, service]);
} catch (err) { } catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`); Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID} (${service})'`);
Logger.error(err as string); Logger.error(err as string);
res.status(500).json({ res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.", message: "Internal Server Error: Could not write marker to the database.",

23
src/routes/postSkipSegments.ts
View File

@@ -168,7 +168,7 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// false for a pass - it was confusing and lead to this bug - any use of this function in // false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem. // the future could have the same problem.
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[] }) { submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service }) {
if (apiVideoInfo) { if (apiVideoInfo) {
const { err, data } = apiVideoInfo; const { err, data } = apiVideoInfo;
if (err) return false; if (err) return false;
@@ -238,7 +238,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
const startTime = parseFloat(segments[i].segment[0]); const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]); const endTime = parseFloat(segments[i].segment[1]);
const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime); const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime, submission.service);
// Send to Discord // Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook // Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
@@ -407,7 +407,8 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID,
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, isVIP: boolean, service:string, apiVideoInfo: APIVideoInfo, decreaseVotes: number): Promise<CheckResult & { decreaseVotes: number; } > { async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, isVIP: boolean, service:string, apiVideoInfo: APIVideoInfo, decreaseVotes: number): Promise<CheckResult & { decreaseVotes: number; } > {
// Auto moderator check // Auto moderator check
if (!isVIP && service == Service.YouTube) { if (!isVIP && service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments });//startTime, endTime, category: segments[i].category}); const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service });//startTime, endTime, category: segments[i].category});
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") { if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
// If NB automod rejects, the submission will start with -2 votes. // If NB automod rejects, the submission will start with -2 votes.
// Note, if one submission is bad all submissions will be affected. // Note, if one submission is bad all submissions will be affected.
@@ -431,8 +432,8 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
}; };
} }
async function updateDataIfVideoDurationChange(videoID: VideoID, service: string, videoDuration: VideoDuration, videoDurationParam: VideoDuration) { async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
let lockedCategoryList = await db.prepare("all", 'SELECT category, reason from "lockCategories" where "videoID" = ?', [videoID]); let lockedCategoryList = await db.prepare("all", 'SELECT category, reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
const previousSubmissions = await db.prepare("all", const previousSubmissions = await db.prepare("all",
`SELECT "videoDuration", "UUID" `SELECT "videoDuration", "UUID"
@@ -465,7 +466,7 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: string
await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]); await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
} }
lockedCategoryList = []; lockedCategoryList = [];
deleteLockCategories(videoID, null); deleteLockCategories(videoID, null, service);
} }
return { return {
@@ -478,7 +479,7 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: string
// Disable max submissions for now // Disable max submissions for now
// Disable IP ratelimiting for now // Disable IP ratelimiting for now
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
async function checkRateLimit(userID:string, videoID: VideoID, timeSubmitted: number, hashedIP: string, options: { async function checkRateLimit(userID:string, videoID: VideoID, service: Service, timeSubmitted: number, hashedIP: string, options: {
enableCheckByIP: boolean; enableCheckByIP: boolean;
enableCheckByUserID: boolean; enableCheckByUserID: boolean;
} = { } = {
@@ -489,7 +490,7 @@ async function checkRateLimit(userID:string, videoID: VideoID, timeSubmitted: nu
if (options.enableCheckByIP) { if (options.enableCheckByIP) {
//check to see if this ip has submitted too many sponsors today //check to see if this ip has submitted too many sponsors today
const rateLimitCheckRow = await privateDB.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "hashedIP" = ? AND "videoID" = ? AND "timeSubmitted" > ?`, [hashedIP, videoID, yesterday]); const rateLimitCheckRow = await privateDB.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "hashedIP" = ? AND "videoID" = ? AND "timeSubmitted" > ? AND "service" = ?`, [hashedIP, videoID, yesterday, service]);
if (rateLimitCheckRow.count >= 10) { if (rateLimitCheckRow.count >= 10) {
//too many sponsors for the same video from the same ip address //too many sponsors for the same video from the same ip address
@@ -613,7 +614,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//get current time //get current time
const timeSubmitted = Date.now(); const timeSubmitted = Date.now();
// const rateLimitCheckResult = checkRateLimit(userID, videoID, timeSubmitted, hashedIP); // const rateLimitCheckResult = checkRateLimit(userID, videoID, service, timeSubmitted, hashedIP);
// if (!rateLimitCheckResult.pass) { // if (!rateLimitCheckResult.pass) {
// return res.status(rateLimitCheckResult.errorCode).send(rateLimitCheckResult.errorMessage); // return res.status(rateLimitCheckResult.errorCode).send(rateLimitCheckResult.errorMessage);
// } // }
@@ -635,7 +636,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//this can just be a hash of the data //this can just be a hash of the data
//it's better than generating an actual UUID like what was used before //it's better than generating an actual UUID like what was used before
//also better for duplication checking //also better for duplication checking
const UUID = getSubmissionUUID(videoID, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1])); const UUID = getSubmissionUUID(videoID, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service);
const hashedVideoID = getHash(videoID, 1); const hashedVideoID = getHash(videoID, 1);
const startingLocked = isVIP ? 1 : 0; const startingLocked = isVIP ? 1 : 0;
@@ -648,7 +649,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
); );
//add to private db as well //add to private db as well
await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?)`, [videoID, hashedIP, timeSubmitted]); await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?, ?)`, [videoID, hashedIP, timeSubmitted, service]);
// Clear redis cache for this video // Clear redis cache for this video
QueryCacher.clearVideoCache({ QueryCacher.clearVideoCache({

4
src/routes/shadowBanUser.ts
View File

@@ -57,7 +57,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
//find all previous submissions and unhide them //find all previous submissions and unhide them
if (unHideOldSubmissions) { if (unHideOldSubmissions) {
const segmentsToIgnore = (await db.prepare("all", `SELECT "UUID" FROM "sponsorTimes" st const segmentsToIgnore = (await db.prepare("all", `SELECT "UUID" FROM "sponsorTimes" st
JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?` JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category AND "st"."service" = "ns"."service" WHERE "st"."userID" = ?`
, [userID])).map((item: {UUID: string}) => item.UUID); , [userID])).map((item: {UUID: string}) => item.UUID);
const allSegments = (await db.prepare("all", `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID])) const allSegments = (await db.prepare("all", `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID]))
.map((item: {UUID: string}) => item.UUID); .map((item: {UUID: string}) => item.UUID);
@@ -120,7 +120,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
async function unHideSubmissions(categories: string[], userID: UserID) { async function unHideSubmissions(categories: string[], userID: UserID) {
await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")}) await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})
AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
"sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]); "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
// clear cache for all old videos // clear cache for all old videos
(await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID])) (await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]))

3
src/routes/voteOnSponsorTime.ts
View File

@@ -290,7 +290,8 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
if (!isVIP && type != 1) { if (!isVIP && type != 1) {
const isSegmentLocked = async () => !!(await db.prepare("get", `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked; const isSegmentLocked = async () => !!(await db.prepare("get", `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked;
const isVideoLocked = async () => !!(await db.prepare("get", `SELECT "lockCategories".category from "lockCategories" left join "sponsorTimes" const isVideoLocked = async () => !!(await db.prepare("get", `SELECT "lockCategories".category from "lockCategories" left join "sponsorTimes"
on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and "lockCategories".category = "sponsorTimes".category) on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and
"lockCategories"."service" = "sponsorTimes"."service" and "lockCategories".category = "sponsorTimes".category)
where "UUID" = ?`, [UUID])); where "UUID" = ?`, [UUID]));
if (await isSegmentLocked() || await isVideoLocked()) { if (await isSegmentLocked() || await isVideoLocked()) {

13
src/utils/getSubmissionUUID.ts
View File

@@ -1,8 +1,15 @@
import { getHash } from "./getHash"; import { getHash } from "./getHash";
import { HashedValue } from "../types/hash.model"; import { HashedValue } from "../types/hash.model";
import { ActionType, VideoID } from "../types/segments.model"; import { ActionType, VideoID, Service } from "../types/segments.model";
import { UserID } from "../types/user.model"; import { UserID } from "../types/user.model";
export function getSubmissionUUID(videoID: VideoID, actionType: ActionType, userID: UserID, startTime: number, endTime: number): HashedValue{ export function getSubmissionUUID(
return `4${getHash(`${videoID}${startTime}${endTime}${userID}${actionType}`, 1)}` as HashedValue; videoID: VideoID,
actionType: ActionType,
userID: UserID,
startTime: number,
endTime: number,
service: Service
) : HashedValue {
return `5${getHash(`${videoID}${startTime}${endTime}${userID}${actionType}${service}`, 1)}` as HashedValue;
} }

2
src/utils/reputation.ts
View File

@@ -38,7 +38,7 @@ export async function getReputation(userID: UserID): Promise<number> {
c."category" = "a"."category" LIMIT 1) c."category" = "a"."category" LIMIT 1)
AND EXISTS ( AND EXISTS (
SELECT * FROM "lockCategories" as l SELECT * FROM "lockCategories" as l
WHERE l."videoID" = "a"."videoID" AND l."category" = "a"."category" LIMIT 1) WHERE l."videoID" = "a"."videoID" AND l."service" = "a"."service" AND l."category" = "a"."category" LIMIT 1)
THEN 1 ELSE 0 END) AS "mostUpvotedInLockedVideoSum" THEN 1 ELSE 0 END) AS "mostUpvotedInLockedVideoSum"
FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, weekAgo, pastDate, userID]) as Promise<ReputationDBResult>; FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, weekAgo, pastDate, userID]) as Promise<ReputationDBResult>;

77
test/cases/getLockCategories.ts
View File

@@ -4,19 +4,21 @@ import assert from "assert";
import { client } from "../utils/httpClient"; import { client } from "../utils/httpClient";
const endpoint = "/api/lockCategories"; const endpoint = "/api/lockCategories";
const getLockCategories = (videoID: string) => client.get(endpoint, { params: { videoID } }); const getLockCategories = (videoID: string) => client.get(endpoint, { params: { videoID } });
const getLockCategoriesWithService = (videoID: string, service: string) => client.get(endpoint, { params: { videoID, service } });
describe("getLockCategories", () => { describe("getLockCategories", () => {
before(async () => { before(async () => {
const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
await db.prepare("run", insertVipUserQuery, [getHash("getLockCategoriesVIP")]); await db.prepare("run", insertVipUserQuery, [getHash("getLockCategoriesVIP")]);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES (?, ?, ?, ?)'; const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "sponsor", "1-short"]); await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "sponsor", "1-short", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "interaction", "1-longer-reason"]); await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "interaction", "1-longer-reason", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock2", "preview", "2-reason"]); await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock2", "preview", "2-reason", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "nonmusic", "3-reason"]); await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "nonmusic", "3-reason", "PeerTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "sponsor", "3-reason", "YouTube"]);
}); });
it("Should update the database version when starting the application", async () => { it("Should update the database version when starting the application", async () => {
@@ -74,4 +76,69 @@ describe("getLockCategories", () => {
}) })
.catch(err => done(err)); .catch(err => done(err));
}); });
it("Should be able to get multiple locks with service", (done) => {
getLockCategoriesWithService("getLock1", "YouTube")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
categories: [
"sponsor",
"interaction"
],
reason: "1-longer-reason"
};
assert.deepStrictEqual(res.data, expected);
done();
})
.catch(err => done(err));
});
it("Should be able to get single locks with service", (done) => {
getLockCategoriesWithService("getLock3", "PeerTube")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
categories: [
"nonmusic"
],
reason: "3-reason"
};
assert.deepStrictEqual(res.data, expected);
done();
})
.catch(err => done(err));
});
it("Should be able to get single locks with service", (done) => {
getLockCategoriesWithService("getLock3", "Youtube")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
categories: [
"sponsor"
],
reason: "3-reason"
};
assert.deepStrictEqual(res.data, expected);
done();
})
.catch(err => done(err));
});
it("should return result from Youtube service if service not match", (done) => {
getLockCategoriesWithService("getLock3", "Dailymotion")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
categories: [
"sponsor"
],
reason: "3-reason"
};
assert.deepStrictEqual(res.data, expected);
done();
})
.catch(err => done(err));
});
}); });

6
test/cases/getSubmissionUUID.ts
View File

@@ -1,10 +1,12 @@
import { getSubmissionUUID } from "../../src/utils/getSubmissionUUID"; import { getSubmissionUUID } from "../../src/utils/getSubmissionUUID";
import assert from "assert"; import assert from "assert";
import { ActionType, VideoID } from "../../src/types/segments.model"; import { ActionType, VideoID, Service } from "../../src/types/segments.model";
import { UserID } from "../../src/types/user.model"; import { UserID } from "../../src/types/user.model";
describe("getSubmissionUUID", () => { describe("getSubmissionUUID", () => {
it("Should return the hashed value", () => { it("Should return the hashed value", () => {
assert.strictEqual(getSubmissionUUID("video001" as VideoID, "skip" as ActionType, "testuser001" as UserID, 13.33337, 42.000001), "48ad47e445e67a7b963d9200037b36ec706eddcb752fdadc7bb2f061b56be6a23"); assert.strictEqual(
getSubmissionUUID("video001" as VideoID, "skip" as ActionType, "testuser001" as UserID, 13.33337, 42.000001, Service.YouTube),
"529611b4cdd7319e705a32ae9557a02e59c8dbc1306097b2d2d5807c6405e9b1a");
}); });
}); });

18
test/cases/lockCategoriesRecords.ts
View File

@@ -23,18 +23,18 @@ describe("lockCategoriesRecords", () => {
const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)'; const insertVipUserQuery = 'INSERT INTO "vipUsers" ("userID") VALUES (?)';
await db.prepare("run", insertVipUserQuery, [lockVIPUserHash]); await db.prepare("run", insertVipUserQuery, [lockVIPUserHash]);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason") VALUES (?, ?, ?, ?)'; const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "sponsor", "reason-1"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "sponsor", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "intro", "reason-1"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id", "intro", "reason-1", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "sponsor", "reason-2"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "sponsor", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "intro", "reason-2"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "no-segments-video-id-1", "intro", "reason-2", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "sponsor", "reason-3"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "lockCategoryVideo", "sponsor", "reason-3", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "sponsor", "reason-4"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record", "sponsor", "reason-4", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "sponsor", "reason-5"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "sponsor", "reason-5", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "intro", "reason-5"]); await db.prepare("run", insertLockCategoryQuery, [lockVIPUserHash, "delete-record-1", "intro", "reason-5", "YouTube"]);
}); });
it("Should update the database version when starting the application", async () => { it("Should update the database version when starting the application", async () => {