diff --git a/DatabaseSchema.md b/DatabaseSchema.md index f0d0785..68daf48 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -94,6 +94,7 @@ | -- | :--: | -- | | videoID | TEXT | not null | | userID | TEXT | not null | +| actionType | TEXT | not null, default 'skip' | | category | TEXT | not null | | hashedVideoID | TEXT | not null, default '' | | reason | TEXT | not null, default '' | diff --git a/databases/_upgrade_sponsorTimes_29.sql b/databases/_upgrade_sponsorTimes_29.sql new file mode 100644 index 0000000..8c5e248 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_29.sql @@ -0,0 +1,21 @@ +BEGIN TRANSACTION; + +CREATE TABLE "sqlb_temp_table_29" ( + "videoID" TEXT NOT NULL, + "userID" TEXT NOT NULL, + "actionType" TEXT NOT NULL DEFAULT 'skip', + "category" TEXT NOT NULL, + "hashedVideoID" TEXT NOT NULL default '', + "reason" TEXT NOT NULL default '', + "service" TEXT NOT NULL default 'YouTube' +); + +INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'skip',"category","hashedVideoID","reason","service" FROM "lockCategories"; +INSERT INTO sqlb_temp_table_29 SELECT "videoID","userID",'mute',"category","hashedVideoID","reason","service" FROM "lockCategories"; + +DROP TABLE "lockCategories"; +ALTER TABLE sqlb_temp_table_29 RENAME TO "lockCategories"; + +UPDATE "config" SET value = 29 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index c666a90..65b9b10 100644 --- a/src/config.ts +++ b/src/config.ts @@ -21,8 +21,8 @@ addDefaults(config, { webhooks: [], categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"], categorySupport: { - sponsor: ["skip", "mute"], - selfpromo: ["skip", "mute"], + sponsor: ["skip", "mute", "full"], + selfpromo: ["skip", "mute", "full"], interaction: ["skip", "mute"], intro: ["skip", "mute"], outro: ["skip", "mute"], diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 9633cb3..81c0af2 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -167,7 +167,7 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P // amountOfChoices specifies the maximum amount of choices to return, 1 or more. // Choices are unique // If a predicate is given, it will only filter choices following it, and will leave the rest in the list -function getWeightedRandomChoice(choices: T[], amountOfChoices: number, predicate?: (choice: T) => void): T[] { +function getWeightedRandomChoice(choices: T[], amountOfChoices: number, filterLocked = false, predicate?: (choice: T) => void): T[] { //trivial case: no need to go through the whole process if (amountOfChoices >= choices.length) { return choices; @@ -183,6 +183,10 @@ function getWeightedRandomChoice(choices: T[], amountOf const splitArray = partition(choices, predicate); filteredChoices = splitArray[0]; forceIncludedChoices = splitArray[1]; + + if (filterLocked && filteredChoices.some((value) => value.locked)) { + filteredChoices = filteredChoices.filter((value) => value.locked); + } } //assign a weight to each choice @@ -200,7 +204,7 @@ function getWeightedRandomChoice(choices: T[], amountOf // Nothing to filter for if (amountOfChoices >= choicesWithWeights.length) { - return choices; + return [...forceIncludedChoices, ...filteredChoices]; } //iterate and find amountOfChoices choices @@ -230,11 +234,12 @@ async function chooseSegments(videoID: VideoID, service: Service, segments: DBSe ? await QueryCacher.get(fetchData, skipSegmentGroupsKey(videoID, service)) : await fetchData(); - // Filter for only 1 item for POI categories - return getWeightedRandomChoice(groups, 1, (choice) => getCategoryActionType(choice.segments[0].category) === CategoryActionType.POI) - .map(//randomly choose 1 good segment per group and return them - group => getWeightedRandomChoice(group.segments, 1)[0] - ); + // Filter for only 1 item for POI categories and Full video + let chosenGroups = getWeightedRandomChoice(groups, 1, true, (choice) => choice.segments[0].actionType === ActionType.Full); + chosenGroups = getWeightedRandomChoice(chosenGroups, 1, true, (choice) => getCategoryActionType(choice.segments[0].category) === CategoryActionType.POI); + return chosenGroups.map(//randomly choose 1 good segment per group and return them + group => getWeightedRandomChoice(group.segments, 1)[0] + ); } //This function will find segments that are contained inside of eachother, called similar segments @@ -300,7 +305,7 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm group.segments.forEach((segment) => { const bestGroup = result.find((group) => { // At least one segment in the group must have high % overlap or the same action type - // Since POI segments will always have 0 overlap, they will always be in their own groups + // Since POI and Full video segments will always have <= 0 overlap, they will always be in their own groups return group.segments.some((compareSegment) => { const overlap = Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime); const overallDuration = Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime); diff --git a/src/routes/postLockCategories.ts b/src/routes/postLockCategories.ts index e10eeda..400acd3 100644 --- a/src/routes/postLockCategories.ts +++ b/src/routes/postLockCategories.ts @@ -3,14 +3,15 @@ import { getHashCache } from "../utils/getHashCache"; import { isUserVIP } from "../utils/isUserVIP"; import { db } from "../databases/databases"; import { Request, Response } from "express"; -import { VideoIDHash } from "../types/segments.model"; +import { ActionType, Category, VideoIDHash } from "../types/segments.model"; import { getService } from "../utils/getService"; export async function postLockCategories(req: Request, res: Response): Promise { // Collect user input data const videoID = req.body.videoID; let userID = req.body.userID; - const categories = req.body.categories; + const categories = req.body.categories as Category[]; + const actionTypes = req.body.actionTypes as ActionType[] || [ActionType.Skip, ActionType.Mute]; const reason: string = req.body.reason ?? ""; const service = getService(req.body.service); @@ -20,6 +21,8 @@ export async function postLockCategories(req: Request, res: Response): Promise { - return obj.category; - }); + const existingLocks = (await db.prepare("all", 'SELECT "category", "actionType" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service])) as + { category: Category, actionType: ActionType }[]; + + const filteredCategories = filterData(categories); + const filteredActionTypes = filterData(actionTypes); + + const locksToApply: { category: Category, actionType: ActionType }[] = []; + const overwrittenLocks: { category: Category, actionType: ActionType }[] = []; + for (const category of filteredCategories) { + for (const actionType of filteredActionTypes) { + if (!existingLocks.some((lock) => lock.category === category && lock.actionType === actionType)) { + locksToApply.push({ + category, + actionType + }); + } else { + overwrittenLocks.push({ + category, + actionType + }); + } + } } - // get user categories not already submitted that match accepted format - let filteredCategories = categories.filter((category) => { - return !!category.match(/^[_a-zA-Z]+$/); - }); - // remove any duplicates - filteredCategories = filteredCategories.filter((category, index) => { - return filteredCategories.indexOf(category) === index; - }); - - const categoriesToMark = filteredCategories.filter((category) => { - return noCategoryList.indexOf(category) === -1; - }); - // calculate hash of videoID const hashedVideoID: VideoIDHash = await getHashCache(videoID, 1); // create database entry - for (const category of categoriesToMark) { + for (const lock of locksToApply) { try { - await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason, service]); + await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "actionType", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?, ?)`, [videoID, userID, lock.actionType, lock.category, hashedVideoID, reason, service]); } catch (err) { - Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}' (${service})`); + Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`); Logger.error(err as string); res.status(500).json({ message: "Internal Server Error: Could not write marker to the database.", @@ -78,19 +82,14 @@ export async function postLockCategories(req: Request, res: Response): Promise { - return noCategoryList.indexOf(category) !== -1; - }); - - for (const category of overlapCategories) { + for (const lock of overwrittenLocks) { try { await db.prepare("run", - 'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ? AND "service" = ?', - [reason, userID, videoID, category, service]); + 'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "actionType" = ? AND "category" = ? AND "service" = ?', + [reason, userID, videoID, lock.actionType, lock.category, service]); } catch (err) { - Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID} (${service})'`); + Logger.error(`Error submitting 'lockCategories' marker for category '${lock.category}' and actionType '${lock.actionType}' for video '${videoID}' (${service})`); Logger.error(err as string); res.status(500).json({ message: "Internal Server Error: Could not write marker to the database.", @@ -100,6 +99,20 @@ export async function postLockCategories(req: Request, res: Response): Promise locksToApply.some((lock) => category === lock.category)))] + : [...filteredCategories], // Legacy + submittedValues: [...locksToApply, ...overwrittenLocks], + }); +} + +function filterData(data: T[]): T[] { + // get user categories not already submitted that match accepted format + const filtered = data.filter((elem) => { + return !!elem.match(/^[_a-zA-Z]+$/); + }); + // remove any duplicates + return filtered.filter((elem, index) => { + return filtered.indexOf(elem) === index; }); } diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 9f6598f..50c527e 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -9,17 +9,18 @@ import { getIP } from "../utils/getIP"; import { getFormattedTime } from "../utils/getFormattedTime"; import { dispatchEvent } from "../utils/webhookUtils"; import { Request, Response } from "express"; -import { ActionType, Category, CategoryActionType, IncomingSegment, SegmentUUID, Service, VideoDuration, VideoID } from "../types/segments.model"; +import { ActionType, Category, CategoryActionType, IncomingSegment, IPAddress, SegmentUUID, Service, VideoDuration, VideoID } from "../types/segments.model"; import { deleteLockCategories } from "./deleteLockCategories"; import { getCategoryActionType } from "../utils/categoryInfo"; import { QueryCacher } from "../utils/queryCacher"; import { getReputation } from "../utils/reputation"; import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model"; -import { UserID } from "../types/user.model"; +import { HashedUserID, UserID } from "../types/user.model"; import { isUserVIP } from "../utils/isUserVIP"; import { parseUserAgent } from "../utils/userAgent"; import { getService } from "../utils/getService"; import axios from "axios"; +import { vote } from "./voteOnSponsorTime"; type CheckResult = { pass: boolean, @@ -238,7 +239,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, const startTime = parseFloat(segments[i].segment[0]); const endTime = parseFloat(segments[i].segment[1]); - const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime, submission.service); + const UUID = getSubmissionUUID(submission.videoID, segments[i].category, segments[i].actionType, submission.userID, startTime, endTime, submission.service); // Send to Discord // Note, if this is too spammy. Consider sending all the segments as one Webhook sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data); @@ -343,7 +344,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming return CHECK_PASS; } -async function checkEachSegmentValid(userID: string, videoID: VideoID, +async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, userID: HashedUserID, videoID: VideoID, segments: IncomingSegment[], service: string, isVIP: boolean, lockedCategoryList: Array): Promise { for (let i = 0; i < segments.length; i++) { @@ -357,7 +358,7 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID, } // Reject segment if it's in the locked categories list - const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category); + const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category && segments[i].actionType === c.actionType); if (!isVIP && lockIndex !== -1) { // TODO: Do something about the fradulent submission Logger.warn(`Caught a submission for a locked category. userID: '${userID}', videoID: '${videoID}', category: '${segments[i].category}', times: ${segments[i].segment}`); @@ -383,8 +384,10 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID, if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime - || (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable && startTime === endTime) - || (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime)) { + || (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable + && segments[i].actionType !== ActionType.Full && startTime === endTime) + || (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime) + || (segments[i].actionType === ActionType.Full && (startTime !== 0 || endTime !== 0))) { //invalid request return { pass: false, errorMessage: "One of your segments times are invalid (too short, startTime before endTime, etc.)", errorCode: 400 }; } @@ -394,16 +397,24 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID, return { pass: false, errorMessage: `POI cannot be that early`, errorCode: 400 }; } - if (!isVIP && segments[i].category === "sponsor" && Math.abs(startTime - endTime) < 1) { + if (!isVIP && segments[i].category === "sponsor" + && segments[i].actionType !== ActionType.Full && Math.abs(startTime - endTime) < 1) { // Too short - return { pass: false, errorMessage: "Sponsors must be longer than 1 second long", errorCode: 400 }; + return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 }; } //check if this info has already been submitted before - const duplicateCheck2Row = await db.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ? + const duplicateCheck2Row = await db.prepare("get", `SELECT "UUID" FROM "sponsorTimes" WHERE "startTime" = ? and "endTime" = ? and "category" = ? and "actionType" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, segments[i].actionType, videoID, service]); - if (duplicateCheck2Row.count > 0) { - return { pass: false, errorMessage: "Sponsors has already been submitted before.", errorCode: 409 }; + if (duplicateCheck2Row) { + if (segments[i].actionType === ActionType.Full) { + // Forward as vote + await vote(rawIP, duplicateCheck2Row.UUID, paramUserID, 1); + segments[i].ignoreSegment = true; + continue; + } else { + return { pass: false, errorMessage: "Segment has already been submitted before.", errorCode: 409 }; + } } } @@ -439,13 +450,14 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array -2 AND "videoDuration" != 0`, [videoID, service] ) as {videoDuration: VideoDuration, UUID: SegmentUUID}[]; @@ -573,15 +585,15 @@ export async function postSkipSegments(req: Request, res: Response): Promise { + , hashedIP: HashedIP, finalResponse: FinalResponse): Promise<{ status: number, message?: string }> { // Check if they've already made a vote const usersLastVoteInfo = await privateDB.prepare("get", `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]); if (usersLastVoteInfo?.category === category) { // Double vote, ignore - return res.sendStatus(finalResponse.finalStatus); + return { status: finalResponse.finalStatus }; } const videoInfo = (await db.prepare("get", `SELECT "category", "videoID", "hashedVideoID", "service", "userID", "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number}; if (!videoInfo) { // Submission doesn't exist - return res.status(400).send("Submission doesn't exist."); + return { status: 400, message: "Submission doesn't exist." }; } if (!config.categoryList.includes(category)) { - return res.status(400).send("Category doesn't exist."); + return { status: 400, message: "Category doesn't exist." }; } if (getCategoryActionType(category) !== CategoryActionType.Skippable) { - return res.status(400).send("Cannot vote for this category"); + return { status: 400, message: "Cannot vote for this category" }; } // Ignore vote if the next category is locked const nextCategoryLocked = await db.prepare("get", `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, [videoInfo.videoID, videoInfo.service, category]); if (nextCategoryLocked && !isVIP) { - return res.sendStatus(200); + return { status: 200 }; } // Ignore vote if the segment is locked if (!isVIP && videoInfo.locked === 1) { - return res.sendStatus(200); + return { status: 200 }; } const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]); @@ -279,7 +279,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i QueryCacher.clearSegmentCache(videoInfo); - return res.sendStatus(finalResponse.finalStatus); + return { status: finalResponse.finalStatus }; } export function getUserID(req: Request): UserID { @@ -289,16 +289,30 @@ export function getUserID(req: Request): UserID { export async function voteOnSponsorTime(req: Request, res: Response): Promise { const UUID = req.query.UUID as SegmentUUID; const paramUserID = getUserID(req); - let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined; + const type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined; const category = req.query.category as Category; + const ip = getIP(req); + const result = await vote(ip, UUID, paramUserID, type, category); + + const response = res.status(result.status); + if (result.message) { + return response.send(result.message); + } else if (result.json) { + return response.json(result.json); + } else { + return response.send(); + } +} + +export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID, type: number, category?: Category): Promise<{ status: number, message?: string, json?: unknown }> { if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) { //invalid request - return res.sendStatus(400); + return { status: 400 }; } if (paramUserID.length < 30 && config.mode !== "test") { // Ignore this vote, invalid - return res.sendStatus(200); + return { status: 200 }; } //hash the userID @@ -314,9 +328,6 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise