From 6554e142cc639b2f4d69ab4aa79c07e306470d4d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 4 Apr 2021 23:12:26 -0400 Subject: [PATCH 1/6] Add highlight category --- config.json.example | 2 +- src/config.ts | 2 +- src/routes/postSkipSegments.ts | 5 ++-- src/types/segments.model.ts | 2 +- test.json | 2 +- test/cases/postSkipSegments.ts | 45 ++++++++++++++++++++++++++++++++++ 6 files changed, 52 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index f8548c1..3d9b2ef 100644 --- a/config.json.example +++ b/config.json.example @@ -22,7 +22,7 @@ "mode": "development", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "preview", "music_offtopic", "highlight"], // List of supported categories any other category will be rejected "getTopUsersCacheTimeMinutes": 5, // cacheTime for getTopUsers result in minutes "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire "hoursAfterWarningExpire": 24, diff --git a/src/config.ts b/src/config.ts index 7b15987..66960ab 100644 --- a/src/config.ts +++ b/src/config.ts @@ -16,7 +16,7 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"], + categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"], maxNumberOfActiveWarnings: 3, hoursAfterWarningExpires: 24, adminUserID: "", diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 1f380da..2e0ba6c 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -4,7 +4,7 @@ import {db, privateDB} from '../databases/databases'; import {YouTubeAPI} from '../utils/youtubeApi'; import {getSubmissionUUID} from '../utils/getSubmissionUUID'; import fetch from 'node-fetch'; -import isoDurations from 'iso8601-duration'; +import isoDurations, { end } from 'iso8601-duration'; import {getHash} from '../utils/getHash'; import {getIP} from '../utils/getIP'; import {getFormattedTime} from '../utils/getFormattedTime'; @@ -425,7 +425,8 @@ export async function postSkipSegments(req: Request, res: Response) { let endTime = parseFloat(segments[i].segment[1]); if (isNaN(startTime) || isNaN(endTime) - || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { + || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime + || (segments[i].category !== "highlight" && startTime === endTime) || (segments[i].category === "highlight" && startTime !== endTime)) { //invalid request res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)"); return; diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index a95bac3..329c41a 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -4,7 +4,7 @@ import { SBRecord } from "./lib.model"; export type SegmentUUID = string & { __segmentUUIDBrand: unknown }; export type VideoID = string & { __videoIDBrand: unknown }; export type VideoDuration = number & { __videoDurationBrand: unknown }; -export type Category = string & { __categoryBrand: unknown }; +export type Category = ("sponsor" | "selfpromo" | "interaction" | "intro" | "outro" | "preview" | "music_offtopic" | "highlight") & { __categoryBrand: unknown }; export type VideoIDHash = VideoID & HashedValue; export type IPAddress = string & { __ipAddressBrand: unknown }; export type HashedIP = IPAddress & HashedValue; diff --git a/test.json b/test.json index 58eb72a..3e147fe 100644 --- a/test.json +++ b/test.json @@ -49,7 +49,7 @@ ] } ], - "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"], + "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"], "maxNumberOfActiveWarnings": 3, "hoursAfterWarningExpires": 24, "rateLimit": { diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 512c2ee..e084112 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -500,6 +500,51 @@ describe('postSkipSegments', () => { .catch(err => done("Couldn't call endpoint")); }); + it('Should be rejected if segment starts and ends at the same time', (done: Done) => { + fetch(getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=90&endTime=90&userID=testing&category=intro", { + method: 'POST', + }) + .then(async res => { + if (res.status === 400) done(); // pass + else { + const body = await res.text(); + done("non 400 status code: " + res.status + " (" + body + ")"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be accepted if highlight segment starts and ends at the same time', (done: Done) => { + fetch(getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30&userID=testing&category=highlight", { + method: 'POST', + }) + .then(async res => { + if (res.status === 200) done(); // pass + else { + const body = await res.text(); + done("non 200 status code: " + res.status + " (" + body + ")"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be rejected if highlight segment doesn\'t start and end at the same time', (done: Done) => { + fetch(getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=highlight", { + method: 'POST', + }) + .then(async res => { + if (res.status === 400) done(); // pass + else { + const body = await res.text(); + done("non 400 status code: " + res.status + " (" + body + ")"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + it('Should be rejected if a sponsor is less than 1 second', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing", { From 7bf09906d34c2cba89cd11c0295fff13255a653d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 5 Apr 2021 22:33:28 -0400 Subject: [PATCH 2/6] Prevent voting for highlight category --- src/routes/voteOnSponsorTime.ts | 4 ++++ test/cases/voteOnSponsorTime.ts | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index c0aaac0..361e02f 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -170,6 +170,10 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i res.status(400).send("Category doesn't exist."); return; } + if (category === "highlight") { + res.status(400).send("Cannot vote for this category"); + return; + } const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]); diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts index 47a1f19..8e1cb61 100644 --- a/test/cases/voteOnSponsorTime.ts +++ b/test/cases/voteOnSponsorTime.ts @@ -264,6 +264,24 @@ describe('voteOnSponsorTime', () => { .catch(err => done(err)); }); + it('Should not able to change to highlight category', (done: Done) => { + fetch(getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category&category=highlight") + .then(async res => { + if (res.status === 400) { + let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["incorrect-category"]); + if (row.category === "sponsor") { + done(); + } else { + done("Vote did not succeed. Submission went from sponsor to " + row.category); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + it('Should be able to change your vote for a category and it should add your vote to the database', (done: Done) => { fetch(getbaseURL() + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro") From 8088f3763202a813dfe76ec0a4de3126db24f48f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 5 Apr 2021 23:48:51 -0400 Subject: [PATCH 3/6] Only return one segment for highlight category --- src/routes/getSkipSegments.ts | 12 +++++++----- src/routes/postSkipSegments.ts | 6 ++++-- src/routes/voteOnSponsorTime.ts | 5 +++-- src/types/segments.model.ts | 5 +++++ src/utils/categoryInfo.ts | 10 ++++++++++ test/cases/getSkipSegmentsByHash.ts | 19 ++++++++++++++++++- test/utils.ts | 2 +- 7 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 src/utils/categoryInfo.ts diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index d36a91a..530edd9 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -4,7 +4,8 @@ import { config } from '../config'; import { db, privateDB } from '../databases/databases'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { SBRecord } from '../types/lib.model'; -import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; +import { Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; +import { getCategoryActionType } from '../utils/categoryInfo'; import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; @@ -40,7 +41,8 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: const filteredSegments = segments.filter((_, index) => shouldFilter[index]); - return chooseSegments(filteredSegments).map((chosenSegment) => ({ + const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1 + return chooseSegments(filteredSegments, maxSegments).map((chosenSegment) => ({ category, segment: [chosenSegment.startTime, chosenSegment.endTime], UUID: chosenSegment.UUID, @@ -206,7 +208,7 @@ function getWeightedRandomChoice(choices: T[], amountOf //Only one similar time will be returned, randomly generated based on the sqrt of votes. //This allows new less voted items to still sometimes appear to give them a chance at getting votes. //Segments with less than -1 votes are already ignored before this function is called -function chooseSegments(segments: DBSegment[]): DBSegment[] { +function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { //Create groups of segments that are similar to eachother //Segments must be sorted by their startTime so that we can build groups chronologically: //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group @@ -240,8 +242,8 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] { } }); - //if there are too many groups, find the best 8 - return getWeightedRandomChoice(overlappingSegmentsGroups, 32).map( + //if there are too many groups, find the best ones + return getWeightedRandomChoice(overlappingSegmentsGroups, max).map( //randomly choose 1 good segment per group and return them group => getWeightedRandomChoice(group.segments, 1)[0], ); diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 2e0ba6c..0923182 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -13,8 +13,9 @@ import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; -import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; +import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; import { deleteNoSegments } from './deleteNoSegments'; +import { getCategoryActionType } from '../utils/categoryInfo'; interface APIVideoInfo { err: string | boolean, @@ -426,7 +427,8 @@ export async function postSkipSegments(req: Request, res: Response) { if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime - || (segments[i].category !== "highlight" && startTime === endTime) || (segments[i].category === "highlight" && startTime !== endTime)) { + || (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable && startTime === endTime) + || (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime)) { //invalid request res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)"); return; diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 361e02f..f2e1539 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -13,7 +13,8 @@ import {config} from '../config'; import { UserID } from '../types/user.model'; import redis from '../utils/redis'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; -import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; +import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; +import { getCategoryActionType } from '../utils/categoryInfo'; const voteTypes = { normal: 0, @@ -170,7 +171,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i res.status(400).send("Category doesn't exist."); return; } - if (category === "highlight") { + if (getCategoryActionType(category) !== CategoryActionType.Skippable) { res.status(400).send("Cannot vote for this category"); return; } diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 329c41a..c1606a9 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -72,4 +72,9 @@ export interface VideoData { export interface SegmentCache { shadowHiddenSegmentIPs: SBRecord, userHashedIP?: HashedIP +} + +export enum CategoryActionType { + Skippable, + POI } \ No newline at end of file diff --git a/src/utils/categoryInfo.ts b/src/utils/categoryInfo.ts new file mode 100644 index 0000000..cd8b745 --- /dev/null +++ b/src/utils/categoryInfo.ts @@ -0,0 +1,10 @@ +import { Category, CategoryActionType } from "../types/segments.model"; + +export function getCategoryActionType(category: Category): CategoryActionType { + switch (category) { + case "highlight": + return CategoryActionType.POI; + default: + return CategoryActionType.Skippable; + } +} diff --git a/test/cases/getSkipSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts index 23d9d49..c6c3414 100644 --- a/test/cases/getSkipSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -19,6 +19,9 @@ describe('getSegmentsByHash', () => { await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b await db.prepare("run", startOfQuery + "('onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, '" + getHash('onlyHidden', 1) + "')"); // hash = f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3 + await db.prepare("run", startOfQuery + "('highlightVid', 60, 60, 2, 'highlightVid-1', 'testman', 0, 50, 'highlight', 'YouTube', 0, 0, '" + getHash('highlightVid', 1) + "')"); // hash = c962d387a9e50170c9118405d20b1081cee8659cd600b856b511f695b91455cb + await db.prepare("run", startOfQuery + "('highlightVid', 70, 70, 2, 'highlightVid-2', 'testman', 0, 50, 'highlight', 'YouTube', 0, 0, '" + getHash('highlightVid', 1) + "')"); // hash = c962d387a9e50170c9118405d20b1081cee8659cd600b856b511f695b91455cb + }); it('Should be able to get a 200', (done: Done) => { @@ -158,7 +161,7 @@ describe('getSegmentsByHash', () => { if (res.status !== 200) done("non 200 status code, was " + res.status); else { const body = await res.json(); - if (body.length !== 1) done("expected 2 videos, got " + body.length); + if (body.length !== 1) done("expected 1 video, got " + body.length); else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor"); else done(); @@ -167,6 +170,20 @@ describe('getSegmentsByHash', () => { .catch(err => done("Couldn't call endpoint")); }); + it('Should only return one segment when fetching highlight segments', (done: Done) => { + fetch(getbaseURL() + '/api/skipSegments/c962?category=highlight') + .then(async res => { + if (res.status !== 200) done("non 200 status code, was " + res.status); + else { + const body = await res.json(); + if (body.length !== 1) done("expected 1 video, got " + body.length); + else if (body[0].segments.length !== 1) done("expected 1 segment, got " + body[0].segments.length); + else done(); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + it('Should be able to post a segment and get it using endpoint', (done: Done) => { let testID = 'abc123goodVideo'; fetch(getbaseURL() + "/api/postVideoSponsorTimes", { diff --git a/test/utils.ts b/test/utils.ts index c76065c..2ff13ae 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -7,4 +7,4 @@ export function getbaseURL() { /** * Duplicated from Mocha types. TypeScript doesn't infer that type by itself for some reason. */ -export type Done = (err?: any) => void; +export type Done = (err?: any) => void; \ No newline at end of file From 6a9b218e2212c24f96367f0ae55279deb766c687 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 8 Apr 2021 20:37:19 -0400 Subject: [PATCH 4/6] Don't always use YouTube API cache --- src/routes/postSkipSegments.ts | 24 ++--- src/routes/voteOnSponsorTime.ts | 163 ++++++++++++++++---------------- src/utils/webhookUtils.ts | 8 +- src/utils/youtubeApi.ts | 66 ++++++------- test.json | 7 -- test/youtubeMock.ts | 92 ++++++++++-------- 6 files changed, 183 insertions(+), 177 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 0923182..ef58c45 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -19,7 +19,7 @@ import { getCategoryActionType } from '../utils/categoryInfo'; interface APIVideoInfo { err: string | boolean, - data: any + data?: any } async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { @@ -276,11 +276,9 @@ function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration { return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null; } -async function getYouTubeVideoInfo(videoID: VideoID): Promise { +async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { if (config.youtubeAPIKey !== null) { - return new Promise((resolve) => { - YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data})); - }); + return YouTubeAPI.listVideos(videoID, ignoreCache); } else { return null; } @@ -368,9 +366,16 @@ export async function postSkipSegments(req: Request, res: Response) { const decreaseVotes = 0; + const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 + AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as + {videoDuration: VideoDuration, UUID: SegmentUUID}[]; + // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden + const videoDurationChanged = (videoDuration: number) => previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); + let apiVideoInfo: APIVideoInfo = null; if (service == Service.YouTube) { - apiVideoInfo = await getYouTubeVideoInfo(videoID); + // Don't use cache if we don't know the video duraton, or the client claims that it has changed + apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDuration || videoDurationChanged(videoDuration)); } const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo); if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) { @@ -378,12 +383,7 @@ export async function postSkipSegments(req: Request, res: Response) { videoDuration = apiVideoDuration || 0 as VideoDuration; } - const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 - AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as - {videoDuration: VideoDuration, UUID: SegmentUUID}[]; - // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden - const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); - if (videoDurationChanged) { + if (videoDurationChanged(videoDuration)) { // Hide all previous submissions for (const submission of previousSubmissions) { await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]); diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index f2e1539..a88c13d 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -60,90 +60,89 @@ async function sendWebhooks(voteData: VoteData) { } if (config.youtubeAPIKey !== null) { - YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => { - if (err || data.items.length === 0) { - err && Logger.error(err.toString()); - return; - } - const isUpvote = voteData.incrementAmount > 0; - // Send custom webhooks - dispatchEvent(isUpvote ? "vote.up" : "vote.down", { - "user": { - "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), - }, - "video": { - "id": submissionInfoRow.videoID, - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, - "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - }, - "submission": { - "UUID": voteData.UUID, - "views": voteData.row.views, - "category": voteData.category, - "startTime": submissionInfoRow.startTime, - "endTime": submissionInfoRow.endTime, - "user": { - "UUID": submissionInfoRow.userID, - "username": submissionInfoRow.userName, - "submissions": { - "total": submissionInfoRow.count, - "ignored": submissionInfoRow.disregarded, - }, - }, - }, - "votes": { - "before": voteData.row.votes, - "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), - }, - }); - - // Send discord message - if (webhookURL !== null && !isUpvote) { - fetch(webhookURL, { - method: 'POST', - body: JSON.stringify({ - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID - + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), - "description": "**" + voteData.row.votes + " Votes Prior | " + - (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views - + " Views**\n\n**Submission ID:** " + voteData.UUID - + "\n**Category:** " + submissionInfoRow.category - + "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID - + "\n\n**Total User Submissions:** " + submissionInfoRow.count - + "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded - + "\n\n**Timestamp:** " + - getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), - "color": 10813440, - "author": { - "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - }, - }], - }), - headers: { - 'Content-Type': 'application/json' - } - }) - .then(async res => { - if (res.status >= 400) { - Logger.error("Error sending reported submission Discord hook"); - Logger.error(JSON.stringify((await res.text()))); - Logger.error("\n"); - } - }) - .catch(err => { - Logger.error("Failed to send reported submission Discord hook."); - Logger.error(JSON.stringify(err)); - Logger.error("\n"); - }); - } + const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID); + if (err || data.items.length === 0) { + if (err) Logger.error(err.toString()); + return; + } + const isUpvote = voteData.incrementAmount > 0; + // Send custom webhooks + dispatchEvent(isUpvote ? "vote.up" : "vote.down", { + // "user": { + // "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + // }, + // "video": { + // "id": submissionInfoRow.videoID, + // "title": data.items[0].snippet.title, + // "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, + // "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + // }, + // "submission": { + // "UUID": voteData.UUID, + // "views": voteData.row.views, + // "category": voteData.category, + // "startTime": submissionInfoRow.startTime, + // "endTime": submissionInfoRow.endTime, + // "user": { + // "UUID": submissionInfoRow.userID, + // "username": submissionInfoRow.userName, + // "submissions": { + // "total": submissionInfoRow.count, + // "ignored": submissionInfoRow.disregarded, + // }, + // }, + // }, + // "votes": { + // "before": voteData.row.votes, + // "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), + // }, }); + + // Send discord message + if (webhookURL !== null && !isUpvote) { + fetch(webhookURL, { + method: 'POST', + body: JSON.stringify({ + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), + "description": "**" + voteData.row.votes + " Votes Prior | " + + (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views + + " Views**\n\n**Submission ID:** " + voteData.UUID + + "\n**Category:** " + submissionInfoRow.category + + "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID + + "\n\n**Total User Submissions:** " + submissionInfoRow.count + + "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded + + "\n\n**Timestamp:** " + + getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), + "color": 10813440, + "author": { + "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + }, + }], + }), + headers: { + 'Content-Type': 'application/json' + } + }) + .then(async res => { + if (res.status >= 400) { + Logger.error("Error sending reported submission Discord hook"); + Logger.error(JSON.stringify((await res.text()))); + Logger.error("\n"); + } + }) + .catch(err => { + Logger.error("Failed to send reported submission Discord hook."); + Logger.error(JSON.stringify(err)); + Logger.error("\n"); + }); + } } } } diff --git a/src/utils/webhookUtils.ts b/src/utils/webhookUtils.ts index 91ed197..591785f 100644 --- a/src/utils/webhookUtils.ts +++ b/src/utils/webhookUtils.ts @@ -1,6 +1,7 @@ import {config} from '../config'; import {Logger} from '../utils/logger'; import fetch from 'node-fetch'; +import AbortController from "abort-controller"; function getVoteAuthorRaw(submissionCount: number, isVIP: boolean, isOwnSubmission: boolean): string { if (isOwnSubmission) { @@ -30,7 +31,8 @@ function dispatchEvent(scope: string, data: any): void { let webhooks = config.webhooks; if (webhooks === undefined || webhooks.length === 0) return; Logger.debug("Dispatching webhooks"); - webhooks.forEach(webhook => { + + for (const webhook of webhooks) { let webhookURL = webhook.url; let authKey = webhook.key; let scopes = webhook.scopes || []; @@ -43,13 +45,13 @@ function dispatchEvent(scope: string, data: any): void { "Authorization": authKey, "Event-Type": scope, // Maybe change this in the future? 'Content-Type': 'application/json' - }, + } }) .catch(err => { Logger.warn('Couldn\'t send webhook to ' + webhook.url); Logger.warn(err); }); - }); + } } export { diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 1672281..6515996 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -10,43 +10,45 @@ _youTubeAPI.authenticate({ }); export class YouTubeAPI { - static listVideos(videoID: string, callback: (err: string | boolean, data: any) => void) { + static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> { const part = 'contentDetails,snippet'; if (!videoID || videoID.length !== 11 || videoID.includes(".")) { - callback("Invalid video ID", undefined); - return; + return { err: "Invalid video ID" }; } const redisKey = "youtube.video." + videoID; - redis.get(redisKey, (getErr, result) => { - if (getErr || !result) { + if (!ignoreCache) { + const {err, reply} = await redis.getAsync(redisKey); + + if (!err && reply) { Logger.debug("redis: no cache for video information: " + videoID); - _youTubeAPI.videos.list({ - part, - id: videoID, - }, (ytErr: boolean | string, { data }: any) => { - if (!ytErr) { - // Only set cache if data returned - if (data.items.length > 0) { - redis.set(redisKey, JSON.stringify(data), (setErr) => { - if (setErr) { - Logger.warn(setErr.message); - } else { - Logger.debug("redis: video information cache set for: " + videoID); - } - callback(false, data); // don't fail - }); - } else { - callback(false, data); // don't fail - } - } else { - callback(ytErr, data); - } - }); - } else { - Logger.debug("redis: fetched video information from cache: " + videoID); - callback(getErr?.message, JSON.parse(result)); + + return { err: err?.message, data: JSON.parse(reply) } } - }); - }; + } + + const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({ + part, + id: videoID, + }, (ytErr: boolean | string, { data }: any) => resolve({ytErr, data}))); + + if (!ytErr) { + // Only set cache if data returned + if (data.items.length > 0) { + const { err: setErr } = await redis.setAsync(redisKey, JSON.stringify(data)); + + if (setErr) { + Logger.warn(setErr.message); + } else { + Logger.debug("redis: video information cache set for: " + videoID); + } + + return { err: false, data }; // don't fail + } else { + return { err: false, data }; // don't fail + } + } else { + return { err: ytErr, data }; + } + } } diff --git a/test.json b/test.json index 3e147fe..3dbb983 100644 --- a/test.json +++ b/test.json @@ -40,13 +40,6 @@ "vote.up", "vote.down" ] - }, { - "url": "http://unresolvable.host:8081/FailedWebhook", - "key": "superSecretKey", - "scopes": [ - "vote.up", - "vote.down" - ] } ], "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"], diff --git a/test/youtubeMock.ts b/test/youtubeMock.ts index 64249e0..de7e17e 100644 --- a/test/youtubeMock.ts +++ b/test/youtubeMock.ts @@ -9,61 +9,71 @@ YouTubeAPI.videos.list({ export class YouTubeApiMock { - static listVideos(videoID: string, callback: (ytErr: any, data: any) => void) { + static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> { const obj = { id: videoID }; if (obj.id === "knownWrongID") { - callback(undefined, { - pageInfo: { - totalResults: 0, - }, - items: [], - }); + return { + err: null, + data: { + pageInfo: { + totalResults: 0, + }, + items: [], + } + }; } + if (obj.id === "noDuration") { - callback(undefined, { - pageInfo: { - totalResults: 1, - }, - items: [ - { - contentDetails: { - duration: "PT0S", - }, - snippet: { - title: "Example Title", - thumbnails: { - maxres: { - url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + return { + err: null, + data: { + pageInfo: { + totalResults: 1, + }, + items: [ + { + contentDetails: { + duration: "PT0S", + }, + snippet: { + title: "Example Title", + thumbnails: { + maxres: { + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + }, }, }, }, - }, - ], - }); + ], + } + }; } else { - callback(undefined, { - pageInfo: { - totalResults: 1, - }, - items: [ - { - contentDetails: { - duration: "PT1H23M30S", - }, - snippet: { - title: "Example Title", - thumbnails: { - maxres: { - url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + return { + err: null, + data: { + pageInfo: { + totalResults: 1, + }, + items: [ + { + contentDetails: { + duration: "PT1H23M30S", + }, + snippet: { + title: "Example Title", + thumbnails: { + maxres: { + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + }, }, }, }, - }, - ], - }); + ], + } + }; } } } From f6f5570d0cbb8f29b4aa34c586570a2a9634dbbc Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Apr 2021 13:15:42 -0400 Subject: [PATCH 5/6] Improve auto mod reject message --- src/routes/postSkipSegments.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index ef58c45..262fde0 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -412,10 +412,10 @@ export async function postSkipSegments(req: Request, res: Response) { // TODO: Do something about the fradulent submission Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'"); res.status(403).send( - "Request rejected by auto moderator: New submissions are not allowed for the following category: '" + "New submissions are not allowed for the following category: '" + segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n " + (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + - "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "") + "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n" : "") + "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app", ); return; From 5a60dfa988e0452ae1fa214a841a6c8c645f61d5 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 11:19:40 -0400 Subject: [PATCH 6/6] Uncomment webhook --- src/routes/voteOnSponsorTime.ts | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 6b15849..552e8ca 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -69,34 +69,34 @@ async function sendWebhooks(voteData: VoteData) { const isUpvote = voteData.incrementAmount > 0; // Send custom webhooks dispatchEvent(isUpvote ? "vote.up" : "vote.down", { - // "user": { - // "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), - // }, - // "video": { - // "id": submissionInfoRow.videoID, - // "title": data.items[0].snippet.title, - // "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, - // "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - // }, - // "submission": { - // "UUID": voteData.UUID, - // "views": voteData.row.views, - // "category": voteData.category, - // "startTime": submissionInfoRow.startTime, - // "endTime": submissionInfoRow.endTime, - // "user": { - // "UUID": submissionInfoRow.userID, - // "username": submissionInfoRow.userName, - // "submissions": { - // "total": submissionInfoRow.count, - // "ignored": submissionInfoRow.disregarded, - // }, - // }, - // }, - // "votes": { - // "before": voteData.row.votes, - // "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), - // }, + "user": { + "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + }, + "video": { + "id": submissionInfoRow.videoID, + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, + "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + }, + "submission": { + "UUID": voteData.UUID, + "views": voteData.row.views, + "category": voteData.category, + "startTime": submissionInfoRow.startTime, + "endTime": submissionInfoRow.endTime, + "user": { + "UUID": submissionInfoRow.userID, + "username": submissionInfoRow.userName, + "submissions": { + "total": submissionInfoRow.count, + "ignored": submissionInfoRow.disregarded, + }, + }, + }, + "votes": { + "before": voteData.row.votes, + "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), + }, }); // Send discord message