From 6554e142cc639b2f4d69ab4aa79c07e306470d4d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 4 Apr 2021 23:12:26 -0400 Subject: [PATCH 01/85] Add highlight category --- config.json.example | 2 +- src/config.ts | 2 +- src/routes/postSkipSegments.ts | 5 ++-- src/types/segments.model.ts | 2 +- test.json | 2 +- test/cases/postSkipSegments.ts | 45 ++++++++++++++++++++++++++++++++++ 6 files changed, 52 insertions(+), 6 deletions(-) diff --git a/config.json.example b/config.json.example index f8548c1..3d9b2ef 100644 --- a/config.json.example +++ b/config.json.example @@ -22,7 +22,7 @@ "mode": "development", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "preview", "music_offtopic", "highlight"], // List of supported categories any other category will be rejected "getTopUsersCacheTimeMinutes": 5, // cacheTime for getTopUsers result in minutes "maxNumberOfActiveWarnings": 3, // Users with this number of warnings will be blocked until warnings expire "hoursAfterWarningExpire": 24, diff --git a/src/config.ts b/src/config.ts index 7b15987..66960ab 100644 --- a/src/config.ts +++ b/src/config.ts @@ -16,7 +16,7 @@ addDefaults(config, { privateDBSchema: "./databases/_private.db.sql", readOnly: false, webhooks: [], - categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"], + categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"], maxNumberOfActiveWarnings: 3, hoursAfterWarningExpires: 24, adminUserID: "", diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 1f380da..2e0ba6c 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -4,7 +4,7 @@ import {db, privateDB} from '../databases/databases'; import {YouTubeAPI} from '../utils/youtubeApi'; import {getSubmissionUUID} from '../utils/getSubmissionUUID'; import fetch from 'node-fetch'; -import isoDurations from 'iso8601-duration'; +import isoDurations, { end } from 'iso8601-duration'; import {getHash} from '../utils/getHash'; import {getIP} from '../utils/getIP'; import {getFormattedTime} from '../utils/getFormattedTime'; @@ -425,7 +425,8 @@ export async function postSkipSegments(req: Request, res: Response) { let endTime = parseFloat(segments[i].segment[1]); if (isNaN(startTime) || isNaN(endTime) - || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { + || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime + || (segments[i].category !== "highlight" && startTime === endTime) || (segments[i].category === "highlight" && startTime !== endTime)) { //invalid request res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)"); return; diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index a95bac3..329c41a 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -4,7 +4,7 @@ import { SBRecord } from "./lib.model"; export type SegmentUUID = string & { __segmentUUIDBrand: unknown }; export type VideoID = string & { __videoIDBrand: unknown }; export type VideoDuration = number & { __videoDurationBrand: unknown }; -export type Category = string & { __categoryBrand: unknown }; +export type Category = ("sponsor" | "selfpromo" | "interaction" | "intro" | "outro" | "preview" | "music_offtopic" | "highlight") & { __categoryBrand: unknown }; export type VideoIDHash = VideoID & HashedValue; export type IPAddress = string & { __ipAddressBrand: unknown }; export type HashedIP = IPAddress & HashedValue; diff --git a/test.json b/test.json index 58eb72a..3e147fe 100644 --- a/test.json +++ b/test.json @@ -49,7 +49,7 @@ ] } ], - "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"], + "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"], "maxNumberOfActiveWarnings": 3, "hoursAfterWarningExpires": 24, "rateLimit": { diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 512c2ee..e084112 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -500,6 +500,51 @@ describe('postSkipSegments', () => { .catch(err => done("Couldn't call endpoint")); }); + it('Should be rejected if segment starts and ends at the same time', (done: Done) => { + fetch(getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=90&endTime=90&userID=testing&category=intro", { + method: 'POST', + }) + .then(async res => { + if (res.status === 400) done(); // pass + else { + const body = await res.text(); + done("non 400 status code: " + res.status + " (" + body + ")"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be accepted if highlight segment starts and ends at the same time', (done: Done) => { + fetch(getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30&userID=testing&category=highlight", { + method: 'POST', + }) + .then(async res => { + if (res.status === 200) done(); // pass + else { + const body = await res.text(); + done("non 200 status code: " + res.status + " (" + body + ")"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be rejected if highlight segment doesn\'t start and end at the same time', (done: Done) => { + fetch(getbaseURL() + + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing&category=highlight", { + method: 'POST', + }) + .then(async res => { + if (res.status === 400) done(); // pass + else { + const body = await res.text(); + done("non 400 status code: " + res.status + " (" + body + ")"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + it('Should be rejected if a sponsor is less than 1 second', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=qqwerty&startTime=30&endTime=30.5&userID=testing", { From 7bf09906d34c2cba89cd11c0295fff13255a653d Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 5 Apr 2021 22:33:28 -0400 Subject: [PATCH 02/85] Prevent voting for highlight category --- src/routes/voteOnSponsorTime.ts | 4 ++++ test/cases/voteOnSponsorTime.ts | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index c0aaac0..361e02f 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -170,6 +170,10 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i res.status(400).send("Category doesn't exist."); return; } + if (category === "highlight") { + res.status(400).send("Cannot vote for this category"); + return; + } const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]); diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts index 47a1f19..8e1cb61 100644 --- a/test/cases/voteOnSponsorTime.ts +++ b/test/cases/voteOnSponsorTime.ts @@ -264,6 +264,24 @@ describe('voteOnSponsorTime', () => { .catch(err => done(err)); }); + it('Should not able to change to highlight category', (done: Done) => { + fetch(getbaseURL() + + "/api/voteOnSponsorTime?userID=randomID2&UUID=incorrect-category&category=highlight") + .then(async res => { + if (res.status === 400) { + let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["incorrect-category"]); + if (row.category === "sponsor") { + done(); + } else { + done("Vote did not succeed. Submission went from sponsor to " + row.category); + } + } else { + done("Status code was " + res.status); + } + }) + .catch(err => done(err)); + }); + it('Should be able to change your vote for a category and it should add your vote to the database', (done: Done) => { fetch(getbaseURL() + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-4&category=outro") From 8088f3763202a813dfe76ec0a4de3126db24f48f Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 5 Apr 2021 23:48:51 -0400 Subject: [PATCH 03/85] Only return one segment for highlight category --- src/routes/getSkipSegments.ts | 12 +++++++----- src/routes/postSkipSegments.ts | 6 ++++-- src/routes/voteOnSponsorTime.ts | 5 +++-- src/types/segments.model.ts | 5 +++++ src/utils/categoryInfo.ts | 10 ++++++++++ test/cases/getSkipSegmentsByHash.ts | 19 ++++++++++++++++++- test/utils.ts | 2 +- 7 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 src/utils/categoryInfo.ts diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index d36a91a..530edd9 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -4,7 +4,8 @@ import { config } from '../config'; import { db, privateDB } from '../databases/databases'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { SBRecord } from '../types/lib.model'; -import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; +import { Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; +import { getCategoryActionType } from '../utils/categoryInfo'; import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; @@ -40,7 +41,8 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: const filteredSegments = segments.filter((_, index) => shouldFilter[index]); - return chooseSegments(filteredSegments).map((chosenSegment) => ({ + const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1 + return chooseSegments(filteredSegments, maxSegments).map((chosenSegment) => ({ category, segment: [chosenSegment.startTime, chosenSegment.endTime], UUID: chosenSegment.UUID, @@ -206,7 +208,7 @@ function getWeightedRandomChoice(choices: T[], amountOf //Only one similar time will be returned, randomly generated based on the sqrt of votes. //This allows new less voted items to still sometimes appear to give them a chance at getting votes. //Segments with less than -1 votes are already ignored before this function is called -function chooseSegments(segments: DBSegment[]): DBSegment[] { +function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { //Create groups of segments that are similar to eachother //Segments must be sorted by their startTime so that we can build groups chronologically: //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group @@ -240,8 +242,8 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] { } }); - //if there are too many groups, find the best 8 - return getWeightedRandomChoice(overlappingSegmentsGroups, 32).map( + //if there are too many groups, find the best ones + return getWeightedRandomChoice(overlappingSegmentsGroups, max).map( //randomly choose 1 good segment per group and return them group => getWeightedRandomChoice(group.segments, 1)[0], ); diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 2e0ba6c..0923182 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -13,8 +13,9 @@ import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import redis from '../utils/redis'; -import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; +import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; import { deleteNoSegments } from './deleteNoSegments'; +import { getCategoryActionType } from '../utils/categoryInfo'; interface APIVideoInfo { err: string | boolean, @@ -426,7 +427,8 @@ export async function postSkipSegments(req: Request, res: Response) { if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime - || (segments[i].category !== "highlight" && startTime === endTime) || (segments[i].category === "highlight" && startTime !== endTime)) { + || (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable && startTime === endTime) + || (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime)) { //invalid request res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)"); return; diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 361e02f..f2e1539 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -13,7 +13,8 @@ import {config} from '../config'; import { UserID } from '../types/user.model'; import redis from '../utils/redis'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; -import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; +import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; +import { getCategoryActionType } from '../utils/categoryInfo'; const voteTypes = { normal: 0, @@ -170,7 +171,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i res.status(400).send("Category doesn't exist."); return; } - if (category === "highlight") { + if (getCategoryActionType(category) !== CategoryActionType.Skippable) { res.status(400).send("Cannot vote for this category"); return; } diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 329c41a..c1606a9 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -72,4 +72,9 @@ export interface VideoData { export interface SegmentCache { shadowHiddenSegmentIPs: SBRecord, userHashedIP?: HashedIP +} + +export enum CategoryActionType { + Skippable, + POI } \ No newline at end of file diff --git a/src/utils/categoryInfo.ts b/src/utils/categoryInfo.ts new file mode 100644 index 0000000..cd8b745 --- /dev/null +++ b/src/utils/categoryInfo.ts @@ -0,0 +1,10 @@ +import { Category, CategoryActionType } from "../types/segments.model"; + +export function getCategoryActionType(category: Category): CategoryActionType { + switch (category) { + case "highlight": + return CategoryActionType.POI; + default: + return CategoryActionType.Skippable; + } +} diff --git a/test/cases/getSkipSegmentsByHash.ts b/test/cases/getSkipSegmentsByHash.ts index 23d9d49..c6c3414 100644 --- a/test/cases/getSkipSegmentsByHash.ts +++ b/test/cases/getSkipSegmentsByHash.ts @@ -19,6 +19,9 @@ describe('getSegmentsByHash', () => { await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910 await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b await db.prepare("run", startOfQuery + "('onlyHidden', 60, 70, 2, 'onlyHidden', 'testman', 0, 50, 'sponsor', 'YouTube', 1, 0, '" + getHash('onlyHidden', 1) + "')"); // hash = f3a199e1af001d716cdc6599360e2b062c2d2b3fa2885f6d9d2fd741166cbbd3 + await db.prepare("run", startOfQuery + "('highlightVid', 60, 60, 2, 'highlightVid-1', 'testman', 0, 50, 'highlight', 'YouTube', 0, 0, '" + getHash('highlightVid', 1) + "')"); // hash = c962d387a9e50170c9118405d20b1081cee8659cd600b856b511f695b91455cb + await db.prepare("run", startOfQuery + "('highlightVid', 70, 70, 2, 'highlightVid-2', 'testman', 0, 50, 'highlight', 'YouTube', 0, 0, '" + getHash('highlightVid', 1) + "')"); // hash = c962d387a9e50170c9118405d20b1081cee8659cd600b856b511f695b91455cb + }); it('Should be able to get a 200', (done: Done) => { @@ -158,7 +161,7 @@ describe('getSegmentsByHash', () => { if (res.status !== 200) done("non 200 status code, was " + res.status); else { const body = await res.json(); - if (body.length !== 1) done("expected 2 videos, got " + body.length); + if (body.length !== 1) done("expected 1 video, got " + body.length); else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length); else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor"); else done(); @@ -167,6 +170,20 @@ describe('getSegmentsByHash', () => { .catch(err => done("Couldn't call endpoint")); }); + it('Should only return one segment when fetching highlight segments', (done: Done) => { + fetch(getbaseURL() + '/api/skipSegments/c962?category=highlight') + .then(async res => { + if (res.status !== 200) done("non 200 status code, was " + res.status); + else { + const body = await res.json(); + if (body.length !== 1) done("expected 1 video, got " + body.length); + else if (body[0].segments.length !== 1) done("expected 1 segment, got " + body[0].segments.length); + else done(); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + it('Should be able to post a segment and get it using endpoint', (done: Done) => { let testID = 'abc123goodVideo'; fetch(getbaseURL() + "/api/postVideoSponsorTimes", { diff --git a/test/utils.ts b/test/utils.ts index c76065c..2ff13ae 100644 --- a/test/utils.ts +++ b/test/utils.ts @@ -7,4 +7,4 @@ export function getbaseURL() { /** * Duplicated from Mocha types. TypeScript doesn't infer that type by itself for some reason. */ -export type Done = (err?: any) => void; +export type Done = (err?: any) => void; \ No newline at end of file From 6a9b218e2212c24f96367f0ae55279deb766c687 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 8 Apr 2021 20:37:19 -0400 Subject: [PATCH 04/85] Don't always use YouTube API cache --- src/routes/postSkipSegments.ts | 24 ++--- src/routes/voteOnSponsorTime.ts | 163 ++++++++++++++++---------------- src/utils/webhookUtils.ts | 8 +- src/utils/youtubeApi.ts | 66 ++++++------- test.json | 7 -- test/youtubeMock.ts | 92 ++++++++++-------- 6 files changed, 183 insertions(+), 177 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 0923182..ef58c45 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -19,7 +19,7 @@ import { getCategoryActionType } from '../utils/categoryInfo'; interface APIVideoInfo { err: string | boolean, - data: any + data?: any } async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { @@ -276,11 +276,9 @@ function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration { return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null; } -async function getYouTubeVideoInfo(videoID: VideoID): Promise { +async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { if (config.youtubeAPIKey !== null) { - return new Promise((resolve) => { - YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data})); - }); + return YouTubeAPI.listVideos(videoID, ignoreCache); } else { return null; } @@ -368,9 +366,16 @@ export async function postSkipSegments(req: Request, res: Response) { const decreaseVotes = 0; + const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 + AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as + {videoDuration: VideoDuration, UUID: SegmentUUID}[]; + // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden + const videoDurationChanged = (videoDuration: number) => previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); + let apiVideoInfo: APIVideoInfo = null; if (service == Service.YouTube) { - apiVideoInfo = await getYouTubeVideoInfo(videoID); + // Don't use cache if we don't know the video duraton, or the client claims that it has changed + apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDuration || videoDurationChanged(videoDuration)); } const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo); if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) { @@ -378,12 +383,7 @@ export async function postSkipSegments(req: Request, res: Response) { videoDuration = apiVideoDuration || 0 as VideoDuration; } - const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 - AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as - {videoDuration: VideoDuration, UUID: SegmentUUID}[]; - // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden - const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); - if (videoDurationChanged) { + if (videoDurationChanged(videoDuration)) { // Hide all previous submissions for (const submission of previousSubmissions) { await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]); diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index f2e1539..a88c13d 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -60,90 +60,89 @@ async function sendWebhooks(voteData: VoteData) { } if (config.youtubeAPIKey !== null) { - YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => { - if (err || data.items.length === 0) { - err && Logger.error(err.toString()); - return; - } - const isUpvote = voteData.incrementAmount > 0; - // Send custom webhooks - dispatchEvent(isUpvote ? "vote.up" : "vote.down", { - "user": { - "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), - }, - "video": { - "id": submissionInfoRow.videoID, - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, - "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - }, - "submission": { - "UUID": voteData.UUID, - "views": voteData.row.views, - "category": voteData.category, - "startTime": submissionInfoRow.startTime, - "endTime": submissionInfoRow.endTime, - "user": { - "UUID": submissionInfoRow.userID, - "username": submissionInfoRow.userName, - "submissions": { - "total": submissionInfoRow.count, - "ignored": submissionInfoRow.disregarded, - }, - }, - }, - "votes": { - "before": voteData.row.votes, - "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), - }, - }); - - // Send discord message - if (webhookURL !== null && !isUpvote) { - fetch(webhookURL, { - method: 'POST', - body: JSON.stringify({ - "embeds": [{ - "title": data.items[0].snippet.title, - "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID - + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), - "description": "**" + voteData.row.votes + " Votes Prior | " + - (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views - + " Views**\n\n**Submission ID:** " + voteData.UUID - + "\n**Category:** " + submissionInfoRow.category - + "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID - + "\n\n**Total User Submissions:** " + submissionInfoRow.count - + "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded - + "\n\n**Timestamp:** " + - getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), - "color": 10813440, - "author": { - "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), - }, - "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - }, - }], - }), - headers: { - 'Content-Type': 'application/json' - } - }) - .then(async res => { - if (res.status >= 400) { - Logger.error("Error sending reported submission Discord hook"); - Logger.error(JSON.stringify((await res.text()))); - Logger.error("\n"); - } - }) - .catch(err => { - Logger.error("Failed to send reported submission Discord hook."); - Logger.error(JSON.stringify(err)); - Logger.error("\n"); - }); - } + const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID); + if (err || data.items.length === 0) { + if (err) Logger.error(err.toString()); + return; + } + const isUpvote = voteData.incrementAmount > 0; + // Send custom webhooks + dispatchEvent(isUpvote ? "vote.up" : "vote.down", { + // "user": { + // "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + // }, + // "video": { + // "id": submissionInfoRow.videoID, + // "title": data.items[0].snippet.title, + // "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, + // "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + // }, + // "submission": { + // "UUID": voteData.UUID, + // "views": voteData.row.views, + // "category": voteData.category, + // "startTime": submissionInfoRow.startTime, + // "endTime": submissionInfoRow.endTime, + // "user": { + // "UUID": submissionInfoRow.userID, + // "username": submissionInfoRow.userName, + // "submissions": { + // "total": submissionInfoRow.count, + // "ignored": submissionInfoRow.disregarded, + // }, + // }, + // }, + // "votes": { + // "before": voteData.row.votes, + // "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), + // }, }); + + // Send discord message + if (webhookURL !== null && !isUpvote) { + fetch(webhookURL, { + method: 'POST', + body: JSON.stringify({ + "embeds": [{ + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), + "description": "**" + voteData.row.votes + " Votes Prior | " + + (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views + + " Views**\n\n**Submission ID:** " + voteData.UUID + + "\n**Category:** " + submissionInfoRow.category + + "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID + + "\n\n**Total User Submissions:** " + submissionInfoRow.count + + "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded + + "\n\n**Timestamp:** " + + getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), + "color": 10813440, + "author": { + "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + }, + "thumbnail": { + "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + }, + }], + }), + headers: { + 'Content-Type': 'application/json' + } + }) + .then(async res => { + if (res.status >= 400) { + Logger.error("Error sending reported submission Discord hook"); + Logger.error(JSON.stringify((await res.text()))); + Logger.error("\n"); + } + }) + .catch(err => { + Logger.error("Failed to send reported submission Discord hook."); + Logger.error(JSON.stringify(err)); + Logger.error("\n"); + }); + } } } } diff --git a/src/utils/webhookUtils.ts b/src/utils/webhookUtils.ts index 91ed197..591785f 100644 --- a/src/utils/webhookUtils.ts +++ b/src/utils/webhookUtils.ts @@ -1,6 +1,7 @@ import {config} from '../config'; import {Logger} from '../utils/logger'; import fetch from 'node-fetch'; +import AbortController from "abort-controller"; function getVoteAuthorRaw(submissionCount: number, isVIP: boolean, isOwnSubmission: boolean): string { if (isOwnSubmission) { @@ -30,7 +31,8 @@ function dispatchEvent(scope: string, data: any): void { let webhooks = config.webhooks; if (webhooks === undefined || webhooks.length === 0) return; Logger.debug("Dispatching webhooks"); - webhooks.forEach(webhook => { + + for (const webhook of webhooks) { let webhookURL = webhook.url; let authKey = webhook.key; let scopes = webhook.scopes || []; @@ -43,13 +45,13 @@ function dispatchEvent(scope: string, data: any): void { "Authorization": authKey, "Event-Type": scope, // Maybe change this in the future? 'Content-Type': 'application/json' - }, + } }) .catch(err => { Logger.warn('Couldn\'t send webhook to ' + webhook.url); Logger.warn(err); }); - }); + } } export { diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 1672281..6515996 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -10,43 +10,45 @@ _youTubeAPI.authenticate({ }); export class YouTubeAPI { - static listVideos(videoID: string, callback: (err: string | boolean, data: any) => void) { + static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> { const part = 'contentDetails,snippet'; if (!videoID || videoID.length !== 11 || videoID.includes(".")) { - callback("Invalid video ID", undefined); - return; + return { err: "Invalid video ID" }; } const redisKey = "youtube.video." + videoID; - redis.get(redisKey, (getErr, result) => { - if (getErr || !result) { + if (!ignoreCache) { + const {err, reply} = await redis.getAsync(redisKey); + + if (!err && reply) { Logger.debug("redis: no cache for video information: " + videoID); - _youTubeAPI.videos.list({ - part, - id: videoID, - }, (ytErr: boolean | string, { data }: any) => { - if (!ytErr) { - // Only set cache if data returned - if (data.items.length > 0) { - redis.set(redisKey, JSON.stringify(data), (setErr) => { - if (setErr) { - Logger.warn(setErr.message); - } else { - Logger.debug("redis: video information cache set for: " + videoID); - } - callback(false, data); // don't fail - }); - } else { - callback(false, data); // don't fail - } - } else { - callback(ytErr, data); - } - }); - } else { - Logger.debug("redis: fetched video information from cache: " + videoID); - callback(getErr?.message, JSON.parse(result)); + + return { err: err?.message, data: JSON.parse(reply) } } - }); - }; + } + + const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({ + part, + id: videoID, + }, (ytErr: boolean | string, { data }: any) => resolve({ytErr, data}))); + + if (!ytErr) { + // Only set cache if data returned + if (data.items.length > 0) { + const { err: setErr } = await redis.setAsync(redisKey, JSON.stringify(data)); + + if (setErr) { + Logger.warn(setErr.message); + } else { + Logger.debug("redis: video information cache set for: " + videoID); + } + + return { err: false, data }; // don't fail + } else { + return { err: false, data }; // don't fail + } + } else { + return { err: ytErr, data }; + } + } } diff --git a/test.json b/test.json index 3e147fe..3dbb983 100644 --- a/test.json +++ b/test.json @@ -40,13 +40,6 @@ "vote.up", "vote.down" ] - }, { - "url": "http://unresolvable.host:8081/FailedWebhook", - "key": "superSecretKey", - "scopes": [ - "vote.up", - "vote.down" - ] } ], "categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"], diff --git a/test/youtubeMock.ts b/test/youtubeMock.ts index 64249e0..de7e17e 100644 --- a/test/youtubeMock.ts +++ b/test/youtubeMock.ts @@ -9,61 +9,71 @@ YouTubeAPI.videos.list({ export class YouTubeApiMock { - static listVideos(videoID: string, callback: (ytErr: any, data: any) => void) { + static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> { const obj = { id: videoID }; if (obj.id === "knownWrongID") { - callback(undefined, { - pageInfo: { - totalResults: 0, - }, - items: [], - }); + return { + err: null, + data: { + pageInfo: { + totalResults: 0, + }, + items: [], + } + }; } + if (obj.id === "noDuration") { - callback(undefined, { - pageInfo: { - totalResults: 1, - }, - items: [ - { - contentDetails: { - duration: "PT0S", - }, - snippet: { - title: "Example Title", - thumbnails: { - maxres: { - url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + return { + err: null, + data: { + pageInfo: { + totalResults: 1, + }, + items: [ + { + contentDetails: { + duration: "PT0S", + }, + snippet: { + title: "Example Title", + thumbnails: { + maxres: { + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + }, }, }, }, - }, - ], - }); + ], + } + }; } else { - callback(undefined, { - pageInfo: { - totalResults: 1, - }, - items: [ - { - contentDetails: { - duration: "PT1H23M30S", - }, - snippet: { - title: "Example Title", - thumbnails: { - maxres: { - url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + return { + err: null, + data: { + pageInfo: { + totalResults: 1, + }, + items: [ + { + contentDetails: { + duration: "PT1H23M30S", + }, + snippet: { + title: "Example Title", + thumbnails: { + maxres: { + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + }, }, }, }, - }, - ], - }); + ], + } + }; } } } From f6f5570d0cbb8f29b4aa34c586570a2a9634dbbc Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 11 Apr 2021 13:15:42 -0400 Subject: [PATCH 05/85] Improve auto mod reject message --- src/routes/postSkipSegments.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index ef58c45..262fde0 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -412,10 +412,10 @@ export async function postSkipSegments(req: Request, res: Response) { // TODO: Do something about the fradulent submission Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'"); res.status(403).send( - "Request rejected by auto moderator: New submissions are not allowed for the following category: '" + "New submissions are not allowed for the following category: '" + segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n " + (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + - "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "") + "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n" : "") + "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app", ); return; From a47906cac94286cfeb2280860f8dc3e88387ad2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Apr 2021 22:27:42 +0000 Subject: [PATCH 06/85] Bump redis from 3.0.2 to 3.1.1 Bumps [redis](https://github.com/NodeRedis/node-redis) from 3.0.2 to 3.1.1. - [Release notes](https://github.com/NodeRedis/node-redis/releases) - [Changelog](https://github.com/NodeRedis/node-redis/blob/master/CHANGELOG.md) - [Commits](https://github.com/NodeRedis/node-redis/compare/v3.0.2...v3.1.1) Signed-off-by: dependabot[bot] --- package-lock.json | 52 +++++++++++++++++++++++++---------------------- package.json | 2 +- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0877216..da09aff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "iso8601-duration": "^1.2.0", "node-fetch": "^2.6.0", "pg": "^8.5.1", - "redis": "^3.0.2", + "redis": "^3.1.1", "sync-mysql": "^3.0.1", "uuid": "^3.3.2", "youtube-api": "^3.0.1" @@ -992,9 +992,9 @@ "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, "node_modules/denque": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", - "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==", "engines": { "node": ">=0.10" } @@ -3100,23 +3100,27 @@ } }, "node_modules/redis": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", - "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.1.tgz", + "integrity": "sha512-QhkKhOuzhogR1NDJfBD34TQJz2ZJwDhhIC6ZmvpftlmfYShHHQXjjNspAJ+Z2HH5NwSBVYBVganbiZ8bgFMHjg==", "dependencies": { - "denque": "^1.4.1", - "redis-commands": "^1.5.0", + "denque": "^1.5.0", + "redis-commands": "^1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0" }, "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-redis" } }, "node_modules/redis-commands": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz", - "integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ==" + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" }, "node_modules/redis-errors": { "version": "1.2.0", @@ -5009,9 +5013,9 @@ "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" }, "denque": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", - "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" }, "depd": { "version": "1.1.2", @@ -6679,20 +6683,20 @@ } }, "redis": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", - "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.1.1.tgz", + "integrity": "sha512-QhkKhOuzhogR1NDJfBD34TQJz2ZJwDhhIC6ZmvpftlmfYShHHQXjjNspAJ+Z2HH5NwSBVYBVganbiZ8bgFMHjg==", "requires": { - "denque": "^1.4.1", - "redis-commands": "^1.5.0", + "denque": "^1.5.0", + "redis-commands": "^1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0" } }, "redis-commands": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz", - "integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ==" + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" }, "redis-errors": { "version": "1.2.0", diff --git a/package.json b/package.json index 8dbd4d8..b0d52d2 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "iso8601-duration": "^1.2.0", "node-fetch": "^2.6.0", "pg": "^8.5.1", - "redis": "^3.0.2", + "redis": "^3.1.1", "sync-mysql": "^3.0.1", "uuid": "^3.3.2", "youtube-api": "^3.0.1" From b0a4b6ebed576747d9669c5799092a318da6831c Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 6 May 2021 15:53:31 -0400 Subject: [PATCH 07/85] Update indexes file table name --- databases/_sponsorTimes_indexes.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index c90ef51..2ad3f75 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -51,10 +51,10 @@ CREATE INDEX IF NOT EXISTS "warnings_issueTime" ("issueTime" ASC NULLS LAST) TABLESPACE pg_default; --- noSegments +-- lockCategories CREATE INDEX IF NOT EXISTS "noSegments_videoID" - ON public."noSegments" USING btree + ON public."lockCategories" USING btree ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; From cd66399049711406b1bd9e24b4bb40874fef4ecd Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 6 May 2021 16:03:26 -0400 Subject: [PATCH 08/85] Add csp for API --- src/app.ts | 2 ++ src/middleware/apiCsp.ts | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 src/middleware/apiCsp.ts diff --git a/src/app.ts b/src/app.ts index 60a0b08..4530fa2 100644 --- a/src/app.ts +++ b/src/app.ts @@ -25,6 +25,7 @@ import {endpoint as getSkipSegments} from './routes/getSkipSegments'; import {userCounter} from './middleware/userCounter'; import {loggerMiddleware} from './middleware/logger'; import {corsMiddleware} from './middleware/cors'; +import {apiCspMiddleware} from './middleware/apiCsp'; import {rateLimitMiddleware} from './middleware/requestRateLimit'; import dumpDatabase, {redirectLink} from './routes/dumpDatabase'; @@ -36,6 +37,7 @@ export function createServer(callback: () => void) { //setup CORS correctly app.use(corsMiddleware); app.use(loggerMiddleware); + app.use("/api/", apiCspMiddleware); app.use(express.json()); if (config.userCounterURL) app.use(userCounter); diff --git a/src/middleware/apiCsp.ts b/src/middleware/apiCsp.ts new file mode 100644 index 0000000..deeb791 --- /dev/null +++ b/src/middleware/apiCsp.ts @@ -0,0 +1,6 @@ +import {NextFunction, Request, Response} from 'express'; + +export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) { + res.header("Content-Security-Policy", "script-src 'none'"); + next(); +} \ No newline at end of file From 60a118f3918b2bdadbb81083e3a07c5ee613861a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 6 May 2021 16:14:11 -0400 Subject: [PATCH 09/85] Add object src to csp --- src/middleware/apiCsp.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middleware/apiCsp.ts b/src/middleware/apiCsp.ts index deeb791..6d5f1ef 100644 --- a/src/middleware/apiCsp.ts +++ b/src/middleware/apiCsp.ts @@ -1,6 +1,6 @@ import {NextFunction, Request, Response} from 'express'; export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) { - res.header("Content-Security-Policy", "script-src 'none'"); + res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'"); next(); } \ No newline at end of file From 72aff3a69585d7582bb76bdb543d41a9f597ccd6 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 12 May 2021 18:57:36 -0400 Subject: [PATCH 10/85] Block username change --- src/routes/setUsername.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/routes/setUsername.ts b/src/routes/setUsername.ts index c44c38a..002ceb8 100644 --- a/src/routes/setUsername.ts +++ b/src/routes/setUsername.ts @@ -39,6 +39,12 @@ export async function setUsername(req: Request, res: Response) { //hash the userID userID = getHash(userID); } + + if (userID === "7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97") { + // Don't allow + res.sendStatus(200); + return; + } try { //check if username is already set From 0c64f4b0066e76b0a0dd4370acaae43ab0ba9e3c Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 13 May 2021 14:59:12 -0400 Subject: [PATCH 11/85] Check for array in non hash prefix method --- src/routes/getSkipSegments.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index d36a91a..b2a3811 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -266,6 +266,10 @@ async function handleGetSegments(req: Request, res: Response): Promise val == service)) { From e71399f5af5e9999dd989f533a949772e0f13bca Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 13 May 2021 21:24:53 -0400 Subject: [PATCH 12/85] Add banned user --- src/routes/setUsername.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/setUsername.ts b/src/routes/setUsername.ts index 002ceb8..0283f4d 100644 --- a/src/routes/setUsername.ts +++ b/src/routes/setUsername.ts @@ -40,7 +40,7 @@ export async function setUsername(req: Request, res: Response) { userID = getHash(userID); } - if (userID === "7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97") { + if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148"].includes(userID)) { // Don't allow res.sendStatus(200); return; From 5a60dfa988e0452ae1fa214a841a6c8c645f61d5 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 11:19:40 -0400 Subject: [PATCH 13/85] Uncomment webhook --- src/routes/voteOnSponsorTime.ts | 56 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 6b15849..552e8ca 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -69,34 +69,34 @@ async function sendWebhooks(voteData: VoteData) { const isUpvote = voteData.incrementAmount > 0; // Send custom webhooks dispatchEvent(isUpvote ? "vote.up" : "vote.down", { - // "user": { - // "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), - // }, - // "video": { - // "id": submissionInfoRow.videoID, - // "title": data.items[0].snippet.title, - // "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, - // "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", - // }, - // "submission": { - // "UUID": voteData.UUID, - // "views": voteData.row.views, - // "category": voteData.category, - // "startTime": submissionInfoRow.startTime, - // "endTime": submissionInfoRow.endTime, - // "user": { - // "UUID": submissionInfoRow.userID, - // "username": submissionInfoRow.userName, - // "submissions": { - // "total": submissionInfoRow.count, - // "ignored": submissionInfoRow.disregarded, - // }, - // }, - // }, - // "votes": { - // "before": voteData.row.votes, - // "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), - // }, + "user": { + "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + }, + "video": { + "id": submissionInfoRow.videoID, + "title": data.items[0].snippet.title, + "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, + "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + }, + "submission": { + "UUID": voteData.UUID, + "views": voteData.row.views, + "category": voteData.category, + "startTime": submissionInfoRow.startTime, + "endTime": submissionInfoRow.endTime, + "user": { + "UUID": submissionInfoRow.userID, + "username": submissionInfoRow.userName, + "submissions": { + "total": submissionInfoRow.count, + "ignored": submissionInfoRow.disregarded, + }, + }, + }, + "votes": { + "before": voteData.row.votes, + "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), + }, }); // Send discord message From c7b7732092e81a83b8d20988c612fc2329ee3d1a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 11:27:18 -0400 Subject: [PATCH 14/85] Update lodash --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index da09aff..1689804 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2121,9 +2121,9 @@ } }, "node_modules/lodash": { - "version": "4.17.19", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "node_modules/lodash.get": { @@ -5892,9 +5892,9 @@ } }, "lodash": { - "version": "4.17.19", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", - "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", "dev": true }, "lodash.get": { From 96ccbbe4a2182f50671978c8ba8347e612024ab8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 11:35:02 -0400 Subject: [PATCH 15/85] Removed unnecessary conditionals --- src/routes/postSkipSegments.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index c7225d8..f131e47 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -290,14 +290,10 @@ function proxySubmission(req: Request) { body: req.body, }) .then(async res => { - if (config.mode === 'development') { - Logger.debug('Proxy Submission: ' + res.status + ' (' + (await res.text()) + ')'); - } + Logger.debug('Proxy Submission: ' + res.status + ' (' + (await res.text()) + ')'); }) .catch(err => { - if (config.mode === 'development') { - Logger.error("Proxy Submission: Failed to make call"); - } + Logger.error("Proxy Submission: Failed to make call"); }); } From 9d19c59d44a88b80502a91520085ec291e4a5382 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 12:03:05 -0400 Subject: [PATCH 16/85] Add caching for raw videoID fetching --- src/middleware/redisKeys.ts | 4 +- src/routes/getSkipSegments.ts | 81 ++++++++++++++++----------------- src/routes/postSkipSegments.ts | 2 +- src/routes/voteOnSponsorTime.ts | 2 +- 4 files changed, 43 insertions(+), 46 deletions(-) diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts index 56b1aab..920b2da 100644 --- a/src/middleware/redisKeys.ts +++ b/src/middleware/redisKeys.ts @@ -1,8 +1,8 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { Logger } from "../utils/logger"; -export function skipSegmentsKey(videoID: VideoID): string { - return "segments-" + videoID; +export function skipSegmentsKey(videoID: VideoID, service: Service): string { + return "segments." + service + ".videoID." + videoID; } export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 9d59ad1..c3e7cbf 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -10,6 +10,7 @@ import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; import redis from '../utils/redis'; +import { getSkipSegmentsByHash } from './getSkipSegmentsByHash'; async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise { @@ -50,7 +51,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: })); } -async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise { +async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories: Category[], service: Service): Promise { const cache: SegmentCache = {shadowHiddenSegmentIPs: {}}; const segments: Segment[] = []; @@ -58,13 +59,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C categories = categories.filter((category) => !/[^a-z|_|-]/.test(category)); if (categories.length === 0) return null; - const segmentsByCategory: SBRecord = (await db - .prepare( - 'all', - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" - WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, - [videoID, service] - )).reduce((acc: SBRecord, segment: DBSegment) => { + const segmentsByCategory: SBRecord = (await getSegmentsFromDBByVideoID(videoID, service)) + .filter((segment: DBSegment) => categories.includes(segment?.category)) + .reduce((acc: SBRecord, segment: DBSegment) => { acc[segment.category] = acc[segment.category] || []; acc[segment.category].push(segment); @@ -72,7 +69,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C }, {}); for (const [category, categorySegments] of Object.entries(segmentsByCategory)) { - segments.push(...(await prepareCategorySegments(req, videoID as VideoID, category as Category, categorySegments, cache))); + segments.push(...(await prepareCategorySegments(req, videoID, category as Category, categorySegments, cache))); } return segments; @@ -94,7 +91,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category))); if (categories.length === 0) return null; - const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service)) + const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service)) .filter((segment: DBSegment) => categories.includes(segment?.category)) .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { acc[segment.videoID] = acc[segment.videoID] || { @@ -129,7 +126,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, } } -async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { +async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { const fetchFromDB = () => db .prepare( 'all', @@ -139,27 +136,42 @@ async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Serv ); if (hashedVideoIDPrefix.length === 4) { - const key = skipSegmentsHashKey(hashedVideoIDPrefix, service); - const {err, reply} = await redis.getAsync(key); - - if (!err && reply) { - try { - Logger.debug("Got data from redis: " + reply); - return JSON.parse(reply); - } catch (e) { - // If all else, continue on to fetching from the database - } - } - - const data = await fetchFromDB(); - - redis.setAsync(key, JSON.stringify(data)); - return data; + return await getSegmentsFromDB(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service)) } return await fetchFromDB(); } +async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise { + const fetchFromDB = () => db + .prepare( + 'all', + `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" + WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, + [videoID, service] + ); + + return await getSegmentsFromDB(fetchFromDB, skipSegmentsKey(videoID, service)) +} + +async function getSegmentsFromDB(fetchFromDB: () => Promise, key: string): Promise { + const {err, reply} = await redis.getAsync(key); + + if (!err && reply) { + try { + Logger.debug("Got data from redis: " + reply); + return JSON.parse(reply); + } catch (e) { + // If all else, continue on to fetching from the database + } + } + + const data = await fetchFromDB(); + + redis.setAsync(key, JSON.stringify(data)); + return data; +} + //gets a weighted random choice from the choices array based on their `votes` property. //amountOfChoices specifies the maximum amount of choices to return, 1 or more. //choices are unique @@ -278,18 +290,6 @@ async function handleGetSegments(req: Request, res: Response): Promise Date: Sun, 23 May 2021 14:56:04 -0400 Subject: [PATCH 17/85] Update mocha --- package-lock.json | 1759 ++++++++++++++++++++------------------------- package.json | 4 +- 2 files changed, 765 insertions(+), 998 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1689804..61b1181 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,13 +26,13 @@ "@types/better-sqlite3": "^5.4.0", "@types/express": "^4.17.8", "@types/express-rate-limit": "^5.1.0", - "@types/mocha": "^8.0.3", + "@types/mocha": "^8.2.2", "@types/node": "^14.11.9", "@types/node-fetch": "^2.5.7", "@types/pg": "^7.14.10", "@types/redis": "^2.8.28", "@types/request": "^2.48.5", - "mocha": "^7.1.1", + "mocha": "^8.4.0", "nodemon": "^2.0.2", "sinon": "^9.2.0", "ts-mock-imports": "^1.3.0", @@ -164,9 +164,9 @@ "dev": true }, "node_modules/@types/mocha": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-8.0.3.tgz", - "integrity": "sha512-vyxR57nv8NfcU0GZu8EUXZLTbCMupIUwy95LJ6lllN+JRPG25CwMHoB1q5xKh8YKhQnHYRAn4yW2yuHbf/5xgg==", + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-8.2.2.tgz", + "integrity": "sha512-Lwh0lzzqT5Pqh6z61P3c3P5nm6fzQK/MMHl9UKeneAeInVflBSz1O2EkX6gM6xfJd7FBXBY5purtLx7fUiZ7Hw==", "dev": true }, "node_modules/@types/node": { @@ -259,6 +259,12 @@ "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==", "dev": true }, + "node_modules/@ungap/promise-all-settled": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", + "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", + "dev": true + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -325,9 +331,9 @@ } }, "node_modules/ansi-colors": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", - "integrity": "sha512-LEHHyuhlPY3TmuUYMh2oz89lTShfvgbmzaBcxve9t/9Wuy7Dwf4yoAKcND7KFT1HAQfqZ12qtc+DUrBMeKF9nw==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", "dev": true, "engines": { "node": ">=6" @@ -341,6 +347,39 @@ "node": ">=0.10.0" } }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/ansi-styles/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/ansi-styles/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, "node_modules/anymatch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", @@ -407,13 +446,10 @@ "dev": true }, "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "dependencies": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "node_modules/array-flatten": { "version": "1.1.1", @@ -688,6 +724,18 @@ "node": ">= 0.8" } }, + "node_modules/camelcase": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz", + "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/capture-stack-trace": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", @@ -697,10 +745,26 @@ "node": ">=0.10.0" } }, + "node_modules/chalk": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", + "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/chokidar": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.3.0.tgz", - "integrity": "sha512-dGmKLDdT3Gdl7fBUe8XK+gAtGmzy5Fn0XkkWQuYxGIgWVPPse2CxFA5mtrlD0TOHaHjEUqkWNyP1XdHoJES/4A==", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", + "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", "dev": true, "dependencies": { "anymatch": "~3.1.1", @@ -709,13 +773,13 @@ "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", - "readdirp": "~3.2.0" + "readdirp": "~3.5.0" }, "engines": { "node": ">= 8.10.0" }, "optionalDependencies": { - "fsevents": "~2.1.1" + "fsevents": "~2.3.1" } }, "node_modules/chownr": { @@ -742,49 +806,58 @@ } }, "node_modules/cliui": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", - "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, "dependencies": { - "string-width": "^3.1.0", - "strip-ansi": "^5.2.0", - "wrap-ansi": "^5.1.0" + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" } }, "node_modules/cliui/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true, "engines": { - "node": ">=6" + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" } }, "node_modules/cliui/node_modules/string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", "dev": true, "dependencies": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/cliui/node_modules/strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "dependencies": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.0" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/code-point-at": { @@ -938,12 +1011,15 @@ } }, "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", "dev": true, "engines": { - "node": ">=0.10.0" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/decompress-response": { @@ -965,18 +1041,6 @@ "node": ">=4.0.0" } }, - "node_modules/define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, - "dependencies": { - "object-keys": "^1.0.12" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -1024,9 +1088,9 @@ } }, "node_modules/diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", "dev": true, "engines": { "node": ">=0.3.1" @@ -1072,9 +1136,9 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "node_modules/emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, "node_modules/encodeurl": { @@ -1093,40 +1157,13 @@ "once": "^1.4.0" } }, - "node_modules/es-abstract": { - "version": "1.17.5", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", - "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", "dev": true, - "dependencies": { - "es-to-primitive": "^1.2.1", - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.1", - "is-callable": "^1.1.5", - "is-regex": "^1.0.5", - "object-inspect": "^1.7.0", - "object-keys": "^1.1.1", - "object.assign": "^4.1.0", - "string.prototype.trimleft": "^2.1.1", - "string.prototype.trimright": "^2.1.1" - }, "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-to-primitive": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dev": true, - "dependencies": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" + "node": ">=6" } }, "node_modules/escape-html": { @@ -1143,19 +1180,6 @@ "node": ">=0.8.0" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -1293,25 +1317,26 @@ } }, "node_modules/find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, "dependencies": { - "locate-path": "^3.0.0" + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" }, "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/flat": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/flat/-/flat-4.1.0.tgz", - "integrity": "sha512-Px/TiLIznH7gEDlPXcUD4KnBusa6kR6ayRUVcnEAbreRIuhkqow/mun59BuRXwoYk7ZQOLW1ZM05ilIvK38hFw==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", "dev": true, - "dependencies": { - "is-buffer": "~2.0.3" - }, "bin": { "flat": "cli.js" } @@ -1369,10 +1394,11 @@ "dev": true }, "node_modules/fsevents": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", - "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "dev": true, + "hasInstallScript": true, "optional": true, "os": [ "darwin" @@ -1381,12 +1407,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true - }, "node_modules/gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", @@ -1493,9 +1513,9 @@ "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" }, "node_modules/glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "dev": true, "dependencies": { "fs.realpath": "^1.0.0", @@ -1507,6 +1527,9 @@ }, "engines": { "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/glob-parent": { @@ -1675,18 +1698,6 @@ "node": ">=4.0.0" } }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -1696,15 +1707,6 @@ "node": ">=4" } }, - "node_modules/has-symbols": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", - "dev": true, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", @@ -1861,24 +1863,6 @@ "node": ">=8" } }, - "node_modules/is-buffer": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", - "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/is-callable": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", - "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", - "dev": true, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/is-ci": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz", @@ -1891,15 +1875,6 @@ "is-ci": "bin.js" } }, - "node_modules/is-date-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", - "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", - "dev": true, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1981,6 +1956,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/is-redirect": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", @@ -1990,18 +1974,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-regex": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", - "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", - "dev": true, - "dependencies": { - "has": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/is-retry-allowed": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", @@ -2020,18 +1992,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-symbol": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", - "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", - "dev": true, - "dependencies": { - "has-symbols": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -2050,13 +2010,12 @@ "integrity": "sha512-ErTBd++b17E8nmWII1K1uZtBgD1E8RjyvwmxlCjPHNqHMD7gmcMHOw0E8Ro/6+QT4PhHRSnnMo7bxa1vFPkwhg==" }, "node_modules/js-yaml": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", - "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", "dev": true, "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -2108,24 +2067,20 @@ } }, "node_modules/locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, "dependencies": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" + "p-locate": "^5.0.0" }, "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true - }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -2133,53 +2088,15 @@ "dev": true }, "node_modules/log-symbols": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-3.0.0.tgz", - "integrity": "sha512-dSkNGuI7iG3mfvDzUuYZyvk5dD9ocYCYzNU6CYDE6+Xqd+gwme6Z00NS3dUh8mq/73HaEtT7m6W+yUPtU6BZnQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", + "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", "dev": true, "dependencies": { - "chalk": "^2.4.2" + "chalk": "^4.0.0" }, "engines": { - "node": ">=8" - } - }, - "node_modules/log-symbols/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/log-symbols/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/log-symbols/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" + "node": ">=10" } }, "node_modules/lowercase-keys": { @@ -2344,81 +2261,140 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "node_modules/mocha": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-7.1.1.tgz", - "integrity": "sha512-3qQsu3ijNS3GkWcccT5Zw0hf/rWvu1fTN9sPvEd81hlwsr30GX2GcDSSoBxo24IR8FelmrAydGC6/1J5QQP4WA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", + "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", "dev": true, + "license": "MIT", "dependencies": { - "ansi-colors": "3.2.3", + "@ungap/promise-all-settled": "1.1.2", + "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.3.0", - "debug": "3.2.6", - "diff": "3.5.0", - "escape-string-regexp": "1.0.5", - "find-up": "3.0.0", - "glob": "7.1.3", + "chokidar": "3.5.1", + "debug": "4.3.1", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.1.6", "growl": "1.10.5", "he": "1.2.0", - "js-yaml": "3.13.1", - "log-symbols": "3.0.0", + "js-yaml": "4.0.0", + "log-symbols": "4.0.0", "minimatch": "3.0.4", - "mkdirp": "0.5.3", - "ms": "2.1.1", - "node-environment-flags": "1.0.6", - "object.assign": "4.1.0", - "strip-json-comments": "2.0.1", - "supports-color": "6.0.0", - "which": "1.3.1", + "ms": "2.1.3", + "nanoid": "3.1.20", + "serialize-javascript": "5.0.1", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "which": "2.0.2", "wide-align": "1.1.3", - "yargs": "13.3.2", - "yargs-parser": "13.1.2", - "yargs-unparser": "1.6.0" + "workerpool": "6.1.0", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" }, "bin": { "_mocha": "bin/_mocha", "mocha": "bin/mocha" }, "engines": { - "node": ">= 8.0.0" + "node": ">= 10.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mochajs" } }, "node_modules/mocha/node_modules/debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "dev": true, "dependencies": { - "ms": "^2.1.1" + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/mocha/node_modules/mkdirp": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.3.tgz", - "integrity": "sha512-P+2gwrFqx8lhew375MQHHeTlY8AuOJSrGf0R5ddkEndUkmwpgUob/vQuBD1V22/Cw1/lJr4x+EjllSezBThzBg==", + "node_modules/mocha/node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/mocha/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true, - "dependencies": { - "minimist": "^1.2.5" + "engines": { + "node": ">=10" }, - "bin": { - "mkdirp": "bin/cmd.js" + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mocha/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" } }, "node_modules/mocha/node_modules/ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mocha/node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mocha/node_modules/supports-color": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.0.0.tgz", - "integrity": "sha512-on9Kwidc1IUQo+bQdhi8+Tijpo0e1SS6RoGo2guUwn5vdaxw8RXOF9Vb2ws+ihWOmh4JnCJOvaziZWP1VABaLg==", + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "dependencies": { - "has-flag": "^3.0.0" + "has-flag": "^4.0.0" }, "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/mocha/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" } }, "node_modules/ms": { @@ -2472,6 +2448,18 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/nanoid": { + "version": "3.1.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", + "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -2515,16 +2503,6 @@ "semver": "^5.4.1" } }, - "node_modules/node-environment-flags": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.6.tgz", - "integrity": "sha512-5Evy2epuL+6TM0lCQGpFIj6KwiEsGh1SrHUhTbNX+sLbBtjidPZFAnVK9y5yU1+h//RitLbRHTIMyxQPtxMdHw==", - "dev": true, - "dependencies": { - "object.getownpropertydescriptors": "^2.0.3", - "semver": "^5.7.0" - } - }, "node_modules/node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", @@ -2657,49 +2635,6 @@ "node": ">=0.10.0" } }, - "node_modules/object-inspect": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", - "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", - "dev": true - }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", - "dev": true, - "dependencies": { - "define-properties": "^1.1.2", - "function-bind": "^1.1.1", - "has-symbols": "^1.0.0", - "object-keys": "^1.0.11" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/object.getownpropertydescriptors": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz", - "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==", - "dev": true, - "dependencies": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -2729,36 +2664,33 @@ } }, "node_modules/p-limit": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", - "integrity": "sha512-WGR+xHecKTr7EbUEhyLSh5Dube9JtdiG78ufaeLxTgpudf/20KqyMioIUZJAezlTIi6evxuoUs9YXc11cU+yzQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "dev": true, "dependencies": { - "p-try": "^2.0.0" + "yocto-queue": "^0.1.0" }, "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, "dependencies": { - "p-limit": "^2.0.0" + "p-limit": "^3.0.2" }, "engines": { - "node": ">=6" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/package-json": { @@ -2790,12 +2722,12 @@ } }, "node_modules/path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, "engines": { - "node": ">=4" + "node": ">=8" } }, "node_modules/path-is-absolute": { @@ -3038,6 +2970,15 @@ "node": ">=0.6" } }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -3088,15 +3029,15 @@ } }, "node_modules/readdirp": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.2.0.tgz", - "integrity": "sha512-crk4Qu3pmXwgxdSgGhgA/eXiJAPQiX4GMOZZMXnqKxHX7TaoL+3gQVo/WeuAiogr07DpnfjIMpXXa+PAIvwPGQ==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", "dev": true, "dependencies": { - "picomatch": "^2.0.4" + "picomatch": "^2.2.1" }, "engines": { - "node": ">= 8" + "node": ">=8.10.0" } }, "node_modules/redis": { @@ -3177,12 +3118,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", - "dev": true - }, "node_modules/safe-buffer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", @@ -3241,6 +3176,15 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, + "node_modules/serialize-javascript": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", + "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, "node_modules/serve-static": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", @@ -3371,12 +3315,6 @@ "readable-stream": "^3.0.0" } }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", - "dev": true - }, "node_modules/sqlstring": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", @@ -3432,54 +3370,6 @@ "node": ">=4" } }, - "node_modules/string.prototype.trimend": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.0.tgz", - "integrity": "sha512-EEJnGqa/xNfIg05SxiPSqRS7S9qwDhYts1TSLR1BQfYUfPe1stofgGKvwERK9+9yf+PpfBMlpBaCHucXGPQfUA==", - "dev": true, - "dependencies": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" - } - }, - "node_modules/string.prototype.trimleft": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", - "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", - "dev": true, - "dependencies": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimstart": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/string.prototype.trimright": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", - "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", - "dev": true, - "dependencies": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimend": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/string.prototype.trimstart": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.0.tgz", - "integrity": "sha512-iCP8g01NFYiiBOnwG1Xc3WZLyoo+RuBymwIlWncShXDDJYWN6DbnM3odslBJdgCdRlq94B5s63NWAZlcn2CS4w==", - "dev": true, - "dependencies": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" - } - }, "node_modules/strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -3946,12 +3836,6 @@ "which": "bin/which" } }, - "node_modules/which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", - "dev": true - }, "node_modules/wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", @@ -3972,65 +3856,71 @@ "node": ">=4" } }, + "node_modules/workerpool": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", + "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "dev": true + }, "node_modules/wrap-ansi": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", - "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, "dependencies": { - "ansi-styles": "^3.2.0", - "string-width": "^3.0.0", - "strip-ansi": "^5.0.0" + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" }, "engines": { - "node": ">=6" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, "node_modules/wrap-ansi/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true, "engines": { - "node": ">=6" + "node": ">=8" } }, - "node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, "engines": { - "node": ">=4" + "node": ">=8" } }, "node_modules/wrap-ansi/node_modules/string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", "dev": true, "dependencies": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/wrap-ansi/node_modules/strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "dependencies": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.0" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/wrappy": { @@ -4067,10 +3957,13 @@ } }, "node_modules/y18n": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", - "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==", - "dev": true + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } }, "node_modules/yallist": { "version": "4.0.0", @@ -4078,89 +3971,89 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { - "version": "13.3.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", - "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", "dev": true, "dependencies": { - "cliui": "^5.0.0", - "find-up": "^3.0.0", - "get-caller-file": "^2.0.1", + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^3.0.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^13.1.2" + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" } }, "node_modules/yargs-parser": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", - "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", - "dev": true, - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } - }, - "node_modules/yargs-parser/node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", "dev": true, "engines": { - "node": ">=6" + "node": ">=10" } }, "node_modules/yargs-unparser": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-1.6.0.tgz", - "integrity": "sha512-W9tKgmSn0DpSatfri0nx52Joq5hVXgeLiqR/5G0sZNDoLZFOr/xjBUDcShCOGNsBnEMNo1KAMBkTej1Hm62HTw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", "dev": true, "dependencies": { - "flat": "^4.1.0", - "lodash": "^4.17.15", - "yargs": "^13.3.0" + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" }, "engines": { - "node": ">=6" + "node": ">=10" } }, "node_modules/yargs/node_modules/ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true, "engines": { - "node": ">=6" + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" } }, "node_modules/yargs/node_modules/string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", "dev": true, "dependencies": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/yargs/node_modules/strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "dependencies": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.0" }, "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/yn": { @@ -4172,6 +4065,18 @@ "node": ">=6" } }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/youtube-api": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/youtube-api/-/youtube-api-3.0.1.tgz", @@ -4306,9 +4211,9 @@ "dev": true }, "@types/mocha": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-8.0.3.tgz", - "integrity": "sha512-vyxR57nv8NfcU0GZu8EUXZLTbCMupIUwy95LJ6lllN+JRPG25CwMHoB1q5xKh8YKhQnHYRAn4yW2yuHbf/5xgg==", + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-8.2.2.tgz", + "integrity": "sha512-Lwh0lzzqT5Pqh6z61P3c3P5nm6fzQK/MMHl9UKeneAeInVflBSz1O2EkX6gM6xfJd7FBXBY5purtLx7fUiZ7Hw==", "dev": true }, "@types/node": { @@ -4400,6 +4305,12 @@ "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==", "dev": true }, + "@ungap/promise-all-settled": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", + "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", + "dev": true + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -4456,9 +4367,9 @@ } }, "ansi-colors": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", - "integrity": "sha512-LEHHyuhlPY3TmuUYMh2oz89lTShfvgbmzaBcxve9t/9Wuy7Dwf4yoAKcND7KFT1HAQfqZ12qtc+DUrBMeKF9nw==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", "dev": true }, "ansi-regex": { @@ -4466,6 +4377,32 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + }, + "dependencies": { + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + } + } + }, "anymatch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", @@ -4531,13 +4468,10 @@ "dev": true }, "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "requires": { - "sprintf-js": "~1.0.2" - } + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "array-flatten": { "version": "1.1.1", @@ -4767,26 +4701,42 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, + "camelcase": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz", + "integrity": "sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg==", + "dev": true + }, "capture-stack-trace": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", "integrity": "sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==", "dev": true }, + "chalk": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", + "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, "chokidar": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.3.0.tgz", - "integrity": "sha512-dGmKLDdT3Gdl7fBUe8XK+gAtGmzy5Fn0XkkWQuYxGIgWVPPse2CxFA5mtrlD0TOHaHjEUqkWNyP1XdHoJES/4A==", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", + "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", "dev": true, "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", - "fsevents": "~2.1.1", + "fsevents": "~2.3.1", "glob-parent": "~5.1.0", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", - "readdirp": "~3.2.0" + "readdirp": "~3.5.0" } }, "chownr": { @@ -4807,40 +4757,46 @@ "dev": true }, "cliui": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", - "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", "dev": true, "requires": { - "string-width": "^3.1.0", - "strip-ansi": "^5.2.0", - "wrap-ansi": "^5.1.0" + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", "dev": true, "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" } }, "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "requires": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.0" } } } @@ -4974,9 +4930,9 @@ } }, "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", "dev": true }, "decompress-response": { @@ -4992,15 +4948,6 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" }, - "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, - "requires": { - "object-keys": "^1.0.12" - } - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -5033,9 +4980,9 @@ "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" }, "diff": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", "dev": true }, "dot-prop": { @@ -5072,9 +5019,9 @@ "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "emoji-regex": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", - "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, "encodeurl": { @@ -5090,35 +5037,11 @@ "once": "^1.4.0" } }, - "es-abstract": { - "version": "1.17.5", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", - "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", - "dev": true, - "requires": { - "es-to-primitive": "^1.2.1", - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.1", - "is-callable": "^1.1.5", - "is-regex": "^1.0.5", - "object-inspect": "^1.7.0", - "object-keys": "^1.1.1", - "object.assign": "^4.1.0", - "string.prototype.trimleft": "^2.1.1", - "string.prototype.trimright": "^2.1.1" - } - }, - "es-to-primitive": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dev": true, - "requires": { - "is-callable": "^1.1.4", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.2" - } + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true }, "escape-html": { "version": "1.0.3", @@ -5131,12 +5054,6 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true - }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -5255,22 +5172,20 @@ } }, "find-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", - "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, "requires": { - "locate-path": "^3.0.0" + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" } }, "flat": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/flat/-/flat-4.1.0.tgz", - "integrity": "sha512-Px/TiLIznH7gEDlPXcUD4KnBusa6kR6ayRUVcnEAbreRIuhkqow/mun59BuRXwoYk7ZQOLW1ZM05ilIvK38hFw==", - "dev": true, - "requires": { - "is-buffer": "~2.0.3" - } + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true }, "form-data": { "version": "3.0.0", @@ -5313,18 +5228,12 @@ "dev": true }, "fsevents": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.2.tgz", - "integrity": "sha512-R4wDiBwZ0KzpgOWetKDug1FZcYhqYnUYKtfZYt4mD5SBz76q0KR4Q9o7GIPamsVPGmW3EYPPJ0dOOjvx32ldZA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "dev": true, "optional": true }, - "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true - }, "gauge": { "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", @@ -5411,9 +5320,9 @@ "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" }, "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -5554,27 +5463,12 @@ } } }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, - "requires": { - "function-bind": "^1.1.1" - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", "dev": true }, - "has-symbols": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", - "dev": true - }, "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", @@ -5692,18 +5586,6 @@ "binary-extensions": "^2.0.0" } }, - "is-buffer": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.4.tgz", - "integrity": "sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==", - "dev": true - }, - "is-callable": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", - "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", - "dev": true - }, "is-ci": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz", @@ -5713,12 +5595,6 @@ "ci-info": "^1.5.0" } }, - "is-date-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", - "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", - "dev": true - }, "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -5776,21 +5652,18 @@ "path-is-inside": "^1.0.1" } }, + "is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true + }, "is-redirect": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=", "dev": true }, - "is-regex": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", - "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", - "dev": true, - "requires": { - "has": "^1.0.3" - } - }, "is-retry-allowed": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", @@ -5803,15 +5676,6 @@ "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", "dev": true }, - "is-symbol": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", - "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", - "dev": true, - "requires": { - "has-symbols": "^1.0.1" - } - }, "isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -5830,13 +5694,12 @@ "integrity": "sha512-ErTBd++b17E8nmWII1K1uZtBgD1E8RjyvwmxlCjPHNqHMD7gmcMHOw0E8Ro/6+QT4PhHRSnnMo7bxa1vFPkwhg==" }, "js-yaml": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", - "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", "dev": true, "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" + "argparse": "^2.0.1" } }, "json-bigint": { @@ -5882,21 +5745,14 @@ } }, "locate-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", - "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, "requires": { - "p-locate": "^3.0.0", - "path-exists": "^3.0.0" + "p-locate": "^5.0.0" } }, - "lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true - }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -5904,43 +5760,12 @@ "dev": true }, "log-symbols": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-3.0.0.tgz", - "integrity": "sha512-dSkNGuI7iG3mfvDzUuYZyvk5dD9ocYCYzNU6CYDE6+Xqd+gwme6Z00NS3dUh8mq/73HaEtT7m6W+yUPtU6BZnQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", + "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", "dev": true, "requires": { - "chalk": "^2.4.2" - }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } + "chalk": "^4.0.0" } }, "lowercase-keys": { @@ -6062,68 +5887,95 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, "mocha": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-7.1.1.tgz", - "integrity": "sha512-3qQsu3ijNS3GkWcccT5Zw0hf/rWvu1fTN9sPvEd81hlwsr30GX2GcDSSoBxo24IR8FelmrAydGC6/1J5QQP4WA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", + "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", "dev": true, "requires": { - "ansi-colors": "3.2.3", + "@ungap/promise-all-settled": "1.1.2", + "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.3.0", - "debug": "3.2.6", - "diff": "3.5.0", - "escape-string-regexp": "1.0.5", - "find-up": "3.0.0", - "glob": "7.1.3", + "chokidar": "3.5.1", + "debug": "4.3.1", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.1.6", "growl": "1.10.5", "he": "1.2.0", - "js-yaml": "3.13.1", - "log-symbols": "3.0.0", + "js-yaml": "4.0.0", + "log-symbols": "4.0.0", "minimatch": "3.0.4", - "mkdirp": "0.5.3", - "ms": "2.1.1", - "node-environment-flags": "1.0.6", - "object.assign": "4.1.0", - "strip-json-comments": "2.0.1", - "supports-color": "6.0.0", - "which": "1.3.1", + "ms": "2.1.3", + "nanoid": "3.1.20", + "serialize-javascript": "5.0.1", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "which": "2.0.2", "wide-align": "1.1.3", - "yargs": "13.3.2", - "yargs-parser": "13.1.2", - "yargs-unparser": "1.6.0" + "workerpool": "6.1.0", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" }, "dependencies": { "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } } }, - "mkdirp": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.3.tgz", - "integrity": "sha512-P+2gwrFqx8lhew375MQHHeTlY8AuOJSrGf0R5ddkEndUkmwpgUob/vQuBD1V22/Cw1/lJr4x+EjllSezBThzBg==", - "dev": true, - "requires": { - "minimist": "^1.2.5" - } + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true }, "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, "supports-color": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.0.0.tgz", - "integrity": "sha512-on9Kwidc1IUQo+bQdhi8+Tijpo0e1SS6RoGo2guUwn5vdaxw8RXOF9Vb2ws+ihWOmh4JnCJOvaziZWP1VABaLg==", + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", "dev": true, "requires": { - "has-flag": "^3.0.0" + "has-flag": "^4.0.0" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" } } } @@ -6178,6 +6030,12 @@ } } }, + "nanoid": { + "version": "3.1.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", + "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "dev": true + }, "napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -6220,16 +6078,6 @@ "semver": "^5.4.1" } }, - "node-environment-flags": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.6.tgz", - "integrity": "sha512-5Evy2epuL+6TM0lCQGpFIj6KwiEsGh1SrHUhTbNX+sLbBtjidPZFAnVK9y5yU1+h//RitLbRHTIMyxQPtxMdHw==", - "dev": true, - "requires": { - "object.getownpropertydescriptors": "^2.0.3", - "semver": "^5.7.0" - } - }, "node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", @@ -6334,40 +6182,6 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, - "object-inspect": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", - "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", - "dev": true - }, - "object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true - }, - "object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", - "dev": true, - "requires": { - "define-properties": "^1.1.2", - "function-bind": "^1.1.1", - "has-symbols": "^1.0.0", - "object-keys": "^1.0.11" - } - }, - "object.getownpropertydescriptors": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz", - "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.0-next.1" - } - }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -6391,29 +6205,23 @@ "dev": true }, "p-limit": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz", - "integrity": "sha512-WGR+xHecKTr7EbUEhyLSh5Dube9JtdiG78ufaeLxTgpudf/20KqyMioIUZJAezlTIi6evxuoUs9YXc11cU+yzQ==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "dev": true, "requires": { - "p-try": "^2.0.0" + "yocto-queue": "^0.1.0" } }, "p-locate": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", - "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", "dev": true, "requires": { - "p-limit": "^2.0.0" + "p-limit": "^3.0.2" } }, - "p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true - }, "package-json": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/package-json/-/package-json-4.0.1.tgz", @@ -6437,9 +6245,9 @@ "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" }, "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true }, "path-is-absolute": { @@ -6636,6 +6444,15 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz", "integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==" }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.0" + } + }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -6674,12 +6491,12 @@ } }, "readdirp": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.2.0.tgz", - "integrity": "sha512-crk4Qu3pmXwgxdSgGhgA/eXiJAPQiX4GMOZZMXnqKxHX7TaoL+3gQVo/WeuAiogr07DpnfjIMpXXa+PAIvwPGQ==", + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", "dev": true, "requires": { - "picomatch": "^2.0.4" + "picomatch": "^2.2.1" } }, "redis": { @@ -6741,12 +6558,6 @@ "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", "dev": true }, - "require-main-filename": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", - "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", - "dev": true - }, "safe-buffer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", @@ -6798,6 +6609,15 @@ } } }, + "serialize-javascript": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", + "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "dev": true, + "requires": { + "randombytes": "^2.1.0" + } + }, "serve-static": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", @@ -6901,12 +6721,6 @@ "readable-stream": "^3.0.0" } }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", - "dev": true - }, "sqlstring": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", @@ -6949,48 +6763,6 @@ } } }, - "string.prototype.trimend": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.0.tgz", - "integrity": "sha512-EEJnGqa/xNfIg05SxiPSqRS7S9qwDhYts1TSLR1BQfYUfPe1stofgGKvwERK9+9yf+PpfBMlpBaCHucXGPQfUA==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" - } - }, - "string.prototype.trimleft": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", - "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimstart": "^1.0.0" - } - }, - "string.prototype.trimright": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", - "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5", - "string.prototype.trimend": "^1.0.0" - } - }, - "string.prototype.trimstart": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.0.tgz", - "integrity": "sha512-iCP8g01NFYiiBOnwG1Xc3WZLyoo+RuBymwIlWncShXDDJYWN6DbnM3odslBJdgCdRlq94B5s63NWAZlcn2CS4w==", - "dev": true, - "requires": { - "define-properties": "^1.1.3", - "es-abstract": "^1.17.5" - } - }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -7364,12 +7136,6 @@ "isexe": "^2.0.0" } }, - "which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", - "dev": true - }, "wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", @@ -7387,50 +7153,53 @@ "string-width": "^2.1.1" } }, + "workerpool": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", + "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "dev": true + }, "wrap-ansi": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", - "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, "requires": { - "ansi-styles": "^3.2.0", - "string-width": "^3.0.0", - "strip-ansi": "^5.0.0" + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true }, "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", "dev": true, "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" } }, "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "requires": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.0" } } } @@ -7463,9 +7232,9 @@ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, "y18n": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", - "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "dev": true }, "yallist": { @@ -7474,78 +7243,70 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "yargs": { - "version": "13.3.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", - "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", "dev": true, "requires": { - "cliui": "^5.0.0", - "find-up": "^3.0.0", - "get-caller-file": "^2.0.1", + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^3.0.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^13.1.2" + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, "string-width": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", - "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", "dev": true, "requires": { - "emoji-regex": "^7.0.1", - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^5.1.0" + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" } }, "strip-ansi": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", - "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", "dev": true, "requires": { - "ansi-regex": "^4.1.0" + "ansi-regex": "^5.0.0" } } } }, "yargs-parser": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", - "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "dependencies": { - "camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true - } - } + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true }, "yargs-unparser": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-1.6.0.tgz", - "integrity": "sha512-W9tKgmSn0DpSatfri0nx52Joq5hVXgeLiqR/5G0sZNDoLZFOr/xjBUDcShCOGNsBnEMNo1KAMBkTej1Hm62HTw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", "dev": true, "requires": { - "flat": "^4.1.0", - "lodash": "^4.17.15", - "yargs": "^13.3.0" + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" } }, "yn": { @@ -7554,6 +7315,12 @@ "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", "dev": true }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true + }, "youtube-api": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/youtube-api/-/youtube-api-3.0.1.tgz", diff --git a/package.json b/package.json index b0d52d2..8e0d18b 100644 --- a/package.json +++ b/package.json @@ -30,13 +30,13 @@ "@types/better-sqlite3": "^5.4.0", "@types/express": "^4.17.8", "@types/express-rate-limit": "^5.1.0", - "@types/mocha": "^8.0.3", + "@types/mocha": "^8.2.2", "@types/node": "^14.11.9", "@types/node-fetch": "^2.5.7", "@types/pg": "^7.14.10", "@types/redis": "^2.8.28", "@types/request": "^2.48.5", - "mocha": "^7.1.1", + "mocha": "^8.4.0", "nodemon": "^2.0.2", "sinon": "^9.2.0", "ts-mock-imports": "^1.3.0", From 799aef0b65aafc064182f5d15e46d73ef75eae86 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 15:49:12 -0400 Subject: [PATCH 18/85] Move redis code to middleware --- src/middleware/queryCacher.ts | 35 +++++++++++++++++++++++++++++++++ src/routes/getSkipSegments.ts | 30 +++++----------------------- src/routes/postSkipSegments.ts | 8 ++++++-- src/routes/voteOnSponsorTime.ts | 14 ++++--------- 4 files changed, 50 insertions(+), 37 deletions(-) create mode 100644 src/middleware/queryCacher.ts diff --git a/src/middleware/queryCacher.ts b/src/middleware/queryCacher.ts new file mode 100644 index 0000000..e1f9f09 --- /dev/null +++ b/src/middleware/queryCacher.ts @@ -0,0 +1,35 @@ +import redis from "../utils/redis"; +import { Logger } from "../utils/logger"; +import { skipSegmentsHashKey, skipSegmentsKey } from "./redisKeys"; +import { Service, VideoID, VideoIDHash } from "../types/segments.model"; + +async function get(fetchFromDB: () => Promise, key: string): Promise { + const {err, reply} = await redis.getAsync(key); + + if (!err && reply) { + try { + Logger.debug("Got data from redis: " + reply); + return JSON.parse(reply); + } catch (e) { + // If all else, continue on to fetching from the database + } + } + + const data = await fetchFromDB(); + + redis.setAsync(key, JSON.stringify(data)); + return data; +} + +function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) { + if (videoInfo) { + redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); + redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); + } +} + + +export const QueryCacher = { + get, + clearVideoCache +} \ No newline at end of file diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index c3e7cbf..63ad30e 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -1,5 +1,4 @@ import { Request, Response } from 'express'; -import { RedisClient } from 'redis'; import { config } from '../config'; import { db, privateDB } from '../databases/databases'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; @@ -9,8 +8,7 @@ import { getCategoryActionType } from '../utils/categoryInfo'; import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; -import redis from '../utils/redis'; -import { getSkipSegmentsByHash } from './getSkipSegmentsByHash'; +import { QueryCacher } from '../middleware/queryCacher' async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise { @@ -133,10 +131,10 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [hashedVideoIDPrefix + '%', service] - ); + ) as Promise; if (hashedVideoIDPrefix.length === 4) { - return await getSegmentsFromDB(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service)) + return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service)) } return await fetchFromDB(); @@ -149,27 +147,9 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [videoID, service] - ); + ) as Promise; - return await getSegmentsFromDB(fetchFromDB, skipSegmentsKey(videoID, service)) -} - -async function getSegmentsFromDB(fetchFromDB: () => Promise, key: string): Promise { - const {err, reply} = await redis.getAsync(key); - - if (!err && reply) { - try { - Logger.debug("Got data from redis: " + reply); - return JSON.parse(reply); - } catch (e) { - // If all else, continue on to fetching from the database - } - } - - const data = await fetchFromDB(); - - redis.setAsync(key, JSON.stringify(data)); - return data; + return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service)) } //gets a weighted random choice from the choices array based on their `votes` property. diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 72468b3..2806e8a 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -16,6 +16,7 @@ import redis from '../utils/redis'; import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; import { deleteLockCategories } from './deleteLockCategories'; import { getCategoryActionType } from '../utils/categoryInfo'; +import { QueryCacher } from '../middleware/queryCacher'; interface APIVideoInfo { err: string | boolean, @@ -528,8 +529,11 @@ export async function postSkipSegments(req: Request, res: Response) { await privateDB.prepare('run', `INSERT INTO "sponsorTimes" VALUES(?, ?, ?)`, [videoID, hashedIP, timeSubmitted]); // Clear redis cache for this video - redis.delAsync(skipSegmentsKey(videoID, service)); - redis.delAsync(skipSegmentsHashKey(hashedVideoID, service)); + QueryCacher.clearVideoCache({ + videoID, + hashedVideoID, + service + }); } catch (err) { //a DB change probably occurred res.sendStatus(500); diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 0bc686c..0e3c02a 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -15,6 +15,7 @@ import redis from '../utils/redis'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; import { getCategoryActionType } from '../utils/categoryInfo'; +import { QueryCacher } from '../middleware/queryCacher'; const voteTypes = { normal: 0, @@ -230,7 +231,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i } } - clearRedisCache(videoInfo); + QueryCacher.clearVideoCache(videoInfo); res.sendStatus(finalResponse.finalStatus); } @@ -416,7 +417,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]); } - clearRedisCache(videoInfo); + QueryCacher.clearVideoCache(videoInfo); //for each positive vote, see if a hidden submission can be shown again if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { @@ -465,11 +466,4 @@ export async function voteOnSponsorTime(req: Request, res: Response) { res.status(500).json({error: 'Internal error creating segment vote'}); } -} - -function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) { - if (videoInfo) { - redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); - redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); - } -} +} \ No newline at end of file From cfcb6c6b64c214fa84e9dccbc6db1c869954dfcb Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 16:53:35 -0400 Subject: [PATCH 19/85] Add reputation system --- src/middleware/queryCacher.ts | 2 +- src/middleware/redisKeys.ts | 7 +++++- src/middleware/reputation.ts | 45 +++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 src/middleware/reputation.ts diff --git a/src/middleware/queryCacher.ts b/src/middleware/queryCacher.ts index e1f9f09..ec48114 100644 --- a/src/middleware/queryCacher.ts +++ b/src/middleware/queryCacher.ts @@ -3,7 +3,7 @@ import { Logger } from "../utils/logger"; import { skipSegmentsHashKey, skipSegmentsKey } from "./redisKeys"; import { Service, VideoID, VideoIDHash } from "../types/segments.model"; -async function get(fetchFromDB: () => Promise, key: string): Promise { +async function get(fetchFromDB: () => Promise, key: string): Promise { const {err, reply} = await redis.getAsync(key); if (!err && reply) { diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts index 920b2da..5234a51 100644 --- a/src/middleware/redisKeys.ts +++ b/src/middleware/redisKeys.ts @@ -1,4 +1,5 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model"; +import { UserID } from "../types/user.model"; import { Logger } from "../utils/logger"; export function skipSegmentsKey(videoID: VideoID, service: Service): string { @@ -10,4 +11,8 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix); return "segments." + service + "." + hashedVideoIDPrefix; -} \ No newline at end of file +} + +export function userKey(userID: UserID): string { + return "user." + userID; +} \ No newline at end of file diff --git a/src/middleware/reputation.ts b/src/middleware/reputation.ts new file mode 100644 index 0000000..99bda65 --- /dev/null +++ b/src/middleware/reputation.ts @@ -0,0 +1,45 @@ +import { db } from "../databases/databases"; +import { UserID } from "../types/user.model"; +import { QueryCacher } from "./queryCacher"; +import { userKey } from "./redisKeys"; + +interface ReputationDBResult { + totalSubmissions: number, + downvotedSubmissions: number, + upvotedSum: number, + oldUpvotedSubmissions: number +} + +export async function getReputation(userID: UserID) { + const pastDate = Date.now() - 1000 * 1000 * 60 * 60 * 24 * 45; // 45 days ago + const fetchFromDB = () => db.prepare("get", + `SELECT COUNT(*) AS "totalSubmissions", + SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions", + SUM(CASE WHEN "votes" > 0 THEN "votes" ELSE 0 END) AS "upvotedSum", + SUM(CASE WHEN "timeSubmitted" < ? AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions" + FROM "sponsorTimes" WHERE "userID" = ?`, [pastDate, userID]) as Promise; + + const result = await QueryCacher.get(fetchFromDB, userKey(userID)); + + // Grace period + if (result.totalSubmissions < 5) { + return 0; + } + + const downvoteRatio = result.downvotedSubmissions / result.totalSubmissions; + if (downvoteRatio > 0.3) { + return convertRange(downvoteRatio, 0.3, 1, -0.5, -1.5); + } + + if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) { + return 0 + } + + return convertRange(Math.min(result.upvotedSum, 50), 5, 50, 0, 15); +} + +function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number { + const currentRange = currentMax - currentMin; + const targetRange = targetMax - targetMin; + return ((value - currentMin) / currentRange) * targetRange + targetMin; +} \ No newline at end of file From 0051022906c5461b16625a0cce94d7f5a2b397b1 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 16:55:31 -0400 Subject: [PATCH 20/85] remove old user trustworthy code --- src/routes/voteOnSponsorTime.ts | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 0e3c02a..4418c27 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -5,14 +5,11 @@ import fetch from 'node-fetch'; import {YouTubeAPI} from '../utils/youtubeApi'; import {db, privateDB} from '../databases/databases'; import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from '../utils/webhookUtils'; -import {isUserTrustworthy} from '../utils/isUserTrustworthy'; import {getFormattedTime} from '../utils/getFormattedTime'; import {getIP} from '../utils/getIP'; import {getHash} from '../utils/getHash'; import {config} from '../config'; import { UserID } from '../types/user.model'; -import redis from '../utils/redis'; -import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; import { getCategoryActionType } from '../utils/categoryInfo'; import { QueryCacher } from '../middleware/queryCacher'; @@ -418,31 +415,6 @@ export async function voteOnSponsorTime(req: Request, res: Response) { } QueryCacher.clearVideoCache(videoInfo); - - //for each positive vote, see if a hidden submission can be shown again - if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { - //find the UUID that submitted the submission that was voted on - const submissionUserIDInfo = await db.prepare('get', 'SELECT "userID" FROM "sponsorTimes" WHERE "UUID" = ?', [UUID]); - if (!submissionUserIDInfo) { - // They are voting on a non-existent submission - res.status(400).send("Voting on a non-existent submission"); - return; - } - - const submissionUserID = submissionUserIDInfo.userID; - - //check if any submissions are hidden - const hiddenSubmissionsRow = await db.prepare('get', 'SELECT count(*) as "hiddenSubmissions" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" > 0', [submissionUserID]); - - if (hiddenSubmissionsRow.hiddenSubmissions > 0) { - //see if some of this users submissions should be visible again - - if (await isUserTrustworthy(submissionUserID)) { - //they are trustworthy again, show 2 of their submissions again, if there are two to show - await db.prepare('run', 'UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE ROWID IN (SELECT ROWID FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = 1 LIMIT 2)', [submissionUserID]); - } - } - } } res.status(finalResponse.finalStatus).send(finalResponse.finalMessage ?? undefined); From a5f9c2a022513059aabe6d4ed823a478ae6396dc Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 16:57:41 -0400 Subject: [PATCH 21/85] Don't allow self votes --- src/routes/voteOnSponsorTime.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 4418c27..c7bb6e7 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -383,7 +383,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) { // Only change the database if they have made a submission before and haven't voted recently const ableToVote = isVIP - || ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined + || (!(isOwnSubmission && incrementAmount > 0) + && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined) && finalResponse.finalStatus === 200; From 194c657ba7ba3da8997c2d7ca3617bc5c637b4b6 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 17:05:06 -0400 Subject: [PATCH 22/85] Clear reputation cache --- src/middleware/queryCacher.ts | 6 ++++-- src/middleware/redisKeys.ts | 4 ++-- src/middleware/reputation.ts | 4 ++-- src/routes/postSkipSegments.ts | 3 ++- src/routes/voteOnSponsorTime.ts | 8 ++++---- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/middleware/queryCacher.ts b/src/middleware/queryCacher.ts index ec48114..64cf206 100644 --- a/src/middleware/queryCacher.ts +++ b/src/middleware/queryCacher.ts @@ -1,7 +1,8 @@ import redis from "../utils/redis"; import { Logger } from "../utils/logger"; -import { skipSegmentsHashKey, skipSegmentsKey } from "./redisKeys"; +import { skipSegmentsHashKey, skipSegmentsKey, reputationKey } from "./redisKeys"; import { Service, VideoID, VideoIDHash } from "../types/segments.model"; +import { UserID } from "../types/user.model"; async function get(fetchFromDB: () => Promise, key: string): Promise { const {err, reply} = await redis.getAsync(key); @@ -21,10 +22,11 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { return data; } -function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) { +function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID: UserID; }) { if (videoInfo) { redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); + redis.delAsync(reputationKey(videoInfo.userID)); } } diff --git a/src/middleware/redisKeys.ts b/src/middleware/redisKeys.ts index 5234a51..1a0491c 100644 --- a/src/middleware/redisKeys.ts +++ b/src/middleware/redisKeys.ts @@ -13,6 +13,6 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S return "segments." + service + "." + hashedVideoIDPrefix; } -export function userKey(userID: UserID): string { - return "user." + userID; +export function reputationKey(userID: UserID): string { + return "reputation.user." + userID; } \ No newline at end of file diff --git a/src/middleware/reputation.ts b/src/middleware/reputation.ts index 99bda65..cb18b24 100644 --- a/src/middleware/reputation.ts +++ b/src/middleware/reputation.ts @@ -1,7 +1,7 @@ import { db } from "../databases/databases"; import { UserID } from "../types/user.model"; import { QueryCacher } from "./queryCacher"; -import { userKey } from "./redisKeys"; +import { reputationKey } from "./redisKeys"; interface ReputationDBResult { totalSubmissions: number, @@ -19,7 +19,7 @@ export async function getReputation(userID: UserID) { SUM(CASE WHEN "timeSubmitted" < ? AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions" FROM "sponsorTimes" WHERE "userID" = ?`, [pastDate, userID]) as Promise; - const result = await QueryCacher.get(fetchFromDB, userKey(userID)); + const result = await QueryCacher.get(fetchFromDB, reputationKey(userID)); // Grace period if (result.totalSubmissions < 5) { diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 2806e8a..b46bfe3 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -532,7 +532,8 @@ export async function postSkipSegments(req: Request, res: Response) { QueryCacher.clearVideoCache({ videoID, hashedVideoID, - service + service, + userID }); } catch (err) { //a DB change probably occurred diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index c7bb6e7..39ebfd8 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -156,8 +156,8 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i return; } - const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`, - [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service}; + const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`, + [UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}; if (!videoInfo) { // Submission doesn't exist res.status(400).send("Submission doesn't exist."); @@ -364,8 +364,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) { } //check if the increment amount should be multiplied (downvotes have more power if there have been many views) - const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as - {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number}; + const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as + {videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number, userID: UserID}; if (voteTypeEnum === voteTypes.normal) { if ((isVIP || isOwnSubmission) && incrementAmount < 0) { From 5c2ab9087a2829fea5ec1cf471b92b0b9c17441e Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 17:54:51 -0400 Subject: [PATCH 23/85] Use reputation when sorting segments --- databases/_upgrade_sponsorTimes_12.sql | 31 ++++++++++++++++++++++++++ src/middleware/reputation.ts | 4 ++-- src/routes/getSkipSegments.ts | 24 +++++++++++++------- src/routes/postSkipSegments.ts | 8 ++++--- src/types/segments.model.ts | 5 +++++ 5 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_12.sql diff --git a/databases/_upgrade_sponsorTimes_12.sql b/databases/_upgrade_sponsorTimes_12.sql new file mode 100644 index 0000000..aeace54 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_12.sql @@ -0,0 +1,31 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_12" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "videoDuration" REAL NOT NULL DEFAULT '0', + "hidden" INTEGER NOT NULL DEFAULT '0', + "reputation" REAL NOT NULL DEFAULT 0, + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_12 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration","hidden",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_12 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 12 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/middleware/reputation.ts b/src/middleware/reputation.ts index cb18b24..31672a4 100644 --- a/src/middleware/reputation.ts +++ b/src/middleware/reputation.ts @@ -10,7 +10,7 @@ interface ReputationDBResult { oldUpvotedSubmissions: number } -export async function getReputation(userID: UserID) { +export async function getReputation(userID: UserID): Promise { const pastDate = Date.now() - 1000 * 1000 * 60 * 60 * 24 * 45; // 45 days ago const fetchFromDB = () => db.prepare("get", `SELECT COUNT(*) AS "totalSubmissions", @@ -32,7 +32,7 @@ export async function getReputation(userID: UserID) { } if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) { - return 0 + return 0; } return convertRange(Math.min(result.upvotedSum, 50), 5, 50, 0, 15); diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 63ad30e..0e5ce19 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -9,6 +9,7 @@ import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; import { QueryCacher } from '../middleware/queryCacher' +import { getReputation } from '../middleware/reputation'; async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise { @@ -41,7 +42,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: const filteredSegments = segments.filter((_, index) => shouldFilter[index]); const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1 - return chooseSegments(filteredSegments, maxSegments).map((chosenSegment) => ({ + return (await chooseSegments(filteredSegments, maxSegments)).map((chosenSegment) => ({ category, segment: [chosenSegment.startTime, chosenSegment.endTime], UUID: chosenSegment.UUID, @@ -128,7 +129,7 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service const fetchFromDB = () => db .prepare( 'all', - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" + `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [hashedVideoIDPrefix + '%', service] ) as Promise; @@ -144,7 +145,7 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P const fetchFromDB = () => db .prepare( 'all', - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" + `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [videoID, service] ) as Promise; @@ -170,7 +171,7 @@ function getWeightedRandomChoice(choices: T[], amountOf let choicesWithWeights: TWithWeight[] = choices.map(choice => { //The 3 makes -2 the minimum votes before being ignored completely //this can be changed if this system increases in popularity. - const weight = Math.exp((choice.votes + 3)); + const weight = Math.exp((choice.votes + 3 + choice.reputation)); totalWeight += weight; return {...choice, weight}; @@ -200,7 +201,7 @@ function getWeightedRandomChoice(choices: T[], amountOf //Only one similar time will be returned, randomly generated based on the sqrt of votes. //This allows new less voted items to still sometimes appear to give them a chance at getting votes. //Segments with less than -1 votes are already ignored before this function is called -function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { +async function chooseSegments(segments: DBSegment[], max: number): Promise { //Create groups of segments that are similar to eachother //Segments must be sorted by their startTime so that we can build groups chronologically: //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group @@ -209,9 +210,9 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { const overlappingSegmentsGroups: OverlappingSegmentGroup[] = []; let currentGroup: OverlappingSegmentGroup; let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created - segments.forEach(segment => { + for (const segment of segments) { if (segment.startTime > cursor) { - currentGroup = {segments: [], votes: 0, locked: false}; + currentGroup = {segments: [], votes: 0, reputation: 0, locked: false}; overlappingSegmentsGroups.push(currentGroup); } @@ -221,17 +222,24 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { currentGroup.votes += segment.votes; } + segment.reputation = Math.min(segment.reputation, await getReputation(segment.userID)); + if (segment.reputation > 0) { + currentGroup.reputation += segment.reputation; + } + if (segment.locked) { currentGroup.locked = true; } cursor = Math.max(cursor, segment.endTime); - }); + }; overlappingSegmentsGroups.forEach((group) => { if (group.locked) { group.segments = group.segments.filter((segment) => segment.locked); } + + group.reputation = group.reputation / group.segments.length; }); //if there are too many groups, find the best ones diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index b46bfe3..9c28964 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -17,6 +17,7 @@ import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Se import { deleteLockCategories } from './deleteLockCategories'; import { getCategoryActionType } from '../utils/categoryInfo'; import { QueryCacher } from '../middleware/queryCacher'; +import { getReputation } from '../middleware/reputation'; interface APIVideoInfo { err: string | boolean, @@ -508,6 +509,7 @@ export async function postSkipSegments(req: Request, res: Response) { } let startingVotes = 0 + decreaseVotes; + const reputation = await getReputation(userID); for (const segmentInfo of segments) { //this can just be a hash of the data @@ -519,9 +521,9 @@ export async function postSkipSegments(req: Request, res: Response) { const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" - ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID") - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID, + ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID") + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, reputation, shadowBanned, hashedVideoID, ], ); diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index c1606a9..945f294 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -1,5 +1,6 @@ import { HashedValue } from "./hash.model"; import { SBRecord } from "./lib.model"; +import { UserID } from "./user.model"; export type SegmentUUID = string & { __segmentUUIDBrand: unknown }; export type VideoID = string & { __videoIDBrand: unknown }; @@ -42,11 +43,13 @@ export interface DBSegment { startTime: number; endTime: number; UUID: SegmentUUID; + userID: UserID; votes: number; locked: boolean; shadowHidden: Visibility; videoID: VideoID; videoDuration: VideoDuration; + reputation: number; hashedVideoID: VideoIDHash; } @@ -54,10 +57,12 @@ export interface OverlappingSegmentGroup { segments: DBSegment[], votes: number; locked: boolean; // Contains a locked segment + reputation: number; } export interface VotableObject { votes: number; + reputation: number; } export interface VotableObjectWithWeight extends VotableObject { From d3210d4e274527ab902bf69cf65759fe1d0ac1e0 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 18:00:20 -0400 Subject: [PATCH 24/85] Move files to utils --- src/routes/getSkipSegments.ts | 6 +++--- src/routes/postSkipSegments.ts | 6 +++--- src/routes/voteOnSponsorTime.ts | 2 +- src/{middleware => utils}/queryCacher.ts | 0 src/{middleware => utils}/redisKeys.ts | 2 +- src/{middleware => utils}/reputation.ts | 0 6 files changed, 8 insertions(+), 8 deletions(-) rename src/{middleware => utils}/queryCacher.ts (100%) rename src/{middleware => utils}/redisKeys.ts (94%) rename src/{middleware => utils}/reputation.ts (100%) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 0e5ce19..5e15b1a 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -1,15 +1,15 @@ import { Request, Response } from 'express'; import { config } from '../config'; import { db, privateDB } from '../databases/databases'; -import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys'; import { SBRecord } from '../types/lib.model'; import { Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model"; import { getCategoryActionType } from '../utils/categoryInfo'; import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; -import { QueryCacher } from '../middleware/queryCacher' -import { getReputation } from '../middleware/reputation'; +import { QueryCacher } from '../utils/queryCacher' +import { getReputation } from '../utils/reputation'; async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise { diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 9c28964..a5137f9 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -11,13 +11,13 @@ import {getFormattedTime} from '../utils/getFormattedTime'; import {isUserTrustworthy} from '../utils/isUserTrustworthy'; import {dispatchEvent} from '../utils/webhookUtils'; import {Request, Response} from 'express'; -import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; +import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys'; import redis from '../utils/redis'; import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; import { deleteLockCategories } from './deleteLockCategories'; import { getCategoryActionType } from '../utils/categoryInfo'; -import { QueryCacher } from '../middleware/queryCacher'; -import { getReputation } from '../middleware/reputation'; +import { QueryCacher } from '../utils/queryCacher'; +import { getReputation } from '../utils/reputation'; interface APIVideoInfo { err: string | boolean, diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 39ebfd8..25a331f 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -12,7 +12,7 @@ import {config} from '../config'; import { UserID } from '../types/user.model'; import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model'; import { getCategoryActionType } from '../utils/categoryInfo'; -import { QueryCacher } from '../middleware/queryCacher'; +import { QueryCacher } from '../utils/queryCacher'; const voteTypes = { normal: 0, diff --git a/src/middleware/queryCacher.ts b/src/utils/queryCacher.ts similarity index 100% rename from src/middleware/queryCacher.ts rename to src/utils/queryCacher.ts diff --git a/src/middleware/redisKeys.ts b/src/utils/redisKeys.ts similarity index 94% rename from src/middleware/redisKeys.ts rename to src/utils/redisKeys.ts index 1a0491c..fde37f3 100644 --- a/src/middleware/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -1,6 +1,6 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model"; import { UserID } from "../types/user.model"; -import { Logger } from "../utils/logger"; +import { Logger } from "./logger"; export function skipSegmentsKey(videoID: VideoID, service: Service): string { return "segments." + service + ".videoID." + videoID; diff --git a/src/middleware/reputation.ts b/src/utils/reputation.ts similarity index 100% rename from src/middleware/reputation.ts rename to src/utils/reputation.ts From eb2ffff7801e37a7177cbf7a0323cfa234e69f8a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 18:51:23 -0400 Subject: [PATCH 25/85] Add tests for reputation --- src/utils/reputation.ts | 4 +- test/cases/reputation.ts | 79 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 test/cases/reputation.ts diff --git a/src/utils/reputation.ts b/src/utils/reputation.ts index 31672a4..d01de29 100644 --- a/src/utils/reputation.ts +++ b/src/utils/reputation.ts @@ -11,7 +11,7 @@ interface ReputationDBResult { } export async function getReputation(userID: UserID): Promise { - const pastDate = Date.now() - 1000 * 1000 * 60 * 60 * 24 * 45; // 45 days ago + const pastDate = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago const fetchFromDB = () => db.prepare("get", `SELECT COUNT(*) AS "totalSubmissions", SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions", @@ -20,7 +20,7 @@ export async function getReputation(userID: UserID): Promise { FROM "sponsorTimes" WHERE "userID" = ?`, [pastDate, userID]) as Promise; const result = await QueryCacher.get(fetchFromDB, reputationKey(userID)); - + // Grace period if (result.totalSubmissions < 5) { return 0; diff --git a/test/cases/reputation.ts b/test/cases/reputation.ts new file mode 100644 index 0000000..5b0023e --- /dev/null +++ b/test/cases/reputation.ts @@ -0,0 +1,79 @@ +import assert from 'assert'; +import { db } from '../../src/databases/databases'; +import { UserID } from '../../src/types/user.model'; +import { getHash } from '../../src/utils/getHash'; +import { getReputation } from '../../src/utils/reputation'; + +const userIDLowSubmissions = "reputation-lowsubmissions" as UserID; +const userIDHighDownvotes = "reputation-highdownvotes" as UserID; +const userIDNewSubmissions = "reputation-newsubmissions" as UserID; +const userIDLowSum = "reputation-lowsum" as UserID; +const userIDHighRep = "reputation-highrep" as UserID; + +describe('reputation', () => { + before(async () => { + const videoID = "reputation-videoID"; + + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-0', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-1', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 100, 0, 'reputation-0-uuid-2', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-1-uuid-0', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-1', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-2', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-3', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-4', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-uuid-5', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-6', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-7', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-0', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-1', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-2', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-3', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-4', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-2-uuid-5', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-6', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-7', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-3-uuid-0', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-1', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-2', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-3', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-4', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-3-uuid-5', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-6', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-7', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-0', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-1', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-2', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-3', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-4', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-4-uuid-5', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-6', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-7', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + }); + + it("user in grace period", async () => { + assert.strictEqual(await getReputation(getHash(userIDLowSubmissions)), 0); + }); + + it("user with high downvote ratio", async () => { + assert.strictEqual(await getReputation(getHash(userIDHighDownvotes)), -0.9642857142857144); + }); + + it("user with mostly new submissions", async () => { + assert.strictEqual(await getReputation(getHash(userIDNewSubmissions)), 0); + }); + + it("user with not enough vote sum", async () => { + assert.strictEqual(await getReputation(getHash(userIDLowSum)), 0); + }); + + it("user with high reputation", async () => { + assert.strictEqual(await getReputation(getHash(userIDHighRep)), 1.6666666666666665); + }); + +}); \ No newline at end of file From 52b201ff87ecd663cb136e682a966d49aa4f8747 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 21:43:23 -0400 Subject: [PATCH 26/85] Change to vary boost by votes --- src/routes/getSkipSegments.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 5e15b1a..04600dd 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -169,9 +169,11 @@ function getWeightedRandomChoice(choices: T[], amountOf //assign a weight to each choice let totalWeight = 0; let choicesWithWeights: TWithWeight[] = choices.map(choice => { + const boost = Math.min(choice.reputation, Math.max(0, choice.votes * 2)); + //The 3 makes -2 the minimum votes before being ignored completely //this can be changed if this system increases in popularity. - const weight = Math.exp((choice.votes + 3 + choice.reputation)); + const weight = Math.exp(choice.votes * Math.min(1, choice.reputation + 1) + 3 + boost); totalWeight += weight; return {...choice, weight}; From 30d0cb759029f7f5925017dbdd41bb60523714cb Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 23:19:39 -0400 Subject: [PATCH 27/85] Don't break with old cached data --- src/routes/getSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 04600dd..8b8db40 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -224,7 +224,7 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise 0) { currentGroup.reputation += segment.reputation; } From 994dba86f67cca32c31e6765fe2778a9a3edb643 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 23:30:47 -0400 Subject: [PATCH 28/85] Don't get reputation every time --- src/routes/getSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 8b8db40..a7958bc 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -224,7 +224,7 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise 0) { currentGroup.reputation += segment.reputation; } From d7f352d699fbd125fc41f8d1c0c74e9bc92b70d4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 23:36:16 -0400 Subject: [PATCH 29/85] Revert "Don't get reputation every time" This reverts commit 994dba86f67cca32c31e6765fe2778a9a3edb643. --- src/routes/getSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index a7958bc..8b8db40 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -224,7 +224,7 @@ async function chooseSegments(segments: DBSegment[], max: number): Promise 0) { currentGroup.reputation += segment.reputation; } From 300ee0183ed200b13a1020a179d48a107061dc3c Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 24 May 2021 10:51:32 -0400 Subject: [PATCH 30/85] Add a max initial boost --- src/routes/getSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 8b8db40..30c965b 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -169,7 +169,7 @@ function getWeightedRandomChoice(choices: T[], amountOf //assign a weight to each choice let totalWeight = 0; let choicesWithWeights: TWithWeight[] = choices.map(choice => { - const boost = Math.min(choice.reputation, Math.max(0, choice.votes * 2)); + const boost = Math.min(choice.reputation, 4); //The 3 makes -2 the minimum votes before being ignored completely //this can be changed if this system increases in popularity. From 09fc3ca882572cea2b81b86c596841b456cd93d9 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 24 May 2021 12:43:06 -0400 Subject: [PATCH 31/85] Raise reputation cap and don't count autovote submissions --- src/utils/reputation.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/reputation.ts b/src/utils/reputation.ts index d01de29..19d22d8 100644 --- a/src/utils/reputation.ts +++ b/src/utils/reputation.ts @@ -12,11 +12,12 @@ interface ReputationDBResult { export async function getReputation(userID: UserID): Promise { const pastDate = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago + // 1596240000000 is August 1st 2020, a little after auto upvote was disabled const fetchFromDB = () => db.prepare("get", `SELECT COUNT(*) AS "totalSubmissions", SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions", - SUM(CASE WHEN "votes" > 0 THEN "votes" ELSE 0 END) AS "upvotedSum", - SUM(CASE WHEN "timeSubmitted" < ? AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions" + SUM(CASE WHEN "votes" > 0 AND "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "upvotedSum", + SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions" FROM "sponsorTimes" WHERE "userID" = ?`, [pastDate, userID]) as Promise; const result = await QueryCacher.get(fetchFromDB, reputationKey(userID)); @@ -35,7 +36,7 @@ export async function getReputation(userID: UserID): Promise { return 0; } - return convertRange(Math.min(result.upvotedSum, 50), 5, 50, 0, 15); + return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 15); } function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number { From a732159a3addd1353aebeeef0f9d9752b518d587 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 24 May 2021 12:46:39 -0400 Subject: [PATCH 32/85] Fix comment in sql upgrade file --- databases/_upgrade_sponsorTimes_12.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databases/_upgrade_sponsorTimes_12.sql b/databases/_upgrade_sponsorTimes_12.sql index aeace54..ef4b2d0 100644 --- a/databases/_upgrade_sponsorTimes_12.sql +++ b/databases/_upgrade_sponsorTimes_12.sql @@ -1,6 +1,6 @@ BEGIN TRANSACTION; -/* Add Service field */ +/* Add reputation field */ CREATE TABLE "sqlb_temp_table_12" ( "videoID" TEXT NOT NULL, "startTime" REAL NOT NULL, From 676fc8ea08bdb564b3cabc6d46c8acf64fb53f33 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 24 May 2021 15:56:03 -0400 Subject: [PATCH 33/85] Add reputation to user info --- src/routes/getUserInfo.ts | 9 +++++---- test/cases/getUserInfo.ts | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index 636d2ad..8ea8dda 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -4,6 +4,7 @@ import {isUserVIP} from '../utils/isUserVIP'; import {Request, Response} from 'express'; import {Logger} from '../utils/logger'; import { HashedUserID, UserID } from '../types/user.model'; +import { getReputation } from '../utils/reputation'; async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> { try { @@ -60,16 +61,15 @@ async function dbGetWarningsForUser(userID: HashedUserID): Promise { } export async function getUserInfo(req: Request, res: Response) { - let userID = req.query.userID as UserID; + const userID = req.query.userID as UserID; + const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID; - if (userID == undefined) { + if (hashedUserID == undefined) { //invalid request res.status(400).send('Parameters are not valid'); return; } - //hash the userID - const hashedUserID: HashedUserID = getHash(userID); const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID); if (segmentsSummary) { @@ -80,6 +80,7 @@ export async function getUserInfo(req: Request, res: Response) { segmentCount: segmentsSummary.segmentCount, viewCount: await dbGetViewsForUser(hashedUserID), warnings: await dbGetWarningsForUser(hashedUserID), + reputation: await getReputation(hashedUserID), vip: await isUserVIP(hashedUserID), }); } else { diff --git a/test/cases/getUserInfo.ts b/test/cases/getUserInfo.ts index 0e369e4..9b4073a 100644 --- a/test/cases/getUserInfo.ts +++ b/test/cases/getUserInfo.ts @@ -41,7 +41,7 @@ describe('getUserInfo', () => { .catch(err => done('couldn\'t call endpoint')); }); - it('Should done(info', (done: Done) => { + it('Should be able to get user info', (done: Done) => { fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_user_01') .then(async res => { if (res.status !== 200) { @@ -56,6 +56,8 @@ describe('getUserInfo', () => { done('Returned incorrect viewCount "' + data.viewCount + '"'); } else if (data.segmentCount !== 3) { done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } else if (Math.abs(data.reputation - -0.928) > 0.001) { + done('Returned incorrect reputation "' + data.reputation + '"'); } else { done(); // pass } @@ -78,6 +80,21 @@ describe('getUserInfo', () => { .catch(err => ("couldn't call endpoint")); }); + it('Should get warning data with public ID', async () => { + try { + const res = await fetch(getbaseURL() + '/api/getUserInfo?userID=' + await getHash("getuserinfo_warning_0")) + + if (res.status !== 200) { + return 'non 200 (' + res.status + ')'; + } else { + const data = await res.json();; + if (data.warnings !== 1) return 'wrong number of warnings: ' + data.warnings + ', not ' + 1; + } + } catch (err) { + return "couldn't call endpoint"; + } + }); + it('Should get multiple warnings', (done: Done) => { fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_warning_1') .then(async res => { From 3f682d467dc85232dee6798816f9d3a3a5a48ab4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 24 May 2021 16:00:45 -0400 Subject: [PATCH 34/85] Fix reputation unit tests --- test/cases/reputation.ts | 76 ++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/test/cases/reputation.ts b/test/cases/reputation.ts index 5b0023e..629277e 100644 --- a/test/cases/reputation.ts +++ b/test/cases/reputation.ts @@ -8,6 +8,7 @@ const userIDLowSubmissions = "reputation-lowsubmissions" as UserID; const userIDHighDownvotes = "reputation-highdownvotes" as UserID; const userIDNewSubmissions = "reputation-newsubmissions" as UserID; const userIDLowSum = "reputation-lowsum" as UserID; +const userIDHighRepBeforeManualVote = "reputation-oldhighrep" as UserID; const userIDHighRep = "reputation-highrep" as UserID; describe('reputation', () => { @@ -15,45 +16,54 @@ describe('reputation', () => { const videoID = "reputation-videoID"; let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES'; - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-0', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-1', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 100, 0, 'reputation-0-uuid-2', '${getHash(userIDLowSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-0', '${getHash(userIDLowSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-0-uuid-1', '${getHash(userIDLowSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 100, 0, 'reputation-0-uuid-2', '${getHash(userIDLowSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-1-uuid-0', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-1', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-2', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-3', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-4', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-uuid-5', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-6', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-7', '${getHash(userIDHighDownvotes)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-1-uuid-0', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-1', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-2', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-3', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -2, 0, 'reputation-1-uuid-4', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-uuid-5', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-6', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-7', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-0', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-1', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-2', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-3', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-4', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-2-uuid-5', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-6', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-7', '${getHash(userIDNewSubmissions)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-2-uuid-5', '${getHash(userIDNewSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-6', '${getHash(userIDNewSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-2-uuid-7', '${getHash(userIDNewSubmissions)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-3-uuid-0', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-1', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-2', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-3', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-4', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-3-uuid-5', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-6', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-7', '${getHash(userIDLowSum)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-3-uuid-0', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-1', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-2', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-3', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 1, 0, 'reputation-3-uuid-4', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-3-uuid-5', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-6', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-3-uuid-7', '${getHash(userIDLowSum)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-0', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-1', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-2', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-3', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-4', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-4-uuid-5', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-6', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); - await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-7', '${getHash(userIDHighRep)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-0', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-1', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-2', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-3', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-4-uuid-4', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-4-uuid-5', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-6', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-4-uuid-7', '${getHash(userIDHighRepBeforeManualVote)}', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-0', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-1', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-2', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-3', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-5-uuid-4', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-5-uuid-5', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-5-uuid-6', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-5-uuid-7', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); }); it("user in grace period", async () => { @@ -73,7 +83,11 @@ describe('reputation', () => { }); it("user with high reputation", async () => { - assert.strictEqual(await getReputation(getHash(userIDHighRep)), 1.6666666666666665); + assert.strictEqual(await getReputation(getHash(userIDHighRepBeforeManualVote)), 0); + }); + + it("user with high reputation", async () => { + assert.strictEqual(await getReputation(getHash(userIDHighRep)), 0.5172413793103449); }); }); \ No newline at end of file From 6a58a087815acb297593d00885e007ec080382ab Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 24 May 2021 16:04:32 -0400 Subject: [PATCH 35/85] Rename user info endpoint --- src/app.ts | 1 + test/cases/getUserInfo.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/app.ts b/src/app.ts index 4530fa2..a05a7cc 100644 --- a/src/app.ts +++ b/src/app.ts @@ -112,6 +112,7 @@ function setupRoutes(app: Express) { app.get('/api/getTotalStats', getTotalStats); app.get('/api/getUserInfo', getUserInfo); + app.get('/api/userInfo', getUserInfo); //send out a formatted time saved total app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); diff --git a/test/cases/getUserInfo.ts b/test/cases/getUserInfo.ts index 9b4073a..c809a49 100644 --- a/test/cases/getUserInfo.ts +++ b/test/cases/getUserInfo.ts @@ -24,7 +24,7 @@ describe('getUserInfo', () => { }); it('Should be able to get a 200', (done: Done) => { - fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_user_01') + fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_user_01') .then(res => { if (res.status !== 200) done('non 200 (' + res.status + ')'); else done(); // pass @@ -33,7 +33,7 @@ describe('getUserInfo', () => { }); it('Should be able to get a 400 (No userID parameter)', (done: Done) => { - fetch(getbaseURL() + '/api/getUserInfo') + fetch(getbaseURL() + '/api/userInfo') .then(res => { if (res.status !== 400) done('non 400 (' + res.status + ')'); else done(); // pass @@ -42,7 +42,7 @@ describe('getUserInfo', () => { }); it('Should be able to get user info', (done: Done) => { - fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_user_01') + fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_user_01') .then(async res => { if (res.status !== 200) { done("non 200"); @@ -67,7 +67,7 @@ describe('getUserInfo', () => { }); it('Should get warning data', (done: Done) => { - fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_warning_0') + fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_warning_0') .then(async res => { if (res.status !== 200) { done('non 200 (' + res.status + ')'); @@ -82,7 +82,7 @@ describe('getUserInfo', () => { it('Should get warning data with public ID', async () => { try { - const res = await fetch(getbaseURL() + '/api/getUserInfo?userID=' + await getHash("getuserinfo_warning_0")) + const res = await fetch(getbaseURL() + '/api/userInfo?userID=' + await getHash("getuserinfo_warning_0")) if (res.status !== 200) { return 'non 200 (' + res.status + ')'; @@ -96,7 +96,7 @@ describe('getUserInfo', () => { }); it('Should get multiple warnings', (done: Done) => { - fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_warning_1') + fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_warning_1') .then(async res => { if (res.status !== 200) { done('non 200 (' + res.status + ')'); @@ -110,7 +110,7 @@ describe('getUserInfo', () => { }); it('Should not get warnings if noe', (done: Done) => { - fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_warning_2') + fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_warning_2') .then(async res => { if (res.status !== 200) { done('non 200 (' + res.status + ')'); @@ -124,7 +124,7 @@ describe('getUserInfo', () => { }); it('Should done(userID for userName (No userName set)', (done: Done) => { - fetch(getbaseURL() + '/api/getUserInfo?userID=getuserinfo_user_02') + fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_user_02') .then(async res => { if (res.status !== 200) { done('non 200 (' + res.status + ')'); From ec51ff835a5e5f6ababb41b6deba03c7655f7604 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sat, 29 May 2021 20:48:41 -0400 Subject: [PATCH 36/85] Consider locked segments when calculating reputation --- src/utils/reputation.ts | 4 +++- test/cases/reputation.ts | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/utils/reputation.ts b/src/utils/reputation.ts index 19d22d8..0ca1c78 100644 --- a/src/utils/reputation.ts +++ b/src/utils/reputation.ts @@ -7,6 +7,7 @@ interface ReputationDBResult { totalSubmissions: number, downvotedSubmissions: number, upvotedSum: number, + lockedSum: number, oldUpvotedSubmissions: number } @@ -17,6 +18,7 @@ export async function getReputation(userID: UserID): Promise { `SELECT COUNT(*) AS "totalSubmissions", SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions", SUM(CASE WHEN "votes" > 0 AND "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "upvotedSum", + SUM(locked) AS "lockedSum", SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions" FROM "sponsorTimes" WHERE "userID" = ?`, [pastDate, userID]) as Promise; @@ -36,7 +38,7 @@ export async function getReputation(userID: UserID): Promise { return 0; } - return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 15); + return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum, 50), 0, 50, 0, 20); } function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number { diff --git a/test/cases/reputation.ts b/test/cases/reputation.ts index 629277e..7fe9226 100644 --- a/test/cases/reputation.ts +++ b/test/cases/reputation.ts @@ -10,6 +10,7 @@ const userIDNewSubmissions = "reputation-newsubmissions" as UserID; const userIDLowSum = "reputation-lowsum" as UserID; const userIDHighRepBeforeManualVote = "reputation-oldhighrep" as UserID; const userIDHighRep = "reputation-highrep" as UserID; +const userIDHighRepAndLocked = "reputation-highlockedrep" as UserID; describe('reputation', () => { before(async () => { @@ -64,6 +65,15 @@ describe('reputation', () => { await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-5-uuid-5', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-5-uuid-6', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-5-uuid-7', '${getHash(userIDHighRep)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-0', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-1', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-2', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 1, 'reputation-6-uuid-3', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-6-uuid-4', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-6-uuid-5', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-6-uuid-6', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-6-uuid-7', '${getHash(userIDHighRepAndLocked)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); }); it("user in grace period", async () => { @@ -82,12 +92,16 @@ describe('reputation', () => { assert.strictEqual(await getReputation(getHash(userIDLowSum)), 0); }); - it("user with high reputation", async () => { + it("user with lots of old votes (before autovote was disabled) ", async () => { assert.strictEqual(await getReputation(getHash(userIDHighRepBeforeManualVote)), 0); }); it("user with high reputation", async () => { - assert.strictEqual(await getReputation(getHash(userIDHighRep)), 0.5172413793103449); + assert.strictEqual(await getReputation(getHash(userIDHighRep)), 0.24137931034482757); + }); + + it("user with high reputation and locked segments", async () => { + assert.strictEqual(await getReputation(getHash(userIDHighRepAndLocked)), 1.8413793103448277); }); }); \ No newline at end of file From f20506bf4365473e9fa6a05ea9445aaf73f4eda3 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 1 Jun 2021 16:14:21 -0400 Subject: [PATCH 37/85] Add back youtube api error handling --- src/utils/youtubeApi.ts | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 6515996..9d434be 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -27,28 +27,32 @@ export class YouTubeAPI { } } - const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({ - part, - id: videoID, - }, (ytErr: boolean | string, { data }: any) => resolve({ytErr, data}))); + try { + const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({ + part, + id: videoID, + }, (ytErr: boolean | string, { data }: any) => resolve({ytErr, data}))); - if (!ytErr) { - // Only set cache if data returned - if (data.items.length > 0) { - const { err: setErr } = await redis.setAsync(redisKey, JSON.stringify(data)); + if (!ytErr) { + // Only set cache if data returned + if (data.items.length > 0) { + const { err: setErr } = await redis.setAsync(redisKey, JSON.stringify(data)); - if (setErr) { - Logger.warn(setErr.message); + if (setErr) { + Logger.warn(setErr.message); + } else { + Logger.debug("redis: video information cache set for: " + videoID); + } + + return { err: false, data }; // don't fail } else { - Logger.debug("redis: video information cache set for: " + videoID); + return { err: false, data }; // don't fail } - - return { err: false, data }; // don't fail } else { - return { err: false, data }; // don't fail + return { err: ytErr, data }; } - } else { - return { err: ytErr, data }; - } + } catch (err) { + return {err, data: null} + } } } From 63c8f877768d8e7119bb9880f6bc6d88fe8f7ef4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 1 Jun 2021 16:18:41 -0400 Subject: [PATCH 38/85] Don't deconstruct --- src/utils/youtubeApi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 9d434be..d02665b 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -31,7 +31,7 @@ export class YouTubeAPI { const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({ part, id: videoID, - }, (ytErr: boolean | string, { data }: any) => resolve({ytErr, data}))); + }, (ytErr: boolean | string, result: any) => resolve({ytErr, data: result?.data}))); if (!ytErr) { // Only set cache if data returned From 2453c45b063bb23374f02af83d94500fd28c46c7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 1 Jun 2021 22:20:42 -0400 Subject: [PATCH 39/85] Don't use undefined lockedSum from cache --- src/utils/reputation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/reputation.ts b/src/utils/reputation.ts index 0ca1c78..d40857c 100644 --- a/src/utils/reputation.ts +++ b/src/utils/reputation.ts @@ -38,7 +38,7 @@ export async function getReputation(userID: UserID): Promise { return 0; } - return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum, 50), 0, 50, 0, 20); + return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 20); } function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number { From c1609a826ab7e1280faf3f95ef4fb638a83910fa Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 2 Jun 2021 19:17:29 -0400 Subject: [PATCH 40/85] Don't think duration changed when API fails --- src/routes/postSkipSegments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index a5137f9..cf09762 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -368,7 +368,7 @@ export async function postSkipSegments(req: Request, res: Response) { AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as {videoDuration: VideoDuration, UUID: SegmentUUID}[]; // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden - const videoDurationChanged = (videoDuration: number) => previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); + const videoDurationChanged = (videoDuration: number) => videoDuration != 0 && previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); let apiVideoInfo: APIVideoInfo = null; if (service == Service.YouTube) { From 0904036009bfbd9b89a6a23ff4fb7632c52dbb3a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 2 Jun 2021 22:34:38 -0400 Subject: [PATCH 41/85] Use newleaf instead of YouTube API --- README.MD | 4 -- config.json.example | 2 +- docker/docker-compose.yml | 12 +++- docker/newleaf/configuration.py | 17 +++++ package.json | 3 +- src/config.ts | 2 +- src/routes/postSkipSegments.ts | 36 ++++------- src/routes/voteOnSponsorTime.ts | 16 ++--- src/types/config.model.ts | 2 +- src/types/youtubeApi.model.ts | 111 ++++++++++++++++++++++++++++++++ src/utils/youtubeApi.ts | 51 +++++++-------- test.json | 2 +- test/cases/postSkipSegments.ts | 49 ++++++-------- test/youtubeMock.ts | 74 +++++++-------------- 14 files changed, 227 insertions(+), 154 deletions(-) create mode 100644 docker/newleaf/configuration.py create mode 100644 src/types/youtubeApi.model.ts diff --git a/README.MD b/README.MD index e309e54..59bac07 100644 --- a/README.MD +++ b/README.MD @@ -32,10 +32,6 @@ Run the server with `npm start`. If you want to make changes, run `npm run dev` to automatically reload the server and run tests whenever a file is saved. -# Privacy Policy - -If you set the `youtubeAPIKey` option in `config.json`, you must follow [Google's Privacy Policy](https://policies.google.com/privacy) and [YouTube's Terms of Service](https://www.youtube.com/t/terms) - # API Docs Available [here](https://github.com/ajayyy/SponsorBlock/wiki/API-Docs) diff --git a/config.json.example b/config.json.example index 0773e89..dbd009e 100644 --- a/config.json.example +++ b/config.json.example @@ -4,7 +4,7 @@ "port": 80, "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]", "adminUserID": "[the hashed id of the user who can perform admin actions]", - "youtubeAPIKey": null, //get this from Google Cloud Platform [optional] + "newLeafURL": "http://localhost:3241", "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d4987b2..197e65a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -10,7 +10,7 @@ services: - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports ports: - 5432:5432 - restart: always + restart: unless-stopped redis: container_name: redis image: redis @@ -19,7 +19,15 @@ services: - ./redis/redis.conf:/usr/local/etc/redis/redis.conf ports: - 32773:6379 - restart: always + restart: unless-stopped + newleaf: + image: abeltramo/newleaf:latest + container_name: newleaf + restart: unless-stopped + ports: + - 3241:3000 + volumes: + - ./newleaf/configuration.py:/workdir/configuration.py volumes: database-data: diff --git a/docker/newleaf/configuration.py b/docker/newleaf/configuration.py new file mode 100644 index 0000000..d0b014c --- /dev/null +++ b/docker/newleaf/configuration.py @@ -0,0 +1,17 @@ +# ============================== +# You MUST set these settings. +# ============================== + +# A URL that this site can be accessed on. Do not include a trailing slash. +website_origin = "http://newleaf:3000" + + +# ============================== +# These settings are optional. +# ============================== + +# The address of the interface to bind to. +#bind_host = "0.0.0.0" + +# The port to bind to. +#bind_port = 3000 \ No newline at end of file diff --git a/package.json b/package.json index 8e0d18b..90da4d7 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,7 @@ "pg": "^8.5.1", "redis": "^3.1.1", "sync-mysql": "^3.0.1", - "uuid": "^3.3.2", - "youtube-api": "^3.0.1" + "uuid": "^3.3.2" }, "devDependencies": { "@types/better-sqlite3": "^5.4.0", diff --git a/src/config.ts b/src/config.ts index a684ba7..2f6d774 100644 --- a/src/config.ts +++ b/src/config.ts @@ -44,7 +44,7 @@ addDefaults(config, { }, }, userCounterURL: null, - youtubeAPIKey: null, + newLeafURL: null, maxRewardTimePerSegmentInSeconds: 86400, postgres: null, dumpDatabase: { diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index cf09762..bae0692 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -1,7 +1,7 @@ import {config} from '../config'; import {Logger} from '../utils/logger'; import {db, privateDB} from '../databases/databases'; -import {YouTubeAPI} from '../utils/youtubeApi'; +import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi'; import {getSubmissionUUID} from '../utils/getSubmissionUUID'; import fetch from 'node-fetch'; import isoDurations, { end } from 'iso8601-duration'; @@ -18,16 +18,11 @@ import { deleteLockCategories } from './deleteLockCategories'; import { getCategoryActionType } from '../utils/categoryInfo'; import { QueryCacher } from '../utils/queryCacher'; import { getReputation } from '../utils/reputation'; +import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model'; -interface APIVideoInfo { - err: string | boolean, - data?: any -} - -async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { +async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); const userName = row !== undefined ? row.userName : null; - const video = youtubeData.items[0]; let scopeName = "submissions.other"; if (submissionCount <= 1) { @@ -37,8 +32,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st dispatchEvent(scopeName, { "video": { "id": videoID, - "title": video.snippet.title, - "thumbnail": video.snippet.thumbnails.maxres ? video.snippet.thumbnails.maxres : null, + "title": youtubeData?.title, + "thumbnail": getMaxResThumbnail(youtubeData) || null, "url": "https://www.youtube.com/watch?v=" + videoID, }, "submission": { @@ -76,7 +71,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: method: 'POST', body: JSON.stringify({ "embeds": [{ - "title": data.items[0].snippet.title, + "title": data?.title, "url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2), "description": "Submission ID: " + UUID + "\n\nTimestamp: " + @@ -87,7 +82,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: "name": userID, }, "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + "url": getMaxResThumbnail(data) || "", }, }], }), @@ -177,10 +172,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, const {err, data} = apiVideoInfo; if (err) return false; - // Check to see if video exists - if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID; - - const duration = getYouTubeVideoDuration(apiVideoInfo); + const duration = apiVideoInfo?.data?.lengthSeconds; const segments = submission.segments; let nbString = ""; for (let i = 0; i < segments.length; i++) { @@ -220,8 +212,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, return a[0] - b[0] || a[1] - b[1]; })); - let videoDuration = data.items[0].contentDetails.duration; - videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration)); + const videoDuration = data.lengthSeconds; if (videoDuration != 0) { let allSegmentDuration = 0; //sum all segment times together @@ -273,13 +264,8 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, } } -function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration { - const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration; - return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null; -} - async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { - if (config.youtubeAPIKey !== null) { + if (config.newLeafURL !== null) { return YouTubeAPI.listVideos(videoID, ignoreCache); } else { return null; @@ -375,7 +361,7 @@ export async function postSkipSegments(req: Request, res: Response) { // Don't use cache if we don't know the video duraton, or the client claims that it has changed apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDuration || videoDurationChanged(videoDuration)); } - const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo); + const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration; if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) { // If api duration is far off, take that one instead (it is only precise to seconds, not millis) videoDuration = apiVideoDuration || 0 as VideoDuration; diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 25a331f..2ae0a4c 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -2,7 +2,7 @@ import {Request, Response} from 'express'; import {Logger} from '../utils/logger'; import {isUserVIP} from '../utils/isUserVIP'; import fetch from 'node-fetch'; -import {YouTubeAPI} from '../utils/youtubeApi'; +import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi'; import {db, privateDB} from '../databases/databases'; import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from '../utils/webhookUtils'; import {getFormattedTime} from '../utils/getFormattedTime'; @@ -57,11 +57,11 @@ async function sendWebhooks(voteData: VoteData) { webhookURL = config.discordCompletelyIncorrectReportWebhookURL; } - if (config.youtubeAPIKey !== null) { + if (config.newLeafURL !== null) { const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID); - if (err || data.items.length === 0) { - if (err) Logger.error(err.toString()); + if (err) { + Logger.error(err.toString()); return; } const isUpvote = voteData.incrementAmount > 0; @@ -72,9 +72,9 @@ async function sendWebhooks(voteData: VoteData) { }, "video": { "id": submissionInfoRow.videoID, - "title": data.items[0].snippet.title, + "title": data?.title, "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID, - "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + "thumbnail": getMaxResThumbnail(data) || null, }, "submission": { "UUID": voteData.UUID, @@ -103,7 +103,7 @@ async function sendWebhooks(voteData: VoteData) { method: 'POST', body: JSON.stringify({ "embeds": [{ - "title": data.items[0].snippet.title, + "title": data?.title, "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2), "description": "**" + voteData.row.votes + " Votes Prior | " + @@ -120,7 +120,7 @@ async function sendWebhooks(voteData: VoteData) { "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), }, "thumbnail": { - "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "", + "url": getMaxResThumbnail(data) || "", }, }], }), diff --git a/src/types/config.model.ts b/src/types/config.model.ts index 117c732..aa0a863 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -6,7 +6,7 @@ export interface SBSConfig { mockPort?: number; globalSalt: string; adminUserID: string; - youtubeAPIKey?: string; + newLeafURL?: string; discordReportChannelWebhookURL?: string; discordFirstTimeSubmissionsWebhookURL?: string; discordCompletelyIncorrectReportWebhookURL?: string; diff --git a/src/types/youtubeApi.model.ts b/src/types/youtubeApi.model.ts new file mode 100644 index 0000000..5e869b3 --- /dev/null +++ b/src/types/youtubeApi.model.ts @@ -0,0 +1,111 @@ +export interface APIVideoData { + "title": string, + "videoId": string, + "videoThumbnails": [ + { + "quality": string, + "url": string, + second__originalUrl: string, + "width": number, + "height": number + } + ], + + "description": string, + "descriptionHtml": string, + "published": number, + "publishedText": string, + + "keywords": string[], + "viewCount": number, + "likeCount": number, + "dislikeCount": number, + + "paid": boolean, + "premium": boolean, + "isFamilyFriendly": boolean, + "allowedRegions": string[], + "genre": string, + "genreUrl": string, + + "author": string, + "authorId": string, + "authorUrl": string, + "authorThumbnails": [ + { + "url": string, + "width": number, + "height": number + } + ], + + "subCountText": string, + "lengthSeconds": number, + "allowRatings": boolean, + "rating": number, + "isListed": boolean, + "liveNow": boolean, + "isUpcoming": boolean, + "premiereTimestamp"?: number, + + "hlsUrl"?: string, + "adaptiveFormats": [ + { + "index": string, + "bitrate": string, + "init": string, + "url": string, + "itag": string, + "type": string, + "clen": string, + "lmt": string, + "projectionType": number, + "container": string, + "encoding": string, + "qualityLabel"?: string, + "resolution"?: string + } + ], + "formatStreams": [ + { + "url": string, + "itag": string, + "type": string, + "quality": string, + "container": string, + "encoding": string, + "qualityLabel": string, + "resolution": string, + "size": string + } + ], + "captions": [ + { + "label": string, + "languageCode": string, + "url": string + } + ], + "recommendedVideos": [ + { + "videoId": string, + "title": string, + "videoThumbnails": [ + { + "quality": string, + "url": string, + "width": number, + "height": number + } + ], + "author": string, + "lengthSeconds": number, + "viewCountText": string + } + ] +} + +export interface APIVideoInfo { + err: string | boolean, + data?: APIVideoData +} \ No newline at end of file diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index d02665b..95ef422 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -1,22 +1,16 @@ +import fetch from 'node-fetch'; import {config} from '../config'; import {Logger} from './logger'; import redis from './redis'; -// @ts-ignore -import _youTubeAPI from 'youtube-api'; - -_youTubeAPI.authenticate({ - type: "key", - key: config.youtubeAPIKey, -}); +import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model'; export class YouTubeAPI { - static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> { - const part = 'contentDetails,snippet'; + static async listVideos(videoID: string, ignoreCache = false): Promise { if (!videoID || videoID.length !== 11 || videoID.includes(".")) { return { err: "Invalid video ID" }; } - const redisKey = "youtube.video." + videoID; + const redisKey = "yt.newleaf.video." + videoID; if (!ignoreCache) { const {err, reply} = await redis.getAsync(redisKey); @@ -25,34 +19,37 @@ export class YouTubeAPI { return { err: err?.message, data: JSON.parse(reply) } } - } + } + + if (!config.newLeafURL) return {err: "NewLeaf URL not found", data: null}; try { - const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({ - part, - id: videoID, - }, (ytErr: boolean | string, result: any) => resolve({ytErr, data: result?.data}))); + const result = await fetch(config.newLeafURL + "/api/v1/videos/" + videoID, { method: "GET" }); - if (!ytErr) { - // Only set cache if data returned - if (data.items.length > 0) { - const { err: setErr } = await redis.setAsync(redisKey, JSON.stringify(data)); + if (result.ok) { + const data = await result.json(); + if (data.error) { + return { err: data.err, data: null }; + } - if (setErr) { - Logger.warn(setErr.message); + redis.setAsync(redisKey, JSON.stringify(data)).then((result) => { + if (result?.err) { + Logger.warn(result?.err.message); } else { Logger.debug("redis: video information cache set for: " + videoID); } - - return { err: false, data }; // don't fail - } else { - return { err: false, data }; // don't fail - } + }); + + return { err: false, data }; } else { - return { err: ytErr, data }; + return { err: result.statusText, data: null }; } } catch (err) { return {err, data: null} } } } + +export function getMaxResThumbnail(apiInfo: APIVideoData): string | void { + return apiInfo?.videoThumbnails?.find((elem) => elem.quality === "maxres")?.second__originalUrl; +} \ No newline at end of file diff --git a/test.json b/test.json index 30958ae..c3b1096 100644 --- a/test.json +++ b/test.json @@ -3,7 +3,7 @@ "mockPort": 8081, "globalSalt": "testSalt", "adminUserID": "testUserId", - "youtubeAPIKey": "", + "newLeafURL": "placeholder", "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index ce22506..ec5ccf3 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -118,7 +118,7 @@ describe('postSkipSegments', () => { .then(async res => { if (res.status === 200) { const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]); - if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) { + if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 4980) { done(); } else { done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); @@ -140,7 +140,7 @@ describe('postSkipSegments', () => { body: JSON.stringify({ userID: "test", videoID: "dQw4w9WgXZH", - videoDuration: 5010.20, + videoDuration: 4980.20, segments: [{ segment: [1, 10], category: "sponsor", @@ -150,7 +150,7 @@ describe('postSkipSegments', () => { .then(async res => { if (res.status === 200) { const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]); - if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) { + if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 4980.20) { done(); } else { done("Submitted times were not saved. Actual submission: " + JSON.stringify(row)); @@ -237,6 +237,21 @@ describe('postSkipSegments', () => { } }); + it('Should still not be allowed if youtube thinks duration is 0', (done: Done) => { + fetch(getbaseURL() + + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", { + method: 'POST', + }) + .then(async res => { + if (res.status === 403) done(); // pass + else { + const body = await res.text(); + done("non 403 status code: " + res.status + " (" + body + ")"); + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes", { @@ -666,34 +681,6 @@ describe('postSkipSegments', () => { .catch(err => done(err)); }); - it('Should be allowed if youtube thinks duration is 0', (done: Done) => { - fetch(getbaseURL() - + "/api/postVideoSponsorTimes?videoID=noDuration&startTime=30&endTime=10000&userID=testing", { - method: 'POST', - }) - .then(async res => { - if (res.status === 200) done(); // pass - else { - const body = await res.text(); - done("non 200 status code: " + res.status + " (" + body + ")"); - } - }) - .catch(err => done("Couldn't call endpoint")); - }); - - it('Should be rejected if not a valid videoID', (done: Done) => { - fetch(getbaseURL() - + "/api/postVideoSponsorTimes?videoID=knownWrongID&startTime=30&endTime=1000000&userID=testing") - .then(async res => { - if (res.status === 403) done(); // pass - else { - const body = await res.text(); - done("non 403 status code: " + res.status + " (" + body + ")"); - } - }) - .catch(err => done("Couldn't call endpoint")); - }); - it('Should return 400 for missing params (Params method)', (done: Done) => { fetch(getbaseURL() + "/api/postVideoSponsorTimes?startTime=9&endTime=10&userID=test", { diff --git a/test/youtubeMock.ts b/test/youtubeMock.ts index de7e17e..3e68369 100644 --- a/test/youtubeMock.ts +++ b/test/youtubeMock.ts @@ -1,28 +1,14 @@ -/* -YouTubeAPI.videos.list({ - part: "snippet", - id: videoID -}, function (err, data) {}); - */ - -// https://developers.google.com/youtube/v3/docs/videos - +import { APIVideoData, APIVideoInfo } from "../src/types/youtubeApi.model"; export class YouTubeApiMock { - static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> { + static async listVideos(videoID: string, ignoreCache = false): Promise { const obj = { id: videoID }; if (obj.id === "knownWrongID") { return { - err: null, - data: { - pageInfo: { - totalResults: 0, - }, - items: [], - } + err: "No video found" }; } @@ -30,49 +16,35 @@ export class YouTubeApiMock { return { err: null, data: { - pageInfo: { - totalResults: 1, - }, - items: [ + title: "Example Title", + lengthSeconds: 0, + videoThumbnails: [ { - contentDetails: { - duration: "PT0S", - }, - snippet: { - title: "Example Title", - thumbnails: { - maxres: { - url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", - }, - }, - }, + quality: "maxres", + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + second__originalUrl:"https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + width: 1280, + height: 720 }, - ], - } + ] + } as APIVideoData }; } else { return { err: null, data: { - pageInfo: { - totalResults: 1, - }, - items: [ + title: "Example Title", + lengthSeconds: 4980, + videoThumbnails: [ { - contentDetails: { - duration: "PT1H23M30S", - }, - snippet: { - title: "Example Title", - thumbnails: { - maxres: { - url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", - }, - }, - }, + quality: "maxres", + url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + second__originalUrl:"https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png", + width: 1280, + height: 720 }, - ], - } + ] + } as APIVideoData }; } } From e7337d3cb4e78f01fe5c47097dc1b874c9c8dfd0 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 2 Jun 2021 22:40:18 -0400 Subject: [PATCH 42/85] Add missing dependency --- package-lock.json | 507 +--------------------------------------------- package.json | 1 + 2 files changed, 3 insertions(+), 505 deletions(-) diff --git a/package-lock.json b/package-lock.json index 61b1181..4d3d560 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "license": "MIT", "dependencies": { + "abort-controller": "^3.0.0", "better-sqlite3": "^7.1.5", "dotenv": "^8.2.0", "express": "^4.17.1", @@ -19,8 +20,7 @@ "pg": "^8.5.1", "redis": "^3.1.1", "sync-mysql": "^3.0.1", - "uuid": "^3.3.2", - "youtube-api": "^3.0.1" + "uuid": "^3.3.2" }, "devDependencies": { "@types/better-sqlite3": "^5.4.0", @@ -294,33 +294,6 @@ "node": ">= 0.6" } }, - "node_modules/agent-base": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", - "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/agent-base/node_modules/debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - } - }, - "node_modules/agent-base/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, "node_modules/ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -456,14 +429,6 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "engines": { - "node": ">=8" - } - }, "node_modules/asap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz", @@ -698,11 +663,6 @@ "ieee754": "^1.1.13" } }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" - }, "node_modules/buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -1122,14 +1082,6 @@ "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", "dev": true }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1272,16 +1224,6 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "node_modules/fast-text-encoding": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz", - "integrity": "sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig==" - }, "node_modules/file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -1446,41 +1388,6 @@ "node": ">=0.10.0" } }, - "node_modules/gaxios": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.2.0.tgz", - "integrity": "sha512-+6WPeVzPvOshftpxJwRi2Ozez80tn/hdtOUag7+gajDHRJvAblKxTFSSMPtr2hmnLy7p0mvYz0rMXLBl8pSO7Q==", - "dependencies": { - "abort-controller": "^3.0.0", - "extend": "^3.0.2", - "https-proxy-agent": "^5.0.0", - "is-stream": "^2.0.0", - "node-fetch": "^2.3.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/gaxios/node_modules/is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "engines": { - "node": ">=8" - } - }, - "node_modules/gcp-metadata": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.0.tgz", - "integrity": "sha512-vQZD57cQkqIA6YPGXM/zc+PIZfNRFdukWGsGZ5+LcJzesi5xp6Gn7a02wRJi4eXPyArNMIYpPET4QMxGqtlk6Q==", - "dependencies": { - "gaxios": "^3.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1556,86 +1463,6 @@ "node": ">=4" } }, - "node_modules/google-auth-library": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.1.1.tgz", - "integrity": "sha512-0WfExOx3FrLYnY88RICQxvpaNzdwjz44OsHqHkIoAJfjY6Jck6CZRl1ASWadk+wbJ0LhkQ8rNY4zZebKml4Ghg==", - "dependencies": { - "arrify": "^2.0.0", - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "fast-text-encoding": "^1.0.0", - "gaxios": "^3.0.0", - "gcp-metadata": "^4.1.0", - "gtoken": "^5.0.4", - "jws": "^4.0.0", - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/google-auth-library/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/google-p12-pem": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz", - "integrity": "sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA==", - "dependencies": { - "node-forge": "^0.10.0" - }, - "bin": { - "gp12-pem": "build/src/bin/gp12-pem.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/googleapis": { - "version": "54.1.0", - "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-54.1.0.tgz", - "integrity": "sha512-xkBk3wRRYeJxEp9bVyAaYQM7qt3Eo3wBoznk+XnXbcSAjhl+M7icsUGb6VGNbZJfvpKAU7/tfhsed50pfs/jmA==", - "dependencies": { - "google-auth-library": "^6.0.0", - "googleapis-common": "^4.4.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/googleapis-common": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-4.4.1.tgz", - "integrity": "sha512-F1QcH8oU7TOuZex9p+XW7TeyLY0332NwBwJ3dZoN+51pXZXB5JjrKswrpgbhuREuIe8xAy8J1rlmFqxeP2mFfA==", - "dependencies": { - "extend": "^3.0.2", - "gaxios": "^3.2.0", - "google-auth-library": "^6.0.0", - "qs": "^6.7.0", - "url-template": "^2.0.8", - "uuid": "^8.0.0" - }, - "engines": { - "node": ">=10.10.0" - } - }, - "node_modules/googleapis-common/node_modules/uuid": { - "version": "8.3.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", - "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/got": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", @@ -1673,31 +1500,6 @@ "node": ">=4.x" } }, - "node_modules/gtoken": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.4.tgz", - "integrity": "sha512-U9wnSp4GZ7ov6zRdPuRHG4TuqEWqRRgT1gfXGNArhzBUn9byrPeH8uTmBWU/ZiWJJvTEmkjhDIC3mqHWdVi3xQ==", - "dependencies": { - "gaxios": "^3.0.0", - "google-p12-pem": "^3.0.3", - "jws": "^4.0.0", - "mime": "^2.2.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/gtoken/node_modules/mime": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", - "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -1741,34 +1543,6 @@ "node": ">= 0.6" } }, - "node_modules/https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -2021,39 +1795,12 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, "node_modules/just-extend": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz", "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==", "dev": true }, - "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "dependencies": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "dependencies": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, "node_modules/latest-version": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", @@ -2511,14 +2258,6 @@ "node": "4.x || >=6.0.0" } }, - "node_modules/node-forge": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", - "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/nodemon": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.2.tgz", @@ -3790,11 +3529,6 @@ "node": ">=0.10.0" } }, - "node_modules/url-template": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", - "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE=" - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -4076,14 +3810,6 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } - }, - "node_modules/youtube-api": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/youtube-api/-/youtube-api-3.0.1.tgz", - "integrity": "sha512-r3N8xmE46desGsnTRD1KRiYnfPKElgp1BDMfeCkpyTMrE1qkTb8DxtJrQjEJxFpCE4s6xIQZa/cnWtTbOoNlkA==", - "dependencies": { - "googleapis": "^54.1.0" - } } }, "dependencies": { @@ -4334,29 +4060,6 @@ "negotiator": "0.6.2" } }, - "agent-base": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", - "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", - "requires": { - "debug": "4" - }, - "dependencies": { - "debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -4478,11 +4181,6 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, - "arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" - }, "asap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz", @@ -4681,11 +4379,6 @@ "ieee754": "^1.1.13" } }, - "buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" - }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -5005,14 +4698,6 @@ "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", "dev": true }, - "ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "requires": { - "safe-buffer": "^5.0.1" - } - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -5133,16 +4818,6 @@ "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.1.3.tgz", "integrity": "sha512-TINcxve5510pXj4n9/1AMupkj3iWxl3JuZaWhCdYDlZeoCPqweGZrxbrlqTCFb1CT5wli7s8e2SH/Qz2c9GorA==" }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "fast-text-encoding": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz", - "integrity": "sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig==" - }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -5269,34 +4944,6 @@ } } }, - "gaxios": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.2.0.tgz", - "integrity": "sha512-+6WPeVzPvOshftpxJwRi2Ozez80tn/hdtOUag7+gajDHRJvAblKxTFSSMPtr2hmnLy7p0mvYz0rMXLBl8pSO7Q==", - "requires": { - "abort-controller": "^3.0.0", - "extend": "^3.0.2", - "https-proxy-agent": "^5.0.0", - "is-stream": "^2.0.0", - "node-fetch": "^2.3.0" - }, - "dependencies": { - "is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" - } - } - }, - "gcp-metadata": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.0.tgz", - "integrity": "sha512-vQZD57cQkqIA6YPGXM/zc+PIZfNRFdukWGsGZ5+LcJzesi5xp6Gn7a02wRJi4eXPyArNMIYpPET4QMxGqtlk6Q==", - "requires": { - "gaxios": "^3.0.0", - "json-bigint": "^1.0.0" - } - }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -5351,69 +4998,6 @@ "ini": "^1.3.4" } }, - "google-auth-library": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.1.1.tgz", - "integrity": "sha512-0WfExOx3FrLYnY88RICQxvpaNzdwjz44OsHqHkIoAJfjY6Jck6CZRl1ASWadk+wbJ0LhkQ8rNY4zZebKml4Ghg==", - "requires": { - "arrify": "^2.0.0", - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "fast-text-encoding": "^1.0.0", - "gaxios": "^3.0.0", - "gcp-metadata": "^4.1.0", - "gtoken": "^5.0.4", - "jws": "^4.0.0", - "lru-cache": "^6.0.0" - }, - "dependencies": { - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - } - } - }, - "google-p12-pem": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz", - "integrity": "sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA==", - "requires": { - "node-forge": "^0.10.0" - } - }, - "googleapis": { - "version": "54.1.0", - "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-54.1.0.tgz", - "integrity": "sha512-xkBk3wRRYeJxEp9bVyAaYQM7qt3Eo3wBoznk+XnXbcSAjhl+M7icsUGb6VGNbZJfvpKAU7/tfhsed50pfs/jmA==", - "requires": { - "google-auth-library": "^6.0.0", - "googleapis-common": "^4.4.0" - } - }, - "googleapis-common": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-4.4.1.tgz", - "integrity": "sha512-F1QcH8oU7TOuZex9p+XW7TeyLY0332NwBwJ3dZoN+51pXZXB5JjrKswrpgbhuREuIe8xAy8J1rlmFqxeP2mFfA==", - "requires": { - "extend": "^3.0.2", - "gaxios": "^3.2.0", - "google-auth-library": "^6.0.0", - "qs": "^6.7.0", - "url-template": "^2.0.8", - "uuid": "^8.0.0" - }, - "dependencies": { - "uuid": { - "version": "8.3.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", - "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" - } - } - }, "got": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", @@ -5445,24 +5029,6 @@ "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, - "gtoken": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.4.tgz", - "integrity": "sha512-U9wnSp4GZ7ov6zRdPuRHG4TuqEWqRRgT1gfXGNArhzBUn9byrPeH8uTmBWU/ZiWJJvTEmkjhDIC3mqHWdVi3xQ==", - "requires": { - "gaxios": "^3.0.0", - "google-p12-pem": "^3.0.3", - "jws": "^4.0.0", - "mime": "^2.2.0" - }, - "dependencies": { - "mime": { - "version": "2.4.6", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", - "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" - } - } - }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", @@ -5497,30 +5063,6 @@ "toidentifier": "1.0.0" } }, - "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", - "requires": { - "agent-base": "6", - "debug": "4" - }, - "dependencies": { - "debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -5702,39 +5244,12 @@ "argparse": "^2.0.1" } }, - "json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "requires": { - "bignumber.js": "^9.0.0" - } - }, "just-extend": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz", "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==", "dev": true }, - "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "requires": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, "latest-version": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz", @@ -6083,11 +5598,6 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, - "node-forge": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", - "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" - }, "nodemon": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.2.tgz", @@ -7102,11 +6612,6 @@ "prepend-http": "^1.0.1" } }, - "url-template": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", - "integrity": "sha1-/FZaPMy/93MMd19WQflVV5FDnyE=" - }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -7320,14 +6825,6 @@ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true - }, - "youtube-api": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/youtube-api/-/youtube-api-3.0.1.tgz", - "integrity": "sha512-r3N8xmE46desGsnTRD1KRiYnfPKElgp1BDMfeCkpyTMrE1qkTb8DxtJrQjEJxFpCE4s6xIQZa/cnWtTbOoNlkA==", - "requires": { - "googleapis": "^54.1.0" - } } } } diff --git a/package.json b/package.json index 90da4d7..214f242 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "author": "Ajay Ramachandran", "license": "MIT", "dependencies": { + "abort-controller": "^3.0.0", "better-sqlite3": "^7.1.5", "dotenv": "^8.2.0", "express": "^4.17.1", From 10fcc7885f7a31fcbeaa663665ea9bb03cf8e1bd Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 3 Jun 2021 00:50:05 -0400 Subject: [PATCH 43/85] Raise redis memory --- docker/redis/redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/redis/redis.conf b/docker/redis/redis.conf index 023731b..cd43e41 100644 --- a/docker/redis/redis.conf +++ b/docker/redis/redis.conf @@ -1,2 +1,2 @@ maxmemory-policy allkeys-lru -maxmemory 1000mb \ No newline at end of file +maxmemory 2000mb \ No newline at end of file From 1e5849f5040f1bac4e9a41c59488f2457593f90c Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 3 Jun 2021 11:29:55 -0400 Subject: [PATCH 44/85] Prevent failing on api errors --- src/routes/postSkipSegments.ts | 2 +- src/routes/voteOnSponsorTime.ts | 7 ++----- src/utils/youtubeApi.ts | 3 ++- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index bae0692..3da291a 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -212,7 +212,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, return a[0] - b[0] || a[1] - b[1]; })); - const videoDuration = data.lengthSeconds; + const videoDuration = data?.lengthSeconds; if (videoDuration != 0) { let allSegmentDuration = 0; //sum all segment times together diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 2ae0a4c..0e37732 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -59,11 +59,8 @@ async function sendWebhooks(voteData: VoteData) { if (config.newLeafURL !== null) { const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID); - - if (err) { - Logger.error(err.toString()); - return; - } + if (err) return; + const isUpvote = voteData.incrementAmount > 0; // Send custom webhooks dispatchEvent(isUpvote ? "vote.up" : "vote.down", { diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 95ef422..855a2d6 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -29,7 +29,8 @@ export class YouTubeAPI { if (result.ok) { const data = await result.json(); if (data.error) { - return { err: data.err, data: null }; + Logger.warn("CloudTube API Error: " + data.error) + return { err: data.error, data: null }; } redis.setAsync(redisKey, JSON.stringify(data)).then((result) => { From ec081cf0c5eef109c361834c9336ef5647673989 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 3 Jun 2021 11:38:21 -0400 Subject: [PATCH 45/85] Support multiple newleaf urls --- config.json.example | 2 +- src/config.ts | 2 +- src/routes/postSkipSegments.ts | 2 +- src/routes/voteOnSponsorTime.ts | 4 ++-- src/types/config.model.ts | 2 +- src/utils/youtubeApi.ts | 6 +++--- test.json | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config.json.example b/config.json.example index dbd009e..1934a98 100644 --- a/config.json.example +++ b/config.json.example @@ -4,7 +4,7 @@ "port": 80, "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]", "adminUserID": "[the hashed id of the user who can perform admin actions]", - "newLeafURL": "http://localhost:3241", + "newLeafURLs": ["http://localhost:3241"], "discordReportChannelWebhookURL": null, //URL from discord if you would like notifications when someone makes a report [optional] "discordFirstTimeSubmissionsWebhookURL": null, //URL from discord if you would like notifications when someone makes a first time submission [optional] "discordCompletelyIncorrectReportWebhookURL": null, //URL from discord if you would like notifications when someone reports a submission as completely incorrect [optional] diff --git a/src/config.ts b/src/config.ts index 2f6d774..34e5438 100644 --- a/src/config.ts +++ b/src/config.ts @@ -44,7 +44,7 @@ addDefaults(config, { }, }, userCounterURL: null, - newLeafURL: null, + newLeafURLs: null, maxRewardTimePerSegmentInSeconds: 86400, postgres: null, dumpDatabase: { diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 3da291a..b79b855 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -265,7 +265,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, } async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { - if (config.newLeafURL !== null) { + if (config.newLeafURLs !== null) { return YouTubeAPI.listVideos(videoID, ignoreCache); } else { return null; diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 0e37732..ce59172 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -57,10 +57,10 @@ async function sendWebhooks(voteData: VoteData) { webhookURL = config.discordCompletelyIncorrectReportWebhookURL; } - if (config.newLeafURL !== null) { + if (config.newLeafURLs !== null) { const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID); if (err) return; - + const isUpvote = voteData.incrementAmount > 0; // Send custom webhooks dispatchEvent(isUpvote ? "vote.up" : "vote.down", { diff --git a/src/types/config.model.ts b/src/types/config.model.ts index aa0a863..f5614d9 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -6,7 +6,7 @@ export interface SBSConfig { mockPort?: number; globalSalt: string; adminUserID: string; - newLeafURL?: string; + newLeafURLs?: string[]; discordReportChannelWebhookURL?: string; discordFirstTimeSubmissionsWebhookURL?: string; discordCompletelyIncorrectReportWebhookURL?: string; diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 855a2d6..e68370a 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -21,15 +21,15 @@ export class YouTubeAPI { } } - if (!config.newLeafURL) return {err: "NewLeaf URL not found", data: null}; + if (!config.newLeafURLs || config.newLeafURLs.length <= 0) return {err: "NewLeaf URL not found", data: null}; try { - const result = await fetch(config.newLeafURL + "/api/v1/videos/" + videoID, { method: "GET" }); + const result = await fetch(config.newLeafURLs[Math.floor(Math.random() * config.newLeafURLs.length)] + "/api/v1/videos/" + videoID, { method: "GET" }); if (result.ok) { const data = await result.json(); if (data.error) { - Logger.warn("CloudTube API Error: " + data.error) + Logger.warn("NewLeaf API Error: " + data.error) return { err: data.error, data: null }; } diff --git a/test.json b/test.json index c3b1096..f695671 100644 --- a/test.json +++ b/test.json @@ -3,7 +3,7 @@ "mockPort": 8081, "globalSalt": "testSalt", "adminUserID": "testUserId", - "newLeafURL": "placeholder", + "newLeafURLs": ["placeholder"], "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", "discordCompletelyIncorrectReportWebhookURL": "http://127.0.0.1:8081/CompletelyIncorrectReportWebhook", From 912f878906a671bb72dd811cbbf3f40cdbc01735 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 3 Jun 2021 14:49:01 -0400 Subject: [PATCH 46/85] Print video ID in newleaf errors --- src/utils/youtubeApi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index e68370a..ded8769 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -29,7 +29,7 @@ export class YouTubeAPI { if (result.ok) { const data = await result.json(); if (data.error) { - Logger.warn("NewLeaf API Error: " + data.error) + Logger.warn("NewLeaf API Error for " + videoID + ": " + data.error) return { err: data.error, data: null }; } From 1c8c76831ee7b620243fa9f3b2102975a8d79b1e Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 4 Jun 2021 16:03:18 -0400 Subject: [PATCH 47/85] Make redis not persist --- docker/redis/redis.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/redis/redis.conf b/docker/redis/redis.conf index cd43e41..92a9dfa 100644 --- a/docker/redis/redis.conf +++ b/docker/redis/redis.conf @@ -1,2 +1,5 @@ maxmemory-policy allkeys-lru -maxmemory 2000mb \ No newline at end of file +maxmemory 2000mb + +appendonly no +save "" \ No newline at end of file From 4225d9b3b3a1c69a4fad5f1b15a22a424bfe38ab Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 8 Jun 2021 20:20:05 -0400 Subject: [PATCH 48/85] Silently reject votes --- src/routes/voteOnSponsorTime.ts | 35 +++++++++++++++++++++++++++------ src/types/config.model.ts | 1 + test/cases/voteOnSponsorTime.ts | 8 ++++---- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index ce59172..36e23a0 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -19,9 +19,17 @@ const voteTypes = { incorrect: 1, }; +enum VoteWebhookType { + Normal, + Rejected +} + interface FinalResponse { + blockVote: boolean, finalStatus: number - finalMessage: string + finalMessage: string, + webhookType: VoteWebhookType, + webhookMessage: string } interface VoteData { @@ -52,7 +60,15 @@ async function sendWebhooks(voteData: VoteData) { if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) { let webhookURL: string = null; if (voteData.voteTypeEnum === voteTypes.normal) { - webhookURL = config.discordReportChannelWebhookURL; + switch (voteData.finalResponse.webhookType) { + case VoteWebhookType.Normal: + webhookURL = config.discordReportChannelWebhookURL; + break; + case VoteWebhookType.Rejected: + webhookURL = config.discordFailedReportChannelWebhookURL; + break; + } + } else if (voteData.voteTypeEnum === voteTypes.incorrect) { webhookURL = config.discordCompletelyIncorrectReportWebhookURL; } @@ -114,7 +130,9 @@ async function sendWebhooks(voteData: VoteData) { getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime), "color": 10813440, "author": { - "name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), + "name": voteData.finalResponse?.webhookMessage ?? + voteData.finalResponse?.finalMessage ?? + getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission), }, "thumbnail": { "url": getMaxResThumbnail(data) || "", @@ -252,8 +270,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) { // To force a non 200, change this early let finalResponse: FinalResponse = { + blockVote: false, finalStatus: 200, - finalMessage: null + finalMessage: null, + webhookType: VoteWebhookType.Normal, + webhookMessage: null } //x-forwarded-for if this server is behind a proxy @@ -276,8 +297,9 @@ export async function voteOnSponsorTime(req: Request, res: Response) { ' where "UUID" = ?', [UUID])); if (await isSegmentLocked() || await isVideoLocked()) { - finalResponse.finalStatus = 403; - finalResponse.finalMessage = "Vote rejected: A moderator has decided that this segment is correct" + finalResponse.blockVote = true; + finalResponse.webhookType = VoteWebhookType.Normal + finalResponse.webhookMessage = "Vote rejected: A moderator has decided that this segment is correct" } } @@ -384,6 +406,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined) + && !finalResponse.blockVote && finalResponse.finalStatus === 200; if (ableToVote) { diff --git a/src/types/config.model.ts b/src/types/config.model.ts index f5614d9..dc16d0d 100644 --- a/src/types/config.model.ts +++ b/src/types/config.model.ts @@ -8,6 +8,7 @@ export interface SBSConfig { adminUserID: string; newLeafURLs?: string[]; discordReportChannelWebhookURL?: string; + discordFailedReportChannelWebhookURL?: string; discordFirstTimeSubmissionsWebhookURL?: string; discordCompletelyIncorrectReportWebhookURL?: string; neuralBlockURL?: string; diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts index f649d12..b2cbc27 100644 --- a/test/cases/voteOnSponsorTime.ts +++ b/test/cases/voteOnSponsorTime.ts @@ -446,10 +446,10 @@ describe('voteOnSponsorTime', () => { + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=0") .then(async res => { let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]); - if (res.status === 403 && row.votes === 2) { + if (res.status === 200 && row.votes === 2) { done(); } else { - done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row)); + done("Status code was " + res.status + " instead of 200, row was " + JSON.stringify(row)); } }) .catch(err => done(err)); @@ -474,10 +474,10 @@ describe('voteOnSponsorTime', () => { + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&category=outro") .then(async res => { let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]); - if (res.status === 403 && row.category === "sponsor") { + if (res.status === 200 && row.category === "sponsor") { done(); } else { - done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row)); + done("Status code was " + res.status + " instead of 200, row was " + JSON.stringify(row)); } }) .catch(err => done(err)); From 344e680fe36f05a288c8ca372c2a26658d6ec382 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 9 Jun 2021 15:14:31 -0400 Subject: [PATCH 49/85] Fix rejections not being seperated --- src/routes/voteOnSponsorTime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 36e23a0..0aec481 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -298,7 +298,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { if (await isSegmentLocked() || await isVideoLocked()) { finalResponse.blockVote = true; - finalResponse.webhookType = VoteWebhookType.Normal + finalResponse.webhookType = VoteWebhookType.Rejected finalResponse.webhookMessage = "Vote rejected: A moderator has decided that this segment is correct" } } From b08f5c8390ff9af18d28f3d565a1ae4a9c57eb14 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 13 Jun 2021 16:00:36 -0400 Subject: [PATCH 50/85] Don't break for incorrect votes --- src/routes/voteOnSponsorTime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 0aec481..756ebf2 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -426,7 +426,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { //update the vote count on this sponsorTime //oldIncrementAmount will be zero is row is null - await db.prepare('run', 'UPDATE "sponsorTimes" SET ' + columnName + ' = ' + columnName + ' + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]); + await db.prepare('run', 'UPDATE "sponsorTimes" SET "' + columnName + '" = ' + columnName + ' + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]); if (isVIP && incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { // Lock this submission await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 1 WHERE "UUID" = ?', [UUID]); From 588e0abdd8cf7b954d536dbc51feba6059f5a1f4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 13 Jun 2021 16:22:10 -0400 Subject: [PATCH 51/85] Fix type = 20 vote --- src/routes/voteOnSponsorTime.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 756ebf2..147139e 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -333,7 +333,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { return res.status(403).send('Vote rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?'); } - const voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect; + const voteTypeEnum = (type == 0 || type == 1 || type == 20) ? voteTypes.normal : voteTypes.incorrect; try { //check if vote has already happened @@ -426,7 +426,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { //update the vote count on this sponsorTime //oldIncrementAmount will be zero is row is null - await db.prepare('run', 'UPDATE "sponsorTimes" SET "' + columnName + '" = ' + columnName + ' + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]); + await db.prepare('run', 'UPDATE "sponsorTimes" SET "' + columnName + '" = "' + columnName + '" + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]); if (isVIP && incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { // Lock this submission await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 1 WHERE "UUID" = ?', [UUID]); From 17eb9604e70e0c1208b6b79e5ac71b18c54a4fdf Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 13 Jun 2021 17:13:44 -0400 Subject: [PATCH 52/85] Add banned username --- src/routes/setUsername.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/setUsername.ts b/src/routes/setUsername.ts index 0283f4d..73ff651 100644 --- a/src/routes/setUsername.ts +++ b/src/routes/setUsername.ts @@ -40,7 +40,7 @@ export async function setUsername(req: Request, res: Response) { userID = getHash(userID); } - if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148"].includes(userID)) { + if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148", "c3424f0d1f99631e6b36e5bf634af953e96b790705abd86a9c5eb312239cb765"].includes(userID)) { // Don't allow res.sendStatus(200); return; From bbd478f32262b184fc83f87c20872089f427f3f8 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Mon, 14 Jun 2021 10:47:46 +0200 Subject: [PATCH 53/85] Fix webhook default value Added missing property and changed all defaults back to null --- src/config.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 34e5438..72099f9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,10 +20,11 @@ addDefaults(config, { maxNumberOfActiveWarnings: 3, hoursAfterWarningExpires: 24, adminUserID: "", - discordCompletelyIncorrectReportWebhookURL: "", - discordFirstTimeSubmissionsWebhookURL: "", - discordNeuralBlockRejectWebhookURL: "", - discordReportChannelWebhookURL: "", + discordCompletelyIncorrectReportWebhookURL: null, + discordFirstTimeSubmissionsWebhookURL: null, + discordNeuralBlockRejectWebhookURL: null, + discordFailedReportChannelWebhookURL: null, + discordReportChannelWebhookURL: null, getTopUsersCacheTimeMinutes: 0, globalSalt: null, mode: "", From edbbc62e5caac47e7415844772f2ce5b92822f15 Mon Sep 17 00:00:00 2001 From: Michael C Date: Mon, 14 Jun 2021 13:19:41 -0400 Subject: [PATCH 54/85] add delete to CORS for /api/lockCategories --- nginx/nginx.conf | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index f5edbba..1c39285 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -143,7 +143,7 @@ http { ### CORS if ($request_method = 'OPTIONS') { add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; # # Custom headers and headers various browsers *should* be OK with but aren't # @@ -158,13 +158,19 @@ http { } if ($request_method = 'POST') { add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } if ($request_method = 'GET') { add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + if ($request_method = 'DELETE') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; } From e06eb96fa7b7b08cc95cef9b2ebf7565a3ec36b7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 14 Jun 2021 16:23:39 -0400 Subject: [PATCH 55/85] Add ability to ban specific category --- src/routes/shadowBanUser.ts | 10 ++- test/cases/shadowBanUser.ts | 126 ++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 test/cases/shadowBanUser.ts diff --git a/src/routes/shadowBanUser.ts b/src/routes/shadowBanUser.ts index 4ecfc45..17b9798 100644 --- a/src/routes/shadowBanUser.ts +++ b/src/routes/shadowBanUser.ts @@ -1,6 +1,7 @@ import {db, privateDB} from '../databases/databases'; import {getHash} from '../utils/getHash'; import {Request, Response} from 'express'; +import { config } from '../config'; export async function shadowBanUser(req: Request, res: Response) { const userID = req.query.userID as string; @@ -8,12 +9,15 @@ export async function shadowBanUser(req: Request, res: Response) { let adminUserIDInput = req.query.adminUserID as string; const enabled = req.query.enabled === undefined - ? false + ? true : req.query.enabled === 'true'; //if enabled is false and the old submissions should be made visible again const unHideOldSubmissions = req.query.unHideOldSubmissions !== "false"; + const categories: string[] = req.query.categories ? JSON.parse(req.query.categories as string) : config.categoryList; + categories.filter((category) => typeof category === "string" && !(/[^a-z|_|-]/.test(category))); + if (adminUserIDInput == undefined || (userID == undefined && hashedIP == undefined)) { //invalid request res.sendStatus(400); @@ -42,7 +46,7 @@ export async function shadowBanUser(req: Request, res: Response) { //find all previous submissions and hide them if (unHideOldSubmissions) { - await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? + await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")}) AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]); } @@ -61,7 +65,7 @@ export async function shadowBanUser(req: Request, res: Response) { await Promise.all(allSegments.filter((item: {uuid: string}) => { return segmentsToIgnore.indexOf(item) === -1; }).map((UUID: string) => { - return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ?`, [UUID]); + return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID]); })); } } diff --git a/test/cases/shadowBanUser.ts b/test/cases/shadowBanUser.ts new file mode 100644 index 0000000..5d6af7e --- /dev/null +++ b/test/cases/shadowBanUser.ts @@ -0,0 +1,126 @@ +import fetch from 'node-fetch'; +import {db, privateDB} from '../../src/databases/databases'; +import {Done, getbaseURL} from '../utils'; +import {getHash} from '../../src/utils/getHash'; + +describe('shadowBanUser', () => { + before(() => { + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES'; + db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, 'shadow-1-uuid-0', 'shadowBanned', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')"); + db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, 'shadow-1-uuid-0-1', 'shadowBanned', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')"); + db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, 'shadow-1-uuid-2', 'shadowBanned', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')"); + + db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, 'shadow-2-uuid-0', 'shadowBanned2', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('testtesttest', 1) + "')"); + db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, 'shadow-2-uuid-0-1', 'shadowBanned2', 0, 50, 'sponsor', 'PeerTube', 120, 0, 0, '" + getHash('testtesttest2', 1) + "')"); + db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, 'shadow-2-uuid-2', 'shadowBanned2', 0, 50, 'intro', 'YouTube', 101, 0, 0, '" + getHash('testtesttest', 1) + "')"); + + db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, 'shadow-3-uuid-0', 'shadowBanned3', 0, 50, 'sponsor', 'YouTube', 100, 0, 1, '" + getHash('testtesttest', 1) + "')"); + db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, 'shadow-3-uuid-0-1', 'shadowBanned3', 0, 50, 'sponsor', 'PeerTube', 120, 0, 1, '" + getHash('testtesttest2', 1) + "')"); + db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, 'shadow-3-uuid-2', 'shadowBanned3', 0, 50, 'intro', 'YouTube', 101, 0, 1, '" + getHash('testtesttest', 1) + "')"); + privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('shadowBanned3')`); + + db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("shadow-ban-vip") + "')"); + }); + + + it('Should be able to ban user and hide submissions', (done: Done) => { + fetch(getbaseURL() + "/api/shadowBanUser?userID=shadowBanned&adminUserID=shadow-ban-vip", { + method: 'POST' + }) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned", 1]); + const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]); + + if (shadowRow && videoRow?.length === 3) { + done(); + } else { + done("Ban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow)); + } + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be able to unban user without unhiding submissions', (done: Done) => { + fetch(getbaseURL() + "/api/shadowBanUser?userID=shadowBanned&adminUserID=shadow-ban-vip&enabled=false&unHideOldSubmissions=false", { + method: 'POST' + }) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned", 1]); + const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]); + + if (!shadowRow && videoRow?.length === 3) { + done(); + } else { + done("Unban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow)); + } + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be able to ban user and hide submissions from only some categories', (done: Done) => { + fetch(getbaseURL() + '/api/shadowBanUser?userID=shadowBanned2&adminUserID=shadow-ban-vip&categories=["sponsor"]', { + method: 'POST' + }) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const videoRow: {category: string, shadowHidden: number}[] = (await db.prepare('all', `SELECT "shadowHidden", "category" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned2", 1])); + const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]); + + if (shadowRow && 2 == videoRow?.length && 2 === videoRow?.filter((elem) => elem?.category === "sponsor")?.length) { + done(); + } else { + done("Ban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow)); + } + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be able to unban user and unhide submissions', (done: Done) => { + fetch(getbaseURL() + "/api/shadowBanUser?userID=shadowBanned2&adminUserID=shadow-ban-vip&enabled=false", { + method: 'POST' + }) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned2", 1]); + const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]); + + if (!shadowRow && videoRow?.length === 0) { + done(); + } else { + done("Unban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow)); + } + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + + it('Should be able to unban user and unhide some submissions', (done: Done) => { + fetch(getbaseURL() + `/api/shadowBanUser?userID=shadowBanned3&adminUserID=shadow-ban-vip&enabled=false&categories=["sponsor"]`, { + method: 'POST' + }) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const videoRow = await db.prepare('all', `SELECT "shadowHidden", "category" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned3", 1]); + const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned3"]); + + if (!shadowRow && videoRow?.length === 1 && videoRow[0]?.category === "intro") { + done(); + } else { + done("Unban failed " + JSON.stringify(videoRow) + " " + JSON.stringify(shadowRow)); + } + } + }) + .catch(err => done("Couldn't call endpoint")); + }); + +}); From 8cce2a5977c732e48fbe137d6f3dccd95958b135 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 14 Jun 2021 22:25:24 +0200 Subject: [PATCH 56/85] Update nginx config --- nginx/nginx.conf | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index f5edbba..095bb8d 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,8 +1,8 @@ worker_processes 8; -worker_rlimit_nofile 8192; +worker_rlimit_nofile 65536; events { - worker_connections 132768; ## Default: 1024 + worker_connections 432768; ## Default: 1024 } http { @@ -13,7 +13,7 @@ http { upstream backend_GET { least_conn; server localhost:4441; - server localhost:4442; + #server localhost:4442; #server localhost:4443; #server localhost:4444; #server localhost:4445; @@ -48,9 +48,9 @@ http { server_name sponsor.ajay.app api.sponsor.ajay.app; error_page 404 /404.html; - error_page 500 @myerrordirective_500; - error_page 502 @myerrordirective_502; - error_page 504 @myerrordirective_504; + #error_page 500 @myerrordirective_500; + #error_page 502 @myerrordirective_502; + #error_page 504 @myerrordirective_504; #location = /404 { # root /home/sbadmin/caddy/SponsorBlockSite/public-prod; # internal; @@ -58,15 +58,15 @@ http { #proxy_send_timeout 120s; - location @myerrordirective_500 { - return 400 "Internal Server Error"; - } - location @myerrordirective_502 { - return 400 "Bad Gateway"; - } - location @myerrordirective_504 { - return 400 "Gateway Timeout"; - } + #location @myerrordirective_500 { + # return 400 "Internal Server Error"; + #} + #location @myerrordirective_502 { + # return 400 "Bad Gateway"; + #} + #location @myerrordirective_504 { + # return 400 "Gateway Timeout"; + #} location /news { @@ -106,8 +106,11 @@ http { } location /download/ { + gzip on; + gzip_types text/plain application/json; #alias /home/sbadmin/sponsor/docker/database-export/; - return 307 https://cdnsponsor.ajay.app$request_uri; + alias /home/sbadmin/sponsor/docker/database-export/; + #return 307 https://cdnsponsor.ajay.app$request_uri; } location /database { proxy_pass http://backend_db; @@ -250,7 +253,7 @@ http { server { access_log off; - error_log /dev/null; + error_log /etc/nginx/logs/log.txt; if ($host = api.sponsor.ajay.app) { From 9cf68b8903b250bd4392a46ebb8e69bcfe5c72b8 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Tue, 15 Jun 2021 00:08:55 +0200 Subject: [PATCH 57/85] Fix test config adminUserID --- test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.json b/test.json index f695671..db714b9 100644 --- a/test.json +++ b/test.json @@ -2,7 +2,7 @@ "port": 8080, "mockPort": 8081, "globalSalt": "testSalt", - "adminUserID": "testUserId", + "adminUserID": "4bdfdc9cddf2c7d07a8a87b57bf6d25389fb75d1399674ee0e0938a6a60f4c3b", "newLeafURLs": ["placeholder"], "discordReportChannelWebhookURL": "http://127.0.0.1:8081/ReportChannelWebhook", "discordFirstTimeSubmissionsWebhookURL": "http://127.0.0.1:8081/FirstTimeSubmissionsWebhook", From 34fd78961b4834e40e797c4ce35ce54bbc526400 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Tue, 15 Jun 2021 00:09:37 +0200 Subject: [PATCH 58/85] Add username lock --- databases/_upgrade_sponsorTimes_13.sql | 17 +++ src/routes/setUsername.ts | 19 ++- test/cases/setUsername.ts | 184 +++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_13.sql create mode 100644 test/cases/setUsername.ts diff --git a/databases/_upgrade_sponsorTimes_13.sql b/databases/_upgrade_sponsorTimes_13.sql new file mode 100644 index 0000000..39b7067 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_13.sql @@ -0,0 +1,17 @@ +BEGIN TRANSACTION; + +/* Add reputation field */ +CREATE TABLE "sqlb_temp_table_13" ( + "userID" TEXT NOT NULL, + "userName" TEXT NOT NULL, + "locked" INTEGER NOT NULL default '0' +); + +INSERT INTO sqlb_temp_table_13 SELECT "userID", "userName", 0 FROM "userNames"; + +DROP TABLE "userNames"; +ALTER TABLE sqlb_temp_table_13 RENAME TO "userNames"; + +UPDATE "config" SET value = 13 WHERE key = 'version'; + +COMMIT; diff --git a/src/routes/setUsername.ts b/src/routes/setUsername.ts index 73ff651..fdd99dc 100644 --- a/src/routes/setUsername.ts +++ b/src/routes/setUsername.ts @@ -39,11 +39,18 @@ export async function setUsername(req: Request, res: Response) { //hash the userID userID = getHash(userID); } - - if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148", "c3424f0d1f99631e6b36e5bf634af953e96b790705abd86a9c5eb312239cb765"].includes(userID)) { - // Don't allow - res.sendStatus(200); - return; + + try { + const row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ? AND "locked" = '1'`, [userID]); + if (adminUserIDInput === undefined && row.count > 0) { + res.sendStatus(200); + return; + } + } + catch (error) { + Logger.error(error); + res.sendStatus(500); + return; } try { @@ -55,7 +62,7 @@ export async function setUsername(req: Request, res: Response) { await db.prepare('run', `UPDATE "userNames" SET "userName" = ? WHERE "userID" = ?`, [userName, userID]); } else { //add to the db - await db.prepare('run', `INSERT INTO "userNames" VALUES(?, ?)`, [userID, userName]); + await db.prepare('run', `INSERT INTO "userNames"("userID", "userName") VALUES(?, ?)`, [userID, userName]); } res.sendStatus(200); diff --git a/test/cases/setUsername.ts b/test/cases/setUsername.ts new file mode 100644 index 0000000..a943d39 --- /dev/null +++ b/test/cases/setUsername.ts @@ -0,0 +1,184 @@ +import fetch from 'node-fetch'; +import { Done, getbaseURL } from '../utils'; +import { db } from '../../src/databases/databases'; +import { getHash } from '../../src/utils/getHash'; + +const adminPrivateUserID = 'testUserId'; +const user01PrivateUserID = 'setUsername_01'; +const username01 = 'Username 01'; +const user02PrivateUserID = 'setUsername_02'; +const username02 = 'Username 02'; +const user03PrivateUserID = 'setUsername_03'; +const username03 = 'Username 03'; +const user04PrivateUserID = 'setUsername_04'; +const username04 = 'Username 04'; +const user05PrivateUserID = 'setUsername_05'; +const username05 = 'Username 05'; +const user06PrivateUserID = 'setUsername_06'; +const username06 = 'Username 06'; +const user07PrivateUserID = 'setUsername_07'; +const username07 = 'Username 07'; + +async function addUsername(userID: string, userName: string, locked = 0) { + await db.prepare('run', 'INSERT INTO "userNames" ("userID", "userName", "locked") VALUES(?, ?, ?)', [userID, userName, locked]); +} + +async function getUsername(userID: string) { + const row = await db.prepare('get', 'SELECT "userName" FROM "userNames" WHERE userID = ?', [userID]); + if (!row) { + return null; + } + return row.userName; +} + +describe('setUsername', () => { + before(async () => { + await addUsername(getHash(user01PrivateUserID), username01, 0); + await addUsername(getHash(user02PrivateUserID), username02, 0); + await addUsername(getHash(user03PrivateUserID), username03, 0); + await addUsername(getHash(user04PrivateUserID), username04, 1); + await addUsername(getHash(user05PrivateUserID), username05, 0); + await addUsername(getHash(user06PrivateUserID), username06, 0); + await addUsername(getHash(user07PrivateUserID), username07, 1); + }); + + it('Should return 200', (done: Done) => { + fetch(`${getbaseURL()}/api/setUsername?userID=${user01PrivateUserID}&username=Changed%20Username`, { + method: 'POST', + }) + .then(res => { + if (res.status !== 200) done(`Status code was ${res.status}`); + else done(); // pass + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Should return 400 for missing param "userID"', (done: Done) => { + fetch(`${getbaseURL()}/api/setUsername?username=MyUsername`, { + method: 'POST', + }) + .then(res => { + if (res.status !== 400) done(`Status code was ${res.status}`); + else done(); // pass + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Should return 400 for missing param "username"', (done: Done) => { + fetch(`${getbaseURL()}/api/setUsername?userID=test`, { + method: 'POST', + }) + .then(res => { + if (res.status !== 400) done(`Status code was ${res.status}`); + else done(); // pass + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Should return 400 for "username" longer then 64 characters', (done: Done) => { + const username65 = '0000000000000000000000000000000000000000000000000000000000000000X'; + fetch(`${getbaseURL()}/api/setUsername?userID=test&username=${encodeURIComponent(username65)}`, { + method: 'POST', + }) + .then(res => { + if (res.status !== 400) done(`Status code was ${res.status}`); + else done(); // pass + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Should not change username if it contains "discord"', (done: Done) => { + const newUsername = 'discord.me'; + fetch(`${getbaseURL()}/api/setUsername?userID=${user02PrivateUserID}&username=${encodeURIComponent(newUsername)}`, { + method: 'POST', + }) + .then(async res => { + if (res.status !== 200) done(`Status code was ${res.status}`); + else { + const userName = await getUsername(getHash(user02PrivateUserID)); + if (userName === newUsername) { + done(`Username '${username02}' got changed to '${newUsername}'`); + } + else done(); + } + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Should be able to change username', (done: Done) => { + const newUsername = 'newUsername'; + fetch(`${getbaseURL()}/api/setUsername?userID=${user03PrivateUserID}&username=${encodeURIComponent(newUsername)}`, { + method: 'POST', + }) + .then(async res => { + const username = await getUsername(getHash(user03PrivateUserID)); + if (username !== newUsername) done(`Username did not change`); + else done(); + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Should not be able to change locked username', (done: Done) => { + const newUsername = 'newUsername'; + fetch(`${getbaseURL()}/api/setUsername?userID=${user04PrivateUserID}&username=${encodeURIComponent(newUsername)}`, { + method: 'POST', + }) + .then(async res => { + const username = await getUsername(getHash(user04PrivateUserID)); + if (username === newUsername) done(`Username '${username04}' got changed to '${username}'`); + else done(); + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Should filter out unicode control characters', (done: Done) => { + const newUsername = 'This\nUsername+has\tInvalid+Characters'; + fetch(`${getbaseURL()}/api/setUsername?userID=${user05PrivateUserID}&username=${encodeURIComponent(newUsername)}`, { + method: 'POST', + }) + .then(async res => { + const username = await getUsername(getHash(user05PrivateUserID)); + if (username === newUsername) done(`Username contains unicode control characters`); + else done(); + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Incorrect adminUserID should return 403', (done: Done) => { + const newUsername = 'New Username'; + fetch(`${getbaseURL()}/api/setUsername?adminUserID=invalidAdminID&userID=${getHash(user06PrivateUserID)}&username=${encodeURIComponent(newUsername)}`, { + method: 'POST', + }) + .then(async res => { + if (res.status !== 403) done(`Status code was ${res.status}`); + else done(); + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Admin should be able to change username', (done: Done) => { + const newUsername = 'New Username'; + fetch(`${getbaseURL()}/api/setUsername?adminUserID=${adminPrivateUserID}&userID=${getHash(user06PrivateUserID)}&username=${encodeURIComponent(newUsername)}`, { + method: 'POST', + }) + .then(async res => { + const username = await getUsername(getHash(user06PrivateUserID)); + if (username !== newUsername) done(`Failed to change username from '${username06}' to '${newUsername}'`); + else done(); + }) + .catch(err => done(`couldn't call endpoint`)); + }); + + it('Admin should be able to change locked username', (done: Done) => { + const newUsername = 'New Username'; + fetch(`${getbaseURL()}/api/setUsername?adminUserID=${adminPrivateUserID}&userID=${getHash(user07PrivateUserID)}&username=${encodeURIComponent(newUsername)}`, { + method: 'POST', + }) + .then(async res => { + const username = await getUsername(getHash(user06PrivateUserID)); + if (username !== newUsername) done(`Failed to change username from '${username06}' to '${newUsername}'`); + else done(); + }) + .catch(err => done(`couldn't call endpoint`)); + }); +}); From 062faba8d1cf366fac15781d85e49d11b5798136 Mon Sep 17 00:00:00 2001 From: Michael C Date: Mon, 14 Jun 2021 20:29:02 -0400 Subject: [PATCH 59/85] remove CORS from nginx, add to express --- nginx/nginx.conf | 66 +----------------------------------------- src/middleware/cors.ts | 3 +- 2 files changed, 3 insertions(+), 66 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 1c39285..5dce785 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -139,41 +139,6 @@ http { location / { root /home/sbadmin/SponsorBlockSite/public-prod; - - ### CORS - if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; - # - # Custom headers and headers various browsers *should* be OK with but aren't - # - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - # - # Tell client that this pre-flight info is valid for 20 days - # - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain; charset=utf-8'; - add_header 'Content-Length' 0; - return 204; - } - if ($request_method = 'POST') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } - if ($request_method = 'GET') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } - if ($request_method = 'DELETE') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } } @@ -208,36 +173,7 @@ http { } location / { - root /home/sbadmin/SponsorBlockSite/public-prod; - - ### CORS - if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - # - # Custom headers and headers various browsers *should* be OK with but aren't - # - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - # - # Tell client that this pre-flight info is valid for 20 days - # - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain; charset=utf-8'; - add_header 'Content-Length' 0; - return 204; - } - if ($request_method = 'POST') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } - if ($request_method = 'GET') { - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - } + root /home/sbadmin/SponsorBlockSite/public-prod; } diff --git a/src/middleware/cors.ts b/src/middleware/cors.ts index 2b25d9b..1741319 100644 --- a/src/middleware/cors.ts +++ b/src/middleware/cors.ts @@ -2,6 +2,7 @@ import {NextFunction, Request, Response} from 'express'; export function corsMiddleware(req: Request, res: Response, next: NextFunction) { res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + res.header("Access-Control-Allow-Headers", "X-Requested-With, Content-Type, Accept"); + res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE") next(); } From 75981a3e5ff076b7e1d0c86f0b39bf9994504ba4 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 15 Jun 2021 01:03:37 -0400 Subject: [PATCH 60/85] Fix copy mistake --- databases/_upgrade_sponsorTimes_13.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/databases/_upgrade_sponsorTimes_13.sql b/databases/_upgrade_sponsorTimes_13.sql index 39b7067..6dcb6ce 100644 --- a/databases/_upgrade_sponsorTimes_13.sql +++ b/databases/_upgrade_sponsorTimes_13.sql @@ -1,10 +1,10 @@ BEGIN TRANSACTION; -/* Add reputation field */ +/* Add locked field */ CREATE TABLE "sqlb_temp_table_13" ( "userID" TEXT NOT NULL, "userName" TEXT NOT NULL, - "locked" INTEGER NOT NULL default '0' + "locked" INTEGER NOT NULL default '0' ); INSERT INTO sqlb_temp_table_13 SELECT "userID", "userName", 0 FROM "userNames"; From 859ad6ea3849d49954eb4b1194453ae95df22ed0 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Tue, 15 Jun 2021 17:50:18 +0200 Subject: [PATCH 61/85] Add version to database.json --- src/routes/dumpDatabase.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/routes/dumpDatabase.ts b/src/routes/dumpDatabase.ts index 152e6f2..cb2169a 100644 --- a/src/routes/dumpDatabase.ts +++ b/src/routes/dumpDatabase.ts @@ -143,6 +143,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); } else { res.send({ + dbVersion: await getDbVersion(), lastUpdated: lastUpdate, updateQueued, links: latestDumpFiles.map((item:any) => { @@ -158,6 +159,12 @@ export default async function dumpDatabase(req: Request, res: Response, showPage await queueDump(); } +async function getDbVersion(): Promise { + const row = await db.prepare('get', `SELECT "value" FROM "config" WHERE "key" = 'version'`); + if (row === undefined) return 0; + return row.value; +} + export async function redirectLink(req: Request, res: Response): Promise { if (!config?.dumpDatabase?.enabled) { res.status(404).send("Database dump is disabled"); @@ -210,4 +217,4 @@ async function queueDump(): Promise { updateRunning = false; lastUpdate = startTime; } -} \ No newline at end of file +} From 607b7cbb0a7bf64899743049e255f69e4b635212 Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 15 Jun 2021 15:50:41 -0400 Subject: [PATCH 62/85] add ignored counts --- src/routes/getUserInfo.ts | 21 ++++++++++++++++++++- test/cases/getUserInfo.ts | 4 ++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index 8ea8dda..4ade24e 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -27,6 +27,15 @@ async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ min } } +async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise<{ ignoredSegmentCount: number }> { + try { + let row = await db.prepare("get", `SELECT COUNT(*) as "ignoredSegmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID]); + return row?.ignoredSegmentCount ?? 0 + } catch (err) { + return null; + } +} + async function dbGetUsername(userID: HashedUserID) { try { let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); @@ -50,6 +59,15 @@ async function dbGetViewsForUser(userID: HashedUserID) { } } +async function dbGetIgnoredViewsForUser(userID: HashedUserID) { + try { + let row = await db.prepare('get', `SELECT SUM("views") as "ignoredViewCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID]); + return row?.ignoredViewCount ?? 0; + } catch (err) { + return false; + } +} + async function dbGetWarningsForUser(userID: HashedUserID): Promise { try { let row = await db.prepare('get', `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]); @@ -70,7 +88,6 @@ export async function getUserInfo(req: Request, res: Response) { return; } - const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID); if (segmentsSummary) { res.send({ @@ -78,7 +95,9 @@ export async function getUserInfo(req: Request, res: Response) { userName: await dbGetUsername(hashedUserID), minutesSaved: segmentsSummary.minutesSaved, segmentCount: segmentsSummary.segmentCount, + ignoredSegmentCount: await dbGetIgnoredSegmentCount(hashedUserID), viewCount: await dbGetViewsForUser(hashedUserID), + ignoredViewCount: await dbGetIgnoredViewsForUser(hashedUserID), warnings: await dbGetWarningsForUser(hashedUserID), reputation: await getReputation(hashedUserID), vip: await isUserVIP(hashedUserID), diff --git a/test/cases/getUserInfo.ts b/test/cases/getUserInfo.ts index c809a49..f810d7a 100644 --- a/test/cases/getUserInfo.ts +++ b/test/cases/getUserInfo.ts @@ -54,8 +54,12 @@ describe('getUserInfo', () => { done('Returned incorrect minutesSaved "' + data.minutesSaved + '"'); } else if (data.viewCount !== 30) { done('Returned incorrect viewCount "' + data.viewCount + '"'); + } else if (data.ignoredViewCount !== 20) { + done('Returned incorrect ignoredViewCount "' + data.ignoredViewCount + '"'); } else if (data.segmentCount !== 3) { done('Returned incorrect segmentCount "' + data.segmentCount + '"'); + } else if (data.ignoredSegmentCount !== 2) { + done('Returned incorrect ignoredSegmentCount "' + data.ignoredSegmentCount + '"'); } else if (Math.abs(data.reputation - -0.928) > 0.001) { done('Returned incorrect reputation "' + data.reputation + '"'); } else { From 3b16cdb920d639c26206ee5d464a7269a73b19df Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 15 Jun 2021 17:08:17 -0400 Subject: [PATCH 63/85] add last lastSegmentID --- src/routes/getUserInfo.ts | 13 ++++++++++++- test/cases/getUserInfo.ts | 34 +++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index 4ade24e..8dc8ac9 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -5,6 +5,7 @@ import {Request, Response} from 'express'; import {Logger} from '../utils/logger'; import { HashedUserID, UserID } from '../types/user.model'; import { getReputation } from '../utils/reputation'; +import { SegmentUUID } from "../types/segments.model"; async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> { try { @@ -27,7 +28,7 @@ async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ min } } -async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise<{ ignoredSegmentCount: number }> { +async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise { try { let row = await db.prepare("get", `SELECT COUNT(*) as "ignoredSegmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID]); return row?.ignoredSegmentCount ?? 0 @@ -78,6 +79,15 @@ async function dbGetWarningsForUser(userID: HashedUserID): Promise { } } +async function dbGetLastSegmentForUser(userID: HashedUserID): Promise { + try { + let row = await db.prepare('get', `SELECT "timeSubmitted", "UUID" FROM "sponsorTimes" WHERE "userID" = ? ORDER BY "timeSubmitted" DESC LIMIT 1`, [userID]); + return row?.UUID ?? null; + } catch (err) { + return null; + } +} + export async function getUserInfo(req: Request, res: Response) { const userID = req.query.userID as UserID; const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID; @@ -101,6 +111,7 @@ export async function getUserInfo(req: Request, res: Response) { warnings: await dbGetWarningsForUser(hashedUserID), reputation: await getReputation(hashedUserID), vip: await isUserVIP(hashedUserID), + lastSegmentID: await dbGetLastSegmentForUser(hashedUserID), }); } else { res.status(400).send(); diff --git a/test/cases/getUserInfo.ts b/test/cases/getUserInfo.ts index f810d7a..3e63474 100644 --- a/test/cases/getUserInfo.ts +++ b/test/cases/getUserInfo.ts @@ -8,14 +8,14 @@ describe('getUserInfo', () => { let startOfUserNamesQuery = `INSERT INTO "userNames" ("userID", "userName") VALUES`; await db.prepare("run", startOfUserNamesQuery + "('" + getHash("getuserinfo_user_01") + "', 'Username user 01')"); let startOfSponsorTimesQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden") VALUES'; - await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000001', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); - await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000002', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); - await db.prepare("run", startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -1, 'uuid000003', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 0)"); - await db.prepare("run", startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -2, 'uuid000004', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)"); - await db.prepare("run", startOfSponsorTimesQuery + "('xzzzxxyyy', 1, 11, -5, 'uuid000005', '" + getHash("getuserinfo_user_01") + "', 0, 10, 'sponsor', 1)"); - await db.prepare("run", startOfSponsorTimesQuery + "('zzzxxxyyy', 1, 11, 2, 'uuid000006', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 0)"); - await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000007', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); - await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000008', '" + getHash("getuserinfo_user_02") + "', 0, 10, 'sponsor', 1)"); + await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000001', '" + getHash("getuserinfo_user_01") + "', 1, 10, 'sponsor', 0)"); + await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000002', '" + getHash("getuserinfo_user_01") + "', 2, 10, 'sponsor', 0)"); + await db.prepare("run", startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -1, 'uuid000003', '" + getHash("getuserinfo_user_01") + "', 3, 10, 'sponsor', 0)"); + await db.prepare("run", startOfSponsorTimesQuery + "('yyyxxxzzz', 1, 11, -2, 'uuid000004', '" + getHash("getuserinfo_user_01") + "', 4, 10, 'sponsor', 1)"); + await db.prepare("run", startOfSponsorTimesQuery + "('xzzzxxyyy', 1, 11, -5, 'uuid000005', '" + getHash("getuserinfo_user_01") + "', 5, 10, 'sponsor', 1)"); + await db.prepare("run", startOfSponsorTimesQuery + "('zzzxxxyyy', 1, 11, 2, 'uuid000006', '" + getHash("getuserinfo_user_02") + "', 6, 10, 'sponsor', 0)"); + await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000007', '" + getHash("getuserinfo_user_02") + "', 7, 10, 'sponsor', 1)"); + await db.prepare("run", startOfSponsorTimesQuery + "('xxxyyyzzz', 1, 11, 2, 'uuid000008', '" + getHash("getuserinfo_user_02") + "', 8, 10, 'sponsor', 1)"); await db.prepare("run", `INSERT INTO warnings ("userID", "issueTime", "issuerUserID", enabled) VALUES ('` + getHash('getuserinfo_warning_0') + "', 10, 'getuserinfo_vip', 1)"); @@ -62,6 +62,8 @@ describe('getUserInfo', () => { done('Returned incorrect ignoredSegmentCount "' + data.ignoredSegmentCount + '"'); } else if (Math.abs(data.reputation - -0.928) > 0.001) { done('Returned incorrect reputation "' + data.reputation + '"'); + } else if (data.lastSegmentID !== "uuid000005") { + done('Returned incorrect last segment "' + data.lastSegmentID + '"'); } else { done(); // pass } @@ -113,7 +115,7 @@ describe('getUserInfo', () => { .catch(err => ("couldn't call endpoint")); }); - it('Should not get warnings if noe', (done: Done) => { + it('Should not get warnings if none', (done: Done) => { fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_warning_2') .then(async res => { if (res.status !== 200) { @@ -142,4 +144,18 @@ describe('getUserInfo', () => { }) .catch(err => ('couldn\'t call endpoint')); }); + + it('Should return null segment if none', (done: Done) => { + fetch(getbaseURL() + '/api/userInfo?userID=getuserinfo_null') + .then(async res => { + if (res.status !== 200) { + done('non 200 (' + res.status + ')'); + } else { + const data = await res.json(); + if (data.lastSegmentID !== null) done('returned segment ' + data.warnings + ', not ' + null); + else done(); // pass + } + }) + .catch(err => ("couldn't call endpoint")); + }); }); From 13b105504baa79440423796a98ee20d135c9c37b Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 15 Jun 2021 17:16:32 -0400 Subject: [PATCH 64/85] remove timeSubmitted from query --- src/routes/getUserInfo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts index 8dc8ac9..44db5d5 100644 --- a/src/routes/getUserInfo.ts +++ b/src/routes/getUserInfo.ts @@ -81,7 +81,7 @@ async function dbGetWarningsForUser(userID: HashedUserID): Promise { async function dbGetLastSegmentForUser(userID: HashedUserID): Promise { try { - let row = await db.prepare('get', `SELECT "timeSubmitted", "UUID" FROM "sponsorTimes" WHERE "userID" = ? ORDER BY "timeSubmitted" DESC LIMIT 1`, [userID]); + let row = await db.prepare('get', `SELECT "UUID" FROM "sponsorTimes" WHERE "userID" = ? ORDER BY "timeSubmitted" DESC LIMIT 1`, [userID]); return row?.UUID ?? null; } catch (err) { return null; From a003733e51fae702f4c6907b307a7a3633dbc8e9 Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 15 Jun 2021 21:29:36 -0400 Subject: [PATCH 65/85] fix test suite --- test/cases/getSkipSegments.ts | 110 +++++++++++++++++----------------- test/cases/getUserInfo.ts | 19 +++--- 2 files changed, 64 insertions(+), 65 deletions(-) diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index f40203c..7663764 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -22,111 +22,111 @@ describe('getSkipSegments', () => { }); - it('Should be able to get a time by category 1', () => { + it('Should be able to get a time by category 1', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) { - return; + done(); } else { - return ("Received incorrect body: " + (await res.text())); + done("Received incorrect body: " + (await res.text())); } } }) .catch(err => "Couldn't call endpoint"); }); - it('Should be able to get a time by category for a different service 1', () => { - fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube") + it('Should be able to get a time by category for a different service 1', (done: Done) => { + fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest2&category=sponsor&service=PeerTube") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) { - return; + done(); } else { - return ("Received incorrect body: " + (await res.text())); + done("Received incorrect body: " + (await res.text())); } } }) .catch(err => "Couldn't call endpoint"); }); - it('Should be able to get a time by category 2', () => { + it('Should be able to get a time by category 2', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=intro") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33 && data[0].category === "intro" && data[0].UUID === "1-uuid-2") { - return; + done(); } else { - return ("Received incorrect body: " + (await res.text())); + done("Received incorrect body: " + (await res.text())); } } }) .catch(err => ("Couldn't call endpoint")); }); - it('Should be able to get a time by categories array', () => { + it('Should be able to get a time by categories array', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[\"sponsor\"]") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) { - return; + done(); } else { - return ("Received incorrect body: " + (await res.text())); + done("Received incorrect body: " + (await res.text())); } } }) .catch(err => ("Couldn't call endpoint")); }); - it('Should be able to get a time by categories array 2', () => { + it('Should be able to get a time by categories array 2', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[\"intro\"]") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33 && data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) { - return; + done(); } else { - return ("Received incorrect body: " + (await res.text())); + done("Received incorrect body: " + (await res.text())); } } }) .catch(err => ("Couldn't call endpoint")); }); - it('Should be empty if all submissions are hidden', () => { + it('Should be empty if all submissions are hidden', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const data = await res.json(); if (data.length === 0) { - return; + done(); } else { - return ("Received incorrect body: " + (await res.text())); + done("Received incorrect body: " + (await res.text())); } } }) .catch(err => ("Couldn't call endpoint")); }); - it('Should be able to get multiple times by category', () => { + it('Should be able to get multiple times by category', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=multiple&categories=[\"intro\"]") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200)done("Status code was: " + res.status); else { const body = await res.text(); const data = JSON.parse(body); @@ -142,20 +142,20 @@ describe('getSkipSegments', () => { } } - if (success) return; - else return ("Received incorrect body: " + body); + if (success) done(); + else done("Received incorrect body: " + body); } else { - return ("Received incorrect body: " + body); + done("Received incorrect body: " + body); } } }) .catch(err => ("Couldn't call endpoint\n\n" + err)); }); - it('Should be able to get multiple times by multiple categories', () => { + it('Should be able to get multiple times by multiple categories', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[\"sponsor\", \"intro\"]") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const body = await res.text(); const data = JSON.parse(body); @@ -172,91 +172,91 @@ describe('getSkipSegments', () => { } } - if (success) return; - else return ("Received incorrect body: " + body); + if (success) done(); + else done("Received incorrect body: " + body); } else { - return ("Received incorrect body: " + body); + done("Received incorrect body: " + body); } } }) .catch(err => ("Couldn't call endpoint")); }); - it('Should be possible to send unexpected query parameters', () => { + it('Should be possible to send unexpected query parameters', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&fakeparam=hello&category=sponsor") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const body = await res.text(); const data = JSON.parse(body); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") { - return; + done(); } else { - return ("Received incorrect body: " + body); + done("Received incorrect body: " + body); } } }) - .catch(err => ("Couldn't call endpoint")); + .catch(err => done("Couldn't call endpoint")); }); - it('Low voted submissions should be hidden', () => { + it('Low voted submissions should be hidden', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=test3&category=sponsor") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done("Status code was: " + res.status); else { const body = await res.text(); const data = JSON.parse(body); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 && data[0].category === "sponsor" && data[0].UUID === "1-uuid-4") { - return; + done(); } else { - return ("Received incorrect body: " + body); + done("Received incorrect body: " + body); } } }) .catch(err => ("Couldn't call endpoint")); }); - it('Should return 404 if no segment found', () => { + it('Should return 404 if no segment found', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=notarealvideo") .then(res => { - if (res.status !== 404) return ("non 404 respone code: " + res.status); - else return; // pass + if (res.status !== 404) done("non 404 respone code: " + res.status); + else done(); // pass }) .catch(err => ("couldn't call endpoint")); }); - it('Should be able send a comma in a query param', () => { + it('Should be able send a comma in a query param', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest,test&category=sponsor") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done ("Status code was: " + res.status); else { const body = await res.text(); const data = JSON.parse(body); if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11 && data[0].category === "sponsor" && data[0].UUID === "1-uuid-1") { - return; + done(); } else { - return ("Received incorrect body: " + body); + done("Received incorrect body: " + body); } } }) .catch(err => ("Couldn't call endpoint")); }); - it('Should always get locked segment', () => { + it('Should always get locked segment', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=locked&category=intro") .then(async res => { - if (res.status !== 200) return ("Status code was: " + res.status); + if (res.status !== 200) done ("Status code was: " + res.status); else { const data = await res.json(); if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33 && data[0].category === "intro" && data[0].UUID === "1-uuid-locked-8") { - return; + done(); } else { - return ("Received incorrect body: " + (await res.text())); + done("Received incorrect body: " + (await res.text())); } } }) diff --git a/test/cases/getUserInfo.ts b/test/cases/getUserInfo.ts index c809a49..b8cc25c 100644 --- a/test/cases/getUserInfo.ts +++ b/test/cases/getUserInfo.ts @@ -80,19 +80,18 @@ describe('getUserInfo', () => { .catch(err => ("couldn't call endpoint")); }); - it('Should get warning data with public ID', async () => { - try { - const res = await fetch(getbaseURL() + '/api/userInfo?userID=' + await getHash("getuserinfo_warning_0")) - + it('Should get warning data with public ID', (done: Done) => { + fetch(getbaseURL() + '/api/userInfo?publicUserID=' + getHash("getuserinfo_warning_0")) + .then(async res => { if (res.status !== 200) { - return 'non 200 (' + res.status + ')'; + done('non 200 (' + res.status + ')'); } else { - const data = await res.json();; - if (data.warnings !== 1) return 'wrong number of warnings: ' + data.warnings + ', not ' + 1; + const data = await res.json(); + if (data.warnings !== 1) done('wrong number of warnings: ' + data.warnings + ', not ' + 1); + else done(); } - } catch (err) { - return "couldn't call endpoint"; - } + }) + .catch(err => ("couldn't call endpoint")); }); it('Should get multiple warnings', (done: Done) => { From 31071ddb177ee9a8882404883fd1ff118c474446 Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 15 Jun 2021 21:56:20 -0400 Subject: [PATCH 66/85] new test cases --- test/cases/getSkipSegments.ts | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index 7663764..150dff2 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -13,7 +13,7 @@ describe('getSkipSegments', () => { await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('test3', 1) + "')"); await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('test3', 1) + "')"); await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, 0, '" + getHash('multiple', 1) + "')"); - await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, 0, '" + getHash('multiple', 1) + "')"); + await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, 0, '" + getHash('multiple', 1) + "')"); await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, 0, '" + getHash('locked', 1) + "')"); await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, 0, '" + getHash('locked', 1) + "')"); await db.prepare("run", startOfQuery + "('onlyHiddenSegments', 20, 34, 100000, 0, 'onlyHiddenSegments', 'testman', 0, 50, 'sponsor', 'YouTube', 190, 1, 0, '" + getHash('onlyHiddenSegments', 1) + "')"); @@ -107,18 +107,11 @@ describe('getSkipSegments', () => { .catch(err => ("Couldn't call endpoint")); }); - it('Should be empty if all submissions are hidden', (done: Done) => { + it('Should return 404 if all submissions are hidden', (done: Done) => { fetch(getbaseURL() + "/api/skipSegments?videoID=onlyHiddenSegments") - .then(async res => { - if (res.status !== 200) done("Status code was: " + res.status); - else { - const data = await res.json(); - if (data.length === 0) { - done(); - } else { - done("Received incorrect body: " + (await res.text())); - } - } + .then(res => { + if (res.status !== 404) done("non 404 respone code: " + res.status); + else done(); // pass }) .catch(err => ("Couldn't call endpoint")); }); @@ -134,9 +127,9 @@ describe('getSkipSegments', () => { let success = true; for (const segment of data) { if ((segment.segment[0] !== 20 || segment.segment[1] !== 33 - || segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) && + || segment.category !== "intro" || segment.UUID !== "1-uuid-7") && (segment.segment[0] !== 1 || segment.segment[1] !== 11 - || segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) { + || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) { success = false; break; } From 0a8f7aa39d44631f6e4ad274209f944e94c98a5d Mon Sep 17 00:00:00 2001 From: Michael C Date: Tue, 15 Jun 2021 23:01:26 -0400 Subject: [PATCH 67/85] skipSegments return 400 if bad categories --- src/routes/getSkipSegments.ts | 4 +++- test/cases/getSkipSegments.ts | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 30c965b..0af3470 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -306,7 +306,9 @@ async function endpoint(req: Request, res: Response): Promise { res.send(segments); } } catch (err) { - res.status(500).send(); + if (err instanceof SyntaxError) { + res.status(400).send("Categories parameter does not match format requirements."); + } else res.status(500).send(); } } diff --git a/test/cases/getSkipSegments.ts b/test/cases/getSkipSegments.ts index f40203c..6b59812 100644 --- a/test/cases/getSkipSegments.ts +++ b/test/cases/getSkipSegments.ts @@ -227,6 +227,14 @@ describe('getSkipSegments', () => { .catch(err => ("couldn't call endpoint")); }); + it('Should return 400 if bad categories argument', (done: Done) => { + fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&categories=[not-quoted,not-quoted]") + .then(res => { + if (res.status !== 400) done("non 400 respone code: " + res.status); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); it('Should be able send a comma in a query param', () => { fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest,test&category=sponsor") From 8dcc1a4a5352636d79065e3c0759c2e05c8451fc Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 16 Jun 2021 00:33:51 -0400 Subject: [PATCH 68/85] add getSegmentInfo with tests --- src/app.ts | 5 +- src/routes/getSegmentInfo.ts | 75 +++++++++ test/cases/getSegmentInfo.ts | 302 +++++++++++++++++++++++++++++++++++ 3 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 src/routes/getSegmentInfo.ts create mode 100644 test/cases/getSegmentInfo.ts diff --git a/src/app.ts b/src/app.ts index a05a7cc..44a59f6 100644 --- a/src/app.ts +++ b/src/app.ts @@ -28,7 +28,7 @@ import {corsMiddleware} from './middleware/cors'; import {apiCspMiddleware} from './middleware/apiCsp'; import {rateLimitMiddleware} from './middleware/requestRateLimit'; import dumpDatabase, {redirectLink} from './routes/dumpDatabase'; - +import {endpoint as getSegmentInfo} from './routes/getSegmentInfo'; export function createServer(callback: () => void) { // Create a service (the app object is just a callback). @@ -133,6 +133,9 @@ function setupRoutes(app: Express) { //get if user is a vip app.post('/api/segmentShift', postSegmentShift); + //get segment info + app.get('/api/segmentInfo', getSegmentInfo); + if (config.postgres) { app.get('/database', (req, res) => dumpDatabase(req, res, true)); app.get('/database.json', (req, res) => dumpDatabase(req, res, false)); diff --git a/src/routes/getSegmentInfo.ts b/src/routes/getSegmentInfo.ts new file mode 100644 index 0000000..a593d85 --- /dev/null +++ b/src/routes/getSegmentInfo.ts @@ -0,0 +1,75 @@ +import { Request, Response } from 'express'; +import { db } from '../databases/databases'; +import { DBSegment, SegmentUUID } from "../types/segments.model"; + +const isValidSegmentUUID = (str: string): Boolean => /^([a-f0-9]{64}|[a-f0-9]{8}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{12})/.test(str) + +async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise { + try { + return await db.prepare('get', `SELECT * FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) + } catch (err) { + return null + } +} + +async function getSegmentsByUUID(UUIDs: SegmentUUID[]): Promise { + const DBSegments: DBSegment[] = []; + for (let UUID of UUIDs) { + // if UUID is invalid, skip + if (!isValidSegmentUUID(UUID)) { + continue; + } + DBSegments.push(await getSegmentFromDBByUUID(UUID as SegmentUUID)); + } + return DBSegments +} + +async function handleGetSegmentInfo(req: Request, res: Response) { + // If using params instead of JSON, only one UUID can be pulled + let UUIDs = req.query.UUIDs + ? JSON.parse(req.query.UUIDs as string) + : req.query.UUID + ? [req.query.UUID] + : null + // deduplicate with set + UUIDs = [ ...new Set(UUIDs)]; + // if more than 10 entries, slice + if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10); + if (!Array.isArray(UUIDs) || !UUIDs) { + res.status(400).send("UUIDs parameter does not match format requirements."); + return false; + } + const DBSegments = await getSegmentsByUUID(UUIDs); + if (DBSegments === null || DBSegments === undefined) { + res.sendStatus(400); + return false; + } + if (DBSegments.length === 0) { + res.sendStatus(404); + return false; + } + return DBSegments; +} + +async function endpoint(req: Request, res: Response): Promise { + try { + const DBSegments = await handleGetSegmentInfo(req, res); + + // If false, res.send has already been called + if (DBSegments) { + //send result + res.send(DBSegments); + } + } catch (err) { + if (err instanceof SyntaxError) { // catch JSON.parse error + res.status(400).send("UUIDs parameter does not match format requirements."); + } else res.status(500).send(); + } +} + +export { + getSegmentFromDBByUUID, + getSegmentsByUUID, + handleGetSegmentInfo, + endpoint +}; diff --git a/test/cases/getSegmentInfo.ts b/test/cases/getSegmentInfo.ts new file mode 100644 index 0000000..0ab9250 --- /dev/null +++ b/test/cases/getSegmentInfo.ts @@ -0,0 +1,302 @@ +import fetch from 'node-fetch'; +import {db} from '../../src/databases/databases'; +import {Done, getbaseURL} from '../utils'; +import {getHash} from '../../src/utils/getHash'; + +const upvotedID = "a000000000000000000000000000000000000000000000000000000000000000" +const downvotedID = "b000000000000000000000000000000000000000000000000000000000000000" +const lockedupID = "c000000000000000000000000000000000000000000000000000000000000000" +const infvotesID = "d000000000000000000000000000000000000000000000000000000000000000" +const shadowhiddenID = "e000000000000000000000000000000000000000000000000000000000000000" +const lockeddownID = "f000000000000000000000000000000000000000000000000000000000000000" +const hiddenID = "1000000000000000000000000000000000000000000000000000000000000000" +const fillerID1 = "1100000000000000000000000000000000000000000000000000000000000000" +const fillerID2 = "1200000000000000000000000000000000000000000000000000000000000000" +const fillerID3 = "1300000000000000000000000000000000000000000000000000000000000000" +const fillerID4 = "1400000000000000000000000000000000000000000000000000000000000000" +const fillerID5 = "1500000000000000000000000000000000000000000000000000000000000000" +const oldID = "a0000000-0000-0000-0000-000000000000" + +describe('getSegmentInfo', () => { + before(async () => { + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "hidden", "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('upvoted', 1, 10, 2, 0, '" + upvotedID+ "', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, 0, '" + getHash('upvoted', 1) + "')"); + await db.prepare("run", startOfQuery + "('downvoted', 1, 10, -2, 0, '" + downvotedID+ "', 'testman', 0, 50, 'sponsor', 'YouTube', 120, 0, 0, '" + getHash('downvoted', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked-up', 1, 10, 2, 1, '"+ lockedupID +"', 'testman', 0, 50, 'sponsor', 'YouTube', 101, 0, 0, '" + getHash('locked-up', 1) + "')"); + await db.prepare("run", startOfQuery + "('infvotes', 1, 10, 100000, 0, '"+infvotesID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 101, 0, 0, '" + getHash('infvotes', 1) + "')"); + await db.prepare("run", startOfQuery + "('hidden', 1, 10, 2, 0, '"+hiddenID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 1, 0, '" + getHash('hidden', 1) + "')"); + await db.prepare("run", startOfQuery + "('shadowhidden', 1, 10, 2, 0, '"+shadowhiddenID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, 1, '" + getHash('shadowhidden', 1) + "')"); + await db.prepare("run", startOfQuery + "('locked-down', 1, 10, -2, 1, '"+lockeddownID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, 0, '" + getHash('locked-down', 1) + "')"); + await db.prepare("run", startOfQuery + "('oldID', 1, 10, 1, 0, '"+oldID+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('oldID', 1) + "')"); + + await db.prepare("run", startOfQuery + "('filler', 1, 2, 1, 0, '"+fillerID1+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')"); + await db.prepare("run", startOfQuery + "('filler', 2, 3, 1, 0, '"+fillerID2+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')"); + await db.prepare("run", startOfQuery + "('filler', 3, 4, 1, 0, '"+fillerID3+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')"); + await db.prepare("run", startOfQuery + "('filler', 4, 5, 1, 0, '"+fillerID4+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')"); + await db.prepare("run", startOfQuery + "('filler', 5, 6, 1, 0, '"+fillerID5+"', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, 0, '" + getHash('filler', 1) + "')"); + }); + + it('Should be able to retreive upvoted segment', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${upvotedID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "upvoted" && data[0].votes === 2) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive downvoted segment', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${downvotedID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "downvoted" && data[0].votes === -2) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive locked up segment', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${lockedupID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "locked-up" && data[0].locked === 1 && data[0].votes === 2) { + done(); + } else { + done ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive infinite vote segment', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${infvotesID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "infvotes" && data[0].votes === 100000) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive shadowhidden segment', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${shadowhiddenID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "shadowhidden" && data[0].shadowHidden === 1) { + done(); + } else { + done ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive locked down segment', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${lockeddownID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "locked-down" && data[0].votes === -2 && data[0].locked === 1) { + done(); + } else { + done ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive hidden segment', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${hiddenID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "hidden" && data[0].hidden === 1) { + done(); + } else { + done ("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive segment with old ID', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${oldID}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "oldID" && data[0].votes === 1) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive single segment in array', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}"]`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 1 && data[0].videoID === "upvoted" && data[0].votes === 2) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be able to retreive multiple segments in array', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}", "${downvotedID}"]`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 2 && + (data[0].videoID === "upvoted" && data[0].votes === 2) && + (data[1].videoID === "downvoted" && data[1].votes === -2)) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should be possible to send unexpected query parameters', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${upvotedID}&fakeparam=hello&category=sponsor`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data[0].videoID === "upvoted" && data[0].votes === 2) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => "Couldn't call endpoint"); + }); + + it('Should return 404 if array passed to UUID', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=["${upvotedID}", "${downvotedID}"]`) + .then(res => { + if (res.status !== 404) done("non 404 respone code: " + res.status); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should return 400 if bad array passed to UUIDs', (done: Done) => { + fetch(getbaseURL() + "/api/segmentInfo?UUIDs=[not-quoted,not-quoted]") + .then(res => { + if (res.status !== 400) done("non 404 respone code: " + res.status); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should return 404 if bad UUID passed', (done: Done) => { + fetch(getbaseURL() + "/api/segmentInfo?UUID=notarealuuid") + .then(res => { + if (res.status !== 404) done("non 404 respone code: " + res.status); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should return 404 if bad UUIDs passed in array', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["notarealuuid", "anotherfakeuuid"]`) + .then(res => { + if (res.status !== 404) done("non 404 respone code: " + res.status); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should return good UUID when mixed with bad UUIDs', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}", "anotherfakeuuid"]`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 1 && data[0].videoID === "upvoted" && data[0].votes === 2) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should cut off array at 10', (done: Done) => { + const filledIDArray = `["${upvotedID}", "${downvotedID}", "${lockedupID}", "${shadowhiddenID}", "${lockeddownID}", "${hiddenID}", "${fillerID1}", "${fillerID2}", "${fillerID3}", "${fillerID4}", "${fillerID5}"]` + fetch(getbaseURL() + `/api/segmentInfo?UUIDs=${filledIDArray}`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + // last segment should be fillerID4 + if (data.length === 10 && data[0].videoID === "upvoted" && data[0].votes === 2 && data[9].videoID === "filler" && data[9].UUID === fillerID4) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should not duplicate reponses', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${upvotedID}", "${downvotedID}"]`) + .then(async res => { + if (res.status !== 200) done("Status code was: " + res.status); + else { + const data = await res.json(); + if (data.length === 2 && data[0].videoID === "upvoted" && data[0].votes === 2 && data[1].videoID === "downvoted" && data[1].votes === -2) { + done(); + } else { + done("Received incorrect body: " + (await res.text())); + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); +}); From 7fe787c5ab1ab54d6dbb309be0f64db40bcb5853 Mon Sep 17 00:00:00 2001 From: Michael C Date: Wed, 16 Jun 2021 00:53:34 -0400 Subject: [PATCH 69/85] remove extra properties --- src/routes/getSegmentInfo.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/routes/getSegmentInfo.ts b/src/routes/getSegmentInfo.ts index a593d85..af7063d 100644 --- a/src/routes/getSegmentInfo.ts +++ b/src/routes/getSegmentInfo.ts @@ -6,7 +6,11 @@ const isValidSegmentUUID = (str: string): Boolean => /^([a-f0-9]{64}|[a-f0-9]{8} async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise { try { - return await db.prepare('get', `SELECT * FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) + return await db.prepare('get', + `SELECT "videoID", "startTime", "endTime", "votes", "locked", + "UUID", "userID", "timeSubmitted", "views", "category", + "service", "videoDuration", "hidden", "reputation", "shadowHidden" FROM "sponsorTimes" + WHERE "UUID" = ?`, [UUID]) } catch (err) { return null } From 20ae560bb1ac42c73738710ad66014151b9c59bd Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Wed, 16 Jun 2021 13:23:25 -0400 Subject: [PATCH 70/85] Add semicolons --- src/routes/getSegmentInfo.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/routes/getSegmentInfo.ts b/src/routes/getSegmentInfo.ts index af7063d..4262998 100644 --- a/src/routes/getSegmentInfo.ts +++ b/src/routes/getSegmentInfo.ts @@ -10,9 +10,9 @@ async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise { `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "hidden", "reputation", "shadowHidden" FROM "sponsorTimes" - WHERE "UUID" = ?`, [UUID]) + WHERE "UUID" = ?`, [UUID]); } catch (err) { - return null + return null; } } @@ -25,7 +25,7 @@ async function getSegmentsByUUID(UUIDs: SegmentUUID[]): Promise { } DBSegments.push(await getSegmentFromDBByUUID(UUID as SegmentUUID)); } - return DBSegments + return DBSegments; } async function handleGetSegmentInfo(req: Request, res: Response) { @@ -34,7 +34,7 @@ async function handleGetSegmentInfo(req: Request, res: Response) { ? JSON.parse(req.query.UUIDs as string) : req.query.UUID ? [req.query.UUID] - : null + : null; // deduplicate with set UUIDs = [ ...new Set(UUIDs)]; // if more than 10 entries, slice From b9bcc35dd279155b46c3ffc4208ae8933c3a2d82 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 17 Jun 2021 19:08:36 -0400 Subject: [PATCH 71/85] Allow removing warnings created by anyone --- src/routes/postWarning.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postWarning.ts b/src/routes/postWarning.ts index a5e720c..1734012 100644 --- a/src/routes/postWarning.ts +++ b/src/routes/postWarning.ts @@ -32,7 +32,7 @@ export async function postWarning(req: Request, res: Response) { return; } } else { - await db.prepare('run', 'UPDATE "warnings" SET "enabled" = 0 WHERE "userID" = ? AND "issuerUserID" = ?', [userID, issuerUserID]); + await db.prepare('run', 'UPDATE "warnings" SET "enabled" = 0 WHERE "userID" = ?', [userID]); resultStatus = "removed from"; } From 1dcb63f2cc30efb2f78b97542b05231c784bcef1 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 17 Jun 2021 19:09:24 -0400 Subject: [PATCH 72/85] Fix typo in test --- test/cases/postWarning.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cases/postWarning.ts b/test/cases/postWarning.ts index aaa5664..7089fbd 100644 --- a/test/cases/postWarning.ts +++ b/test/cases/postWarning.ts @@ -101,7 +101,7 @@ describe('postWarning', () => { .catch(err => done(err)); }); - it('Should not be able to create warning if vip (exp 403)', (done: Done) => { + it('Should not be able to create warning if not vip (exp 403)', (done: Done) => { let json = { issuerUserID: 'warning-not-vip', userID: 'warning-1', From c92e44bb1da611132823a170afe824f4b30fd796 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 18 Jun 2021 14:43:59 -0400 Subject: [PATCH 73/85] made 400/404 behaviour consistent with API docs --- src/routes/getSegmentInfo.ts | 8 +++++--- test/cases/getSegmentInfo.ts | 22 ++++++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/routes/getSegmentInfo.ts b/src/routes/getSegmentInfo.ts index 4262998..e9711ca 100644 --- a/src/routes/getSegmentInfo.ts +++ b/src/routes/getSegmentInfo.ts @@ -44,12 +44,14 @@ async function handleGetSegmentInfo(req: Request, res: Response) { return false; } const DBSegments = await getSegmentsByUUID(UUIDs); - if (DBSegments === null || DBSegments === undefined) { + // all uuids failed lookup + if (DBSegments.length === 0) { res.sendStatus(400); return false; } - if (DBSegments.length === 0) { - res.sendStatus(404); + // uuids valid but not found + if (DBSegments[0] === null || DBSegments[0] === undefined) { + res.sendStatus(400); return false; } return DBSegments; diff --git a/test/cases/getSegmentInfo.ts b/test/cases/getSegmentInfo.ts index 0ab9250..f2c1299 100644 --- a/test/cases/getSegmentInfo.ts +++ b/test/cases/getSegmentInfo.ts @@ -3,6 +3,7 @@ import {db} from '../../src/databases/databases'; import {Done, getbaseURL} from '../utils'; import {getHash} from '../../src/utils/getHash'; +const ENOENTID = "0000000000000000000000000000000000000000000000000000000000000000" const upvotedID = "a000000000000000000000000000000000000000000000000000000000000000" const downvotedID = "b000000000000000000000000000000000000000000000000000000000000000" const lockedupID = "c000000000000000000000000000000000000000000000000000000000000000" @@ -214,10 +215,10 @@ describe('getSegmentInfo', () => { .catch(err => "Couldn't call endpoint"); }); - it('Should return 404 if array passed to UUID', (done: Done) => { + it('Should return 400 if array passed to UUID', (done: Done) => { fetch(getbaseURL() + `/api/segmentInfo?UUID=["${upvotedID}", "${downvotedID}"]`) .then(res => { - if (res.status !== 404) done("non 404 respone code: " + res.status); + if (res.status !== 400) done("non 400 respone code: " + res.status); else done(); // pass }) .catch(err => ("couldn't call endpoint")); @@ -232,19 +233,19 @@ describe('getSegmentInfo', () => { .catch(err => ("couldn't call endpoint")); }); - it('Should return 404 if bad UUID passed', (done: Done) => { + it('Should return 400 if bad UUID passed', (done: Done) => { fetch(getbaseURL() + "/api/segmentInfo?UUID=notarealuuid") .then(res => { - if (res.status !== 404) done("non 404 respone code: " + res.status); + if (res.status !== 400) done("non 400 respone code: " + res.status); else done(); // pass }) .catch(err => ("couldn't call endpoint")); }); - it('Should return 404 if bad UUIDs passed in array', (done: Done) => { + it('Should return 400 if bad UUIDs passed in array', (done: Done) => { fetch(getbaseURL() + `/api/segmentInfo?UUIDs=["notarealuuid", "anotherfakeuuid"]`) .then(res => { - if (res.status !== 404) done("non 404 respone code: " + res.status); + if (res.status !== 400) done("non 400 respone code: " + res.status); else done(); // pass }) .catch(err => ("couldn't call endpoint")); @@ -299,4 +300,13 @@ describe('getSegmentInfo', () => { }) .catch(err => ("couldn't call endpoint")); }); + + it('Should return 400 if UUID not found', (done: Done) => { + fetch(getbaseURL() + `/api/segmentInfo?UUID=${ENOENTID}`) + .then(res => { + if (res.status !== 400) done("non 400 respone code: " + res.status); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); }); From 4963f4dc088c3321781d314552e8007e9d915d44 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 18 Jun 2021 15:33:14 -0400 Subject: [PATCH 74/85] style fixes --- src/routes/getSegmentInfo.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/routes/getSegmentInfo.ts b/src/routes/getSegmentInfo.ts index e9711ca..354102d 100644 --- a/src/routes/getSegmentInfo.ts +++ b/src/routes/getSegmentInfo.ts @@ -20,9 +20,7 @@ async function getSegmentsByUUID(UUIDs: SegmentUUID[]): Promise { const DBSegments: DBSegment[] = []; for (let UUID of UUIDs) { // if UUID is invalid, skip - if (!isValidSegmentUUID(UUID)) { - continue; - } + if (!isValidSegmentUUID(UUID)) continue; DBSegments.push(await getSegmentFromDBByUUID(UUID as SegmentUUID)); } return DBSegments; @@ -45,7 +43,7 @@ async function handleGetSegmentInfo(req: Request, res: Response) { } const DBSegments = await getSegmentsByUUID(UUIDs); // all uuids failed lookup - if (DBSegments.length === 0) { + if (!DBSegments?.length) { res.sendStatus(400); return false; } From 04da532962755f6ce1c29fa414c0156c78f57d54 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 18 Jun 2021 16:38:24 -0400 Subject: [PATCH 75/85] implement #253 --- src/routes/postClearCache.ts | 55 ++++++++++++++++++++++++++++++++++++ src/utils/queryCacher.ts | 5 ++-- 2 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 src/routes/postClearCache.ts diff --git a/src/routes/postClearCache.ts b/src/routes/postClearCache.ts new file mode 100644 index 0000000..a4c3c21 --- /dev/null +++ b/src/routes/postClearCache.ts @@ -0,0 +1,55 @@ +import { Logger } from '../utils/logger'; +import { db } from '../databases/databases'; +import { getHash } from '../utils/getHash'; +import { Request, Response } from 'express'; +import { Service, VideoID } from '../types/segments.model'; +import { QueryCacher } from '../utils/queryCacher'; +import { UserID } from '../types/user.model'; + +export async function postClearCache(req: Request, res: Response) { + const videoID = req.query.videoID as VideoID; + let userID = req.query.userID as UserID; + let service = req.query.service as Service ?? Service.YouTube; + + const invalidFields = []; + if (typeof videoID !== 'string') { + invalidFields.push('videoID'); + } + if (typeof userID !== 'string') { + invalidFields.push('userID'); + } + + if (invalidFields.length !== 0) { + // invalid request + const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, ''); + res.status(400).send(`No valid ${fields} field(s) provided`); + return false; + } + + // hash the userID + userID = getHash(userID); + // hash videoID + const hashedVideoID = getHash(videoID, 1); + + const isVIP = (await db.prepare("get", `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [userID])).userCount > 0; + // Ensure user is a VIP + if (!isVIP) { + Logger.warn("Permission violation: User " + userID + " attempted to clear cache for video " + videoID + "."); + res.status(403).json({"message": "Not a VIP"}); + return false; + } + + try { + QueryCacher.clearVideoCache({ + videoID, + hashedVideoID, + service + }); + res.status(200).json({ + message: "Cache cleared on video " + videoID + }); + } catch(err) { + res.status(500).send() + return false; + } +} diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index 64cf206..0c4d880 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -22,15 +22,14 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { return data; } -function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID: UserID; }) { +function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }) { if (videoInfo) { redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)); - redis.delAsync(reputationKey(videoInfo.userID)); + if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID)); } } - export const QueryCacher = { get, clearVideoCache From c13bc6cfbdc72092c5175b7348f946ac4f536f40 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 18 Jun 2021 17:46:18 -0400 Subject: [PATCH 76/85] added tests and route --- src/app.ts | 4 ++ src/routes/postClearCache.ts | 6 +-- test/cases/postClearCache.ts | 72 ++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 test/cases/postClearCache.ts diff --git a/src/app.ts b/src/app.ts index 44a59f6..589f352 100644 --- a/src/app.ts +++ b/src/app.ts @@ -29,6 +29,7 @@ import {apiCspMiddleware} from './middleware/apiCsp'; import {rateLimitMiddleware} from './middleware/requestRateLimit'; import dumpDatabase, {redirectLink} from './routes/dumpDatabase'; import {endpoint as getSegmentInfo} from './routes/getSegmentInfo'; +import {postClearCache} from './routes/postClearCache'; export function createServer(callback: () => void) { // Create a service (the app object is just a callback). @@ -136,6 +137,9 @@ function setupRoutes(app: Express) { //get segment info app.get('/api/segmentInfo', getSegmentInfo); + //clear cache as VIP + app.post('/api/clearCache', postClearCache) + if (config.postgres) { app.get('/database', (req, res) => dumpDatabase(req, res, true)); app.get('/database.json', (req, res) => dumpDatabase(req, res, false)); diff --git a/src/routes/postClearCache.ts b/src/routes/postClearCache.ts index a4c3c21..c6f6232 100644 --- a/src/routes/postClearCache.ts +++ b/src/routes/postClearCache.ts @@ -9,7 +9,9 @@ import { UserID } from '../types/user.model'; export async function postClearCache(req: Request, res: Response) { const videoID = req.query.videoID as VideoID; let userID = req.query.userID as UserID; - let service = req.query.service as Service ?? Service.YouTube; + const service = req.query.service as Service ?? Service.YouTube; + // hash the userID as early as possible + userID = getHash(userID); const invalidFields = []; if (typeof videoID !== 'string') { @@ -26,8 +28,6 @@ export async function postClearCache(req: Request, res: Response) { return false; } - // hash the userID - userID = getHash(userID); // hash videoID const hashedVideoID = getHash(videoID, 1); diff --git a/test/cases/postClearCache.ts b/test/cases/postClearCache.ts new file mode 100644 index 0000000..ffedaae --- /dev/null +++ b/test/cases/postClearCache.ts @@ -0,0 +1,72 @@ +import fetch from 'node-fetch'; +import {Done, getbaseURL} from '../utils'; +import {db} from '../../src/databases/databases'; +import {getHash} from '../../src/utils/getHash'; + +describe('postClearCache', () => { + before(async () => { + await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("clearing-vip") + "')"); + let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; + await db.prepare("run", startOfQuery + "('clear-test', 0, 1, 2, 'clear-uuid', 'testman', 0, 50, 'sponsor', 0, '" + getHash('clear-test', 1) + "')"); + }); + + it('Should be able to clear cache for existing video', (done: Done) => { + fetch(getbaseURL() + + "/api/clearCache?userID=clearing-vip&videoID=clear-test", { + method: 'POST' + }) + .then(res => { + if (res.status === 200) done(); + else done("Status code was " + res.status); + }) + .catch(err => done(err)); + }); + + it('Should be able to clear cache for nonexistent video', (done: Done) => { + fetch(getbaseURL() + + "/api/clearCache?userID=clearing-vip&videoID=dne-video", { + method: 'POST' + }) + .then(res => { + if (res.status === 200) done(); + else done("Status code was " + res.status); + }) + .catch(err => done(err)); + }); + + it('Should get 403 as non-vip', (done: Done) => { + fetch(getbaseURL() + + "/api/clearCache?userID=regular-user&videoID=clear-tes", { + method: 'POST' + }) + .then(async res => { + if (res.status !== 403) done('non 403 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => done(err)); + }); + + it('Should give 400 with missing videoID', (done: Done) => { + fetch(getbaseURL() + + "/api/clearCache?userID=clearing-vip", { + method: 'POST' + }) + .then(async res => { + if (res.status !== 400) done('non 400 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => done(err)); + }); + + it('Should give 400 with missing userID', (done: Done) => { + fetch(getbaseURL() + + "/api/clearCache?userID=clearing-vip", { + method: 'POST' + }) + .then(async res => { + if (res.status !== 400) done('non 400 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => done(err)); + }); +}); From b84241c6ad027b54e82ea62a9e0b6e33870e213f Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 18 Jun 2021 18:44:43 -0400 Subject: [PATCH 77/85] use isUserVIP instead --- src/routes/postClearCache.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/routes/postClearCache.ts b/src/routes/postClearCache.ts index c6f6232..f12c439 100644 --- a/src/routes/postClearCache.ts +++ b/src/routes/postClearCache.ts @@ -1,17 +1,16 @@ import { Logger } from '../utils/logger'; -import { db } from '../databases/databases'; +import { HashedUserID, UserID } from '../types/user.model'; import { getHash } from '../utils/getHash'; import { Request, Response } from 'express'; import { Service, VideoID } from '../types/segments.model'; import { QueryCacher } from '../utils/queryCacher'; -import { UserID } from '../types/user.model'; +import { isUserVIP } from '../utils/isUserVIP'; +import { VideoIDHash } from "../types/segments.model"; export async function postClearCache(req: Request, res: Response) { const videoID = req.query.videoID as VideoID; let userID = req.query.userID as UserID; const service = req.query.service as Service ?? Service.YouTube; - // hash the userID as early as possible - userID = getHash(userID); const invalidFields = []; if (typeof videoID !== 'string') { @@ -28,13 +27,14 @@ export async function postClearCache(req: Request, res: Response) { return false; } + // hash the userID as early as possible + const hashedUserID: HashedUserID = getHash(userID); // hash videoID - const hashedVideoID = getHash(videoID, 1); + const hashedVideoID: VideoIDHash = getHash(videoID, 1); - const isVIP = (await db.prepare("get", `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [userID])).userCount > 0; // Ensure user is a VIP - if (!isVIP) { - Logger.warn("Permission violation: User " + userID + " attempted to clear cache for video " + videoID + "."); + if (!await isUserVIP(hashedUserID)){ + Logger.warn("Permission violation: User " + hashedUserID + " attempted to clear cache for video " + videoID + "."); res.status(403).json({"message": "Not a VIP"}); return false; } From 183462ff850c9fce9fb2a098c2f4d37612cab2d7 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Fri, 18 Jun 2021 18:49:37 -0400 Subject: [PATCH 78/85] Add brackets --- src/routes/postClearCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/postClearCache.ts b/src/routes/postClearCache.ts index f12c439..c3bc1ca 100644 --- a/src/routes/postClearCache.ts +++ b/src/routes/postClearCache.ts @@ -33,7 +33,7 @@ export async function postClearCache(req: Request, res: Response) { const hashedVideoID: VideoIDHash = getHash(videoID, 1); // Ensure user is a VIP - if (!await isUserVIP(hashedUserID)){ + if (!(await isUserVIP(hashedUserID))){ Logger.warn("Permission violation: User " + hashedUserID + " attempted to clear cache for video " + videoID + "."); res.status(403).json({"message": "Not a VIP"}); return false; From 47289db13ee4d7dc7ef7b231d96e35bedccbe906 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 18 Jun 2021 20:57:01 -0400 Subject: [PATCH 79/85] clearCache after shadowban --- src/routes/shadowBanUser.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/routes/shadowBanUser.ts b/src/routes/shadowBanUser.ts index 17b9798..3f5e541 100644 --- a/src/routes/shadowBanUser.ts +++ b/src/routes/shadowBanUser.ts @@ -2,6 +2,9 @@ import {db, privateDB} from '../databases/databases'; import {getHash} from '../utils/getHash'; import {Request, Response} from 'express'; import { config } from '../config'; +import { Category, Service, VideoID, VideoIDHash } from '../types/segments.model'; +import { UserID } from '../types/user.model'; +import { QueryCacher } from '../utils/queryCacher'; export async function shadowBanUser(req: Request, res: Response) { const userID = req.query.userID as string; @@ -49,6 +52,11 @@ export async function shadowBanUser(req: Request, res: Response) { await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")}) AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]); + // clear cache for all old videos + (await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID])) + .map((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => { + QueryCacher.clearVideoCache(videoInfo); + }); } } else if (!enabled && row.userCount > 0) { //remove them from the shadow ban list @@ -64,7 +72,13 @@ export async function shadowBanUser(req: Request, res: Response) { await Promise.all(allSegments.filter((item: {uuid: string}) => { return segmentsToIgnore.indexOf(item) === -1; - }).map((UUID: string) => { + }).map(async (UUID: string) => { + // collect list for unshadowbanning + (await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ? AND "shadowHidden" = 1 AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID])) + .map((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => { + QueryCacher.clearVideoCache(videoInfo); + }) + // unhide return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID]); })); } From 85a30369c4ad6b55211713d068f22304439a2705 Mon Sep 17 00:00:00 2001 From: DetachHead Date: Sat, 19 Jun 2021 17:59:18 +1000 Subject: [PATCH 80/85] remove async modifier which is no longer allowed in interfaces --- src/databases/IDatabase.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databases/IDatabase.ts b/src/databases/IDatabase.ts index 33244b0..4a93261 100644 --- a/src/databases/IDatabase.ts +++ b/src/databases/IDatabase.ts @@ -1,5 +1,5 @@ export interface IDatabase { - async init(): Promise; + init(): Promise; prepare(type: QueryType, query: string, params?: any[]): Promise; } From 96015d402bd6b6538ef40bb7b913a48141237b2a Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 20 Jun 2021 12:57:32 -0400 Subject: [PATCH 81/85] Make reputation take into account self downvotes --- src/routes/getSkipSegments.ts | 4 ++-- src/utils/reputation.ts | 15 +++++++++++++-- test/cases/getUserInfo.ts | 2 +- test/cases/reputation.ts | 18 +++++++++++++++++- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 0af3470..3889e32 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -173,8 +173,8 @@ function getWeightedRandomChoice(choices: T[], amountOf //The 3 makes -2 the minimum votes before being ignored completely //this can be changed if this system increases in popularity. - const weight = Math.exp(choice.votes * Math.min(1, choice.reputation + 1) + 3 + boost); - totalWeight += weight; + const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost); + totalWeight += Math.max(weight, 0); return {...choice, weight}; }); diff --git a/src/utils/reputation.ts b/src/utils/reputation.ts index d40857c..3fe42c1 100644 --- a/src/utils/reputation.ts +++ b/src/utils/reputation.ts @@ -6,6 +6,7 @@ import { reputationKey } from "./redisKeys"; interface ReputationDBResult { totalSubmissions: number, downvotedSubmissions: number, + nonSelfDownvotedSubmissions: number, upvotedSum: number, lockedSum: number, oldUpvotedSubmissions: number @@ -17,10 +18,15 @@ export async function getReputation(userID: UserID): Promise { const fetchFromDB = () => db.prepare("get", `SELECT COUNT(*) AS "totalSubmissions", SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions", + SUM(CASE WHEN "votes" < 0 AND "videoID" NOT IN + (SELECT b."videoID" FROM "sponsorTimes" as b + WHERE b."userID" = ? + AND b."votes" > 0 AND b."category" = "a"."category" AND b."videoID" = "a"."videoID" LIMIT 1) + THEN 1 ELSE 0 END) AS "nonSelfDownvotedSubmissions", SUM(CASE WHEN "votes" > 0 AND "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "upvotedSum", SUM(locked) AS "lockedSum", SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions" - FROM "sponsorTimes" WHERE "userID" = ?`, [pastDate, userID]) as Promise; + FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, pastDate, userID]) as Promise; const result = await QueryCacher.get(fetchFromDB, reputationKey(userID)); @@ -31,7 +37,12 @@ export async function getReputation(userID: UserID): Promise { const downvoteRatio = result.downvotedSubmissions / result.totalSubmissions; if (downvoteRatio > 0.3) { - return convertRange(downvoteRatio, 0.3, 1, -0.5, -1.5); + return convertRange(Math.min(downvoteRatio, 0.7), 0.3, 0.7, -0.5, -2.5); + } + + const nonSelfDownvoteRatio = result.nonSelfDownvotedSubmissions / result.totalSubmissions; + if (nonSelfDownvoteRatio > 0.05) { + return convertRange(Math.min(nonSelfDownvoteRatio, 0.4), 0.05, 0.4, -0.5, -2.5); } if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) { diff --git a/test/cases/getUserInfo.ts b/test/cases/getUserInfo.ts index 06c53ec..c20a0ea 100644 --- a/test/cases/getUserInfo.ts +++ b/test/cases/getUserInfo.ts @@ -60,7 +60,7 @@ describe('getUserInfo', () => { done('Returned incorrect segmentCount "' + data.segmentCount + '"'); } else if (data.ignoredSegmentCount !== 2) { done('Returned incorrect ignoredSegmentCount "' + data.ignoredSegmentCount + '"'); - } else if (Math.abs(data.reputation - -0.928) > 0.001) { + } else if (data.reputation !== -2) { done('Returned incorrect reputation "' + data.reputation + '"'); } else if (data.lastSegmentID !== "uuid000005") { done('Returned incorrect last segment "' + data.lastSegmentID + '"'); diff --git a/test/cases/reputation.ts b/test/cases/reputation.ts index 7fe9226..77b4179 100644 --- a/test/cases/reputation.ts +++ b/test/cases/reputation.ts @@ -6,6 +6,7 @@ import { getReputation } from '../../src/utils/reputation'; const userIDLowSubmissions = "reputation-lowsubmissions" as UserID; const userIDHighDownvotes = "reputation-highdownvotes" as UserID; +const userIDHighNonSelfDownvotes = "reputation-highnonselfdownvotes" as UserID; const userIDNewSubmissions = "reputation-newsubmissions" as UserID; const userIDLowSum = "reputation-lowsum" as UserID; const userIDHighRepBeforeManualVote = "reputation-oldhighrep" as UserID; @@ -29,6 +30,17 @@ describe('reputation', () => { await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-uuid-5', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-6', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-uuid-7', '${getHash(userIDHighDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + + // Each downvote is on a different video (ie. they didn't resubmit to fix their downvote) + await db.prepare("run", startOfQuery + `('${videoID}A', 1, 11, 2, 0, 'reputation-1-1-uuid-0', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + // Different category, same video + await db.prepare("run", startOfQuery + `('${videoID}A', 1, 11, -2, 0, 'reputation-1-1-uuid-1', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'intro', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-2', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-3', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-4', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, -1, 0, 'reputation-1-1-uuid-5', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-6', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); + await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 0, 0, 'reputation-1-1-uuid-7', '${getHash(userIDHighNonSelfDownvotes)}', 1606240000000, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-0', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); await db.prepare("run", startOfQuery + `('${videoID}', 1, 11, 2, 0, 'reputation-2-uuid-1', '${getHash(userIDNewSubmissions)}', ${Date.now()}, 50, 'sponsor', 'YouTube', 100, 0, 0, '${getHash(videoID, 1)}')`); @@ -81,7 +93,11 @@ describe('reputation', () => { }); it("user with high downvote ratio", async () => { - assert.strictEqual(await getReputation(getHash(userIDHighDownvotes)), -0.9642857142857144); + assert.strictEqual(await getReputation(getHash(userIDHighDownvotes)), -2.125); + }); + + it("user with high non self downvote ratio", async () => { + assert.strictEqual(await getReputation(getHash(userIDHighNonSelfDownvotes)), -1.6428571428571428); }); it("user with mostly new submissions", async () => { From df1d742339e6c09aca64c9e043f297dbdbb762b6 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 20 Jun 2021 13:04:52 -0400 Subject: [PATCH 82/85] Use for each instead of map --- src/routes/shadowBanUser.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/routes/shadowBanUser.ts b/src/routes/shadowBanUser.ts index 3f5e541..948367f 100644 --- a/src/routes/shadowBanUser.ts +++ b/src/routes/shadowBanUser.ts @@ -52,11 +52,13 @@ export async function shadowBanUser(req: Request, res: Response) { await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")}) AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]); + // clear cache for all old videos (await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID])) - .map((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => { - QueryCacher.clearVideoCache(videoInfo); - }); + .forEach((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => { + QueryCacher.clearVideoCache(videoInfo); + } + ); } } else if (!enabled && row.userCount > 0) { //remove them from the shadow ban list @@ -75,10 +77,11 @@ export async function shadowBanUser(req: Request, res: Response) { }).map(async (UUID: string) => { // collect list for unshadowbanning (await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ? AND "shadowHidden" = 1 AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID])) - .map((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => { + .forEach((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => { QueryCacher.clearVideoCache(videoInfo); - }) - // unhide + } + ); + return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID]); })); } From 48d88614fbeaa9447f762f50a90d1f0408b0c033 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 20 Jun 2021 13:41:35 -0400 Subject: [PATCH 83/85] Move shadow ban list to public db Warning: Migration is not automatic --- databases/_private.db.sql | 3 --- databases/_sponsorTimes_indexes.sql | 7 +++++++ databases/_upgrade_sponsorTimes_14.sql | 9 +++++++++ src/routes/getTopUsers.ts | 8 ++------ src/routes/postSkipSegments.ts | 2 +- src/routes/shadowBanUser.ts | 10 +++++----- src/routes/voteOnSponsorTime.ts | 2 +- test/cases/shadowBanUser.ts | 12 ++++++------ test/cases/unBan.ts | 6 +++--- test/cases/voteOnSponsorTime.ts | 2 +- 10 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_14.sql diff --git a/databases/_private.db.sql b/databases/_private.db.sql index a66ed9b..0da7016 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -1,8 +1,5 @@ BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "shadowBannedUsers" ( - "userID" TEXT NOT NULL -); CREATE TABLE IF NOT EXISTS "votes" ( "UUID" TEXT NOT NULL, "userID" TEXT NOT NULL, diff --git a/databases/_sponsorTimes_indexes.sql b/databases/_sponsorTimes_indexes.sql index 2ad3f75..40fe33e 100644 --- a/databases/_sponsorTimes_indexes.sql +++ b/databases/_sponsorTimes_indexes.sql @@ -63,4 +63,11 @@ CREATE INDEX IF NOT EXISTS "noSegments_videoID" CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public" ON public."categoryVotes" USING btree ("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- shadowBannedUsers + +CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index" + ON public."shadowBannedUsers" USING btree + ("userID" COLLATE pg_catalog."default" ASC NULLS LAST) TABLESPACE pg_default; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_14.sql b/databases/_upgrade_sponsorTimes_14.sql new file mode 100644 index 0000000..c21b83c --- /dev/null +++ b/databases/_upgrade_sponsorTimes_14.sql @@ -0,0 +1,9 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "shadowBannedUsers" ( + "userID" TEXT NOT NULL +); + +UPDATE "config" SET value = 14 WHERE key = 'version'; + +COMMIT; diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index 347791b..2826e25 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -29,8 +29,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole SUM("votes") as "userVotes", ` + additionalFields + `IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" - LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID" - WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL + LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" + WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20 ORDER BY "${sortBy}" DESC LIMIT 100`, []); @@ -71,10 +71,6 @@ export async function getTopUsers(req: Request, res: Response) { return; } - //TODO: remove. This is broken for now - res.status(200).send(); - return; - //setup which sort type to use let sortBy = ''; if (sortType == 0) { diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index b79b855..22515e1 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -485,7 +485,7 @@ export async function postSkipSegments(req: Request, res: Response) { } //check to see if this user is shadowbanned - const shadowBanRow = await privateDB.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); + const shadowBanRow = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); let shadowBanned = shadowBanRow.userCount; diff --git a/src/routes/shadowBanUser.ts b/src/routes/shadowBanUser.ts index 948367f..f02c9a0 100644 --- a/src/routes/shadowBanUser.ts +++ b/src/routes/shadowBanUser.ts @@ -1,4 +1,4 @@ -import {db, privateDB} from '../databases/databases'; +import {db} from '../databases/databases'; import {getHash} from '../utils/getHash'; import {Request, Response} from 'express'; import { config } from '../config'; @@ -39,13 +39,13 @@ export async function shadowBanUser(req: Request, res: Response) { if (userID) { //check to see if this user is already shadowbanned - const row = await privateDB.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); + const row = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); if (enabled && row.userCount == 0) { //add them to the shadow ban list //add it to the table - await privateDB.prepare('run', `INSERT INTO "shadowBannedUsers" VALUES(?)`, [userID]); + await db.prepare('run', `INSERT INTO "shadowBannedUsers" VALUES(?)`, [userID]); //find all previous submissions and hide them if (unHideOldSubmissions) { @@ -62,7 +62,7 @@ export async function shadowBanUser(req: Request, res: Response) { } } else if (!enabled && row.userCount > 0) { //remove them from the shadow ban list - await privateDB.prepare('run', `DELETE FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); + await db.prepare('run', `DELETE FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]); //find all previous submissions and unhide them if (unHideOldSubmissions) { @@ -106,7 +106,7 @@ export async function shadowBanUser(req: Request, res: Response) { } } /*else if (!enabled && row.userCount > 0) { // //remove them from the shadow ban list - // await privateDB.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]); + // await db.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]); // //find all previous submissions and unhide them // if (unHideOldSubmissions) { diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 147139e..c985694 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -404,7 +404,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) { const ableToVote = isVIP || (!(isOwnSubmission && incrementAmount > 0) && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined - && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined + && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined) && !finalResponse.blockVote && finalResponse.finalStatus === 200; diff --git a/test/cases/shadowBanUser.ts b/test/cases/shadowBanUser.ts index 5d6af7e..29dee8e 100644 --- a/test/cases/shadowBanUser.ts +++ b/test/cases/shadowBanUser.ts @@ -17,7 +17,7 @@ describe('shadowBanUser', () => { db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, 'shadow-3-uuid-0', 'shadowBanned3', 0, 50, 'sponsor', 'YouTube', 100, 0, 1, '" + getHash('testtesttest', 1) + "')"); db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, 'shadow-3-uuid-0-1', 'shadowBanned3', 0, 50, 'sponsor', 'PeerTube', 120, 0, 1, '" + getHash('testtesttest2', 1) + "')"); db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, 'shadow-3-uuid-2', 'shadowBanned3', 0, 50, 'intro', 'YouTube', 101, 0, 1, '" + getHash('testtesttest', 1) + "')"); - privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('shadowBanned3')`); + db.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('shadowBanned3')`); db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("shadow-ban-vip") + "')"); }); @@ -31,7 +31,7 @@ describe('shadowBanUser', () => { if (res.status !== 200) done("Status code was: " + res.status); else { const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned", 1]); - const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]); + const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]); if (shadowRow && videoRow?.length === 3) { done(); @@ -51,7 +51,7 @@ describe('shadowBanUser', () => { if (res.status !== 200) done("Status code was: " + res.status); else { const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned", 1]); - const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]); + const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned"]); if (!shadowRow && videoRow?.length === 3) { done(); @@ -71,7 +71,7 @@ describe('shadowBanUser', () => { if (res.status !== 200) done("Status code was: " + res.status); else { const videoRow: {category: string, shadowHidden: number}[] = (await db.prepare('all', `SELECT "shadowHidden", "category" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned2", 1])); - const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]); + const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]); if (shadowRow && 2 == videoRow?.length && 2 === videoRow?.filter((elem) => elem?.category === "sponsor")?.length) { done(); @@ -91,7 +91,7 @@ describe('shadowBanUser', () => { if (res.status !== 200) done("Status code was: " + res.status); else { const videoRow = await db.prepare('all', `SELECT "shadowHidden" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned2", 1]); - const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]); + const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned2"]); if (!shadowRow && videoRow?.length === 0) { done(); @@ -111,7 +111,7 @@ describe('shadowBanUser', () => { if (res.status !== 200) done("Status code was: " + res.status); else { const videoRow = await db.prepare('all', `SELECT "shadowHidden", "category" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = ?`, ["shadowBanned3", 1]); - const shadowRow = await privateDB.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned3"]); + const shadowRow = await db.prepare('get', `SELECT * FROM "shadowBannedUsers" WHERE "userID" = ?`, ["shadowBanned3"]); if (!shadowRow && videoRow?.length === 1 && videoRow[0]?.category === "intro") { done(); diff --git a/test/cases/unBan.ts b/test/cases/unBan.ts index 379fcfc..0f54c8d 100644 --- a/test/cases/unBan.ts +++ b/test/cases/unBan.ts @@ -8,9 +8,9 @@ import { Logger } from '../../src/utils/logger.js'; describe('unBan', () => { before(async () => { - await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testMan-unBan')`); - await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testWoman-unBan')`); - await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testEntity-unBan')`); + await db.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testMan-unBan')`); + await db.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testWoman-unBan')`); + await db.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testEntity-unBan')`); await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-unBan") + "')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')"); diff --git a/test/cases/voteOnSponsorTime.ts b/test/cases/voteOnSponsorTime.ts index b2cbc27..9c9587e 100644 --- a/test/cases/voteOnSponsorTime.ts +++ b/test/cases/voteOnSponsorTime.ts @@ -57,7 +57,7 @@ describe('voteOnSponsorTime', () => { await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser") + "')"); - await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES ('` + getHash("randomID4") + "')"); + await db.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES ('` + getHash("randomID4") + "')"); await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES ('no-sponsor-segments-video', 'someUser', 'sponsor')`); From f6d79616a43a7da74bb3cf3465ab83686b418511 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 20 Jun 2021 13:59:53 -0400 Subject: [PATCH 84/85] Fix leaderboard not working with postgres --- src/routes/getTopUsers.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index 2826e25..6a8abd6 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -28,10 +28,10 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ${maxRewardTimePerSegmentInSeconds} THEN ${maxRewardTimePerSegmentInSeconds} ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved", SUM("votes") as "userVotes", ` + additionalFields + - `IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" + `COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID" WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL - GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20 + GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20 ORDER BY "${sortBy}" DESC LIMIT 100`, []); for (let i = 0; i < rows.length; i++) { From 9351bef61cc332de12fa33cccf28522381e32657 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 20 Jun 2021 15:56:02 -0400 Subject: [PATCH 85/85] Add preview category to leaderboard --- src/routes/getTopUsers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts index 6a8abd6..66cee03 100644 --- a/src/routes/getTopUsers.ts +++ b/src/routes/getTopUsers.ts @@ -21,7 +21,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as "categorySumOutro", SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as "categorySumInteraction", SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as "categorySelfpromo", - SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic", `; + SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic", + SUM(CASE WHEN category = 'preview' THEN 1 ELSE 0 END) as "categorySumPreview", `; } const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", @@ -48,6 +49,7 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole rows[i].categorySumInteraction, rows[i].categorySelfpromo, rows[i].categoryMusicOfftopic, + rows[i].categorySumPreview ]; } }