From 5c2ab9087a2829fea5ec1cf471b92b0b9c17441e Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 23 May 2021 17:54:51 -0400 Subject: [PATCH] Use reputation when sorting segments --- databases/_upgrade_sponsorTimes_12.sql | 31 ++++++++++++++++++++++++++ src/middleware/reputation.ts | 4 ++-- src/routes/getSkipSegments.ts | 24 +++++++++++++------- src/routes/postSkipSegments.ts | 8 ++++--- src/types/segments.model.ts | 5 +++++ 5 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 databases/_upgrade_sponsorTimes_12.sql diff --git a/databases/_upgrade_sponsorTimes_12.sql b/databases/_upgrade_sponsorTimes_12.sql new file mode 100644 index 0000000..aeace54 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_12.sql @@ -0,0 +1,31 @@ +BEGIN TRANSACTION; + +/* Add Service field */ +CREATE TABLE "sqlb_temp_table_12" ( + "videoID" TEXT NOT NULL, + "startTime" REAL NOT NULL, + "endTime" REAL NOT NULL, + "votes" INTEGER NOT NULL, + "locked" INTEGER NOT NULL default '0', + "incorrectVotes" INTEGER NOT NULL default '1', + "UUID" TEXT NOT NULL UNIQUE, + "userID" TEXT NOT NULL, + "timeSubmitted" INTEGER NOT NULL, + "views" INTEGER NOT NULL, + "category" TEXT NOT NULL DEFAULT 'sponsor', + "service" TEXT NOT NULL DEFAULT 'YouTube', + "videoDuration" REAL NOT NULL DEFAULT '0', + "hidden" INTEGER NOT NULL DEFAULT '0', + "reputation" REAL NOT NULL DEFAULT 0, + "shadowHidden" INTEGER NOT NULL, + "hashedVideoID" TEXT NOT NULL default '' +); + +INSERT INTO sqlb_temp_table_12 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service","videoDuration","hidden",0,"shadowHidden","hashedVideoID" FROM "sponsorTimes"; + +DROP TABLE "sponsorTimes"; +ALTER TABLE sqlb_temp_table_12 RENAME TO "sponsorTimes"; + +UPDATE "config" SET value = 12 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/middleware/reputation.ts b/src/middleware/reputation.ts index cb18b24..31672a4 100644 --- a/src/middleware/reputation.ts +++ b/src/middleware/reputation.ts @@ -10,7 +10,7 @@ interface ReputationDBResult { oldUpvotedSubmissions: number } -export async function getReputation(userID: UserID) { +export async function getReputation(userID: UserID): Promise { const pastDate = Date.now() - 1000 * 1000 * 60 * 60 * 24 * 45; // 45 days ago const fetchFromDB = () => db.prepare("get", `SELECT COUNT(*) AS "totalSubmissions", @@ -32,7 +32,7 @@ export async function getReputation(userID: UserID) { } if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) { - return 0 + return 0; } return convertRange(Math.min(result.upvotedSum, 50), 5, 50, 0, 15); diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 63ad30e..0e5ce19 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -9,6 +9,7 @@ import { getHash } from '../utils/getHash'; import { getIP } from '../utils/getIP'; import { Logger } from '../utils/logger'; import { QueryCacher } from '../middleware/queryCacher' +import { getReputation } from '../middleware/reputation'; async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise { @@ -41,7 +42,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category: const filteredSegments = segments.filter((_, index) => shouldFilter[index]); const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1 - return chooseSegments(filteredSegments, maxSegments).map((chosenSegment) => ({ + return (await chooseSegments(filteredSegments, maxSegments)).map((chosenSegment) => ({ category, segment: [chosenSegment.startTime, chosenSegment.endTime], UUID: chosenSegment.UUID, @@ -128,7 +129,7 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service const fetchFromDB = () => db .prepare( 'all', - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" + `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden", "hashedVideoID" FROM "sponsorTimes" WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [hashedVideoIDPrefix + '%', service] ) as Promise; @@ -144,7 +145,7 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P const fetchFromDB = () => db .prepare( 'all', - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" + `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, [videoID, service] ) as Promise; @@ -170,7 +171,7 @@ function getWeightedRandomChoice(choices: T[], amountOf let choicesWithWeights: TWithWeight[] = choices.map(choice => { //The 3 makes -2 the minimum votes before being ignored completely //this can be changed if this system increases in popularity. - const weight = Math.exp((choice.votes + 3)); + const weight = Math.exp((choice.votes + 3 + choice.reputation)); totalWeight += weight; return {...choice, weight}; @@ -200,7 +201,7 @@ function getWeightedRandomChoice(choices: T[], amountOf //Only one similar time will be returned, randomly generated based on the sqrt of votes. //This allows new less voted items to still sometimes appear to give them a chance at getting votes. //Segments with less than -1 votes are already ignored before this function is called -function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { +async function chooseSegments(segments: DBSegment[], max: number): Promise { //Create groups of segments that are similar to eachother //Segments must be sorted by their startTime so that we can build groups chronologically: //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group @@ -209,9 +210,9 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { const overlappingSegmentsGroups: OverlappingSegmentGroup[] = []; let currentGroup: OverlappingSegmentGroup; let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created - segments.forEach(segment => { + for (const segment of segments) { if (segment.startTime > cursor) { - currentGroup = {segments: [], votes: 0, locked: false}; + currentGroup = {segments: [], votes: 0, reputation: 0, locked: false}; overlappingSegmentsGroups.push(currentGroup); } @@ -221,17 +222,24 @@ function chooseSegments(segments: DBSegment[], max: number): DBSegment[] { currentGroup.votes += segment.votes; } + segment.reputation = Math.min(segment.reputation, await getReputation(segment.userID)); + if (segment.reputation > 0) { + currentGroup.reputation += segment.reputation; + } + if (segment.locked) { currentGroup.locked = true; } cursor = Math.max(cursor, segment.endTime); - }); + }; overlappingSegmentsGroups.forEach((group) => { if (group.locked) { group.segments = group.segments.filter((segment) => segment.locked); } + + group.reputation = group.reputation / group.segments.length; }); //if there are too many groups, find the best ones diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index b46bfe3..9c28964 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -17,6 +17,7 @@ import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Se import { deleteLockCategories } from './deleteLockCategories'; import { getCategoryActionType } from '../utils/categoryInfo'; import { QueryCacher } from '../middleware/queryCacher'; +import { getReputation } from '../middleware/reputation'; interface APIVideoInfo { err: string | boolean, @@ -508,6 +509,7 @@ export async function postSkipSegments(req: Request, res: Response) { } let startingVotes = 0 + decreaseVotes; + const reputation = await getReputation(userID); for (const segmentInfo of segments) { //this can just be a hash of the data @@ -519,9 +521,9 @@ export async function postSkipSegments(req: Request, res: Response) { const startingLocked = isVIP ? 1 : 0; try { await db.prepare('run', `INSERT INTO "sponsorTimes" - ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID") - VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ - videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID, + ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID") + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [ + videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, reputation, shadowBanned, hashedVideoID, ], ); diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index c1606a9..945f294 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -1,5 +1,6 @@ import { HashedValue } from "./hash.model"; import { SBRecord } from "./lib.model"; +import { UserID } from "./user.model"; export type SegmentUUID = string & { __segmentUUIDBrand: unknown }; export type VideoID = string & { __videoIDBrand: unknown }; @@ -42,11 +43,13 @@ export interface DBSegment { startTime: number; endTime: number; UUID: SegmentUUID; + userID: UserID; votes: number; locked: boolean; shadowHidden: Visibility; videoID: VideoID; videoDuration: VideoDuration; + reputation: number; hashedVideoID: VideoIDHash; } @@ -54,10 +57,12 @@ export interface OverlappingSegmentGroup { segments: DBSegment[], votes: number; locked: boolean; // Contains a locked segment + reputation: number; } export interface VotableObject { votes: number; + reputation: number; } export interface VotableObjectWithWeight extends VotableObject {