mirror of
https://github.com/ajayyy/SponsorBlockServer.git
synced 2025-12-11 05:57:04 +03:00
Merge pull request #218 from ajayyy/export
Apply indexes after upgrades
This commit is contained in:
32
databases/_private_indexes.sql
Normal file
32
databases/_private_indexes.sql
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
-- sponsorTimes
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedIP"
|
||||||
|
ON public."sponsorTimes" USING btree
|
||||||
|
("hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_videoID"
|
||||||
|
ON public."sponsorTimes" USING btree
|
||||||
|
("videoID" ASC NULLS LAST)
|
||||||
|
;
|
||||||
|
|
||||||
|
-- votes
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "votes_userID"
|
||||||
|
ON public.votes USING btree
|
||||||
|
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
-- shadowBannedUsers
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "shadowBannedUsers_index"
|
||||||
|
ON public."shadowBannedUsers" USING btree
|
||||||
|
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
-- categoryVotes
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "categoryVotes_UUID"
|
||||||
|
ON public."categoryVotes" USING btree
|
||||||
|
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, "userID" COLLATE pg_catalog."default" ASC NULLS LAST, "hashedIP" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
66
databases/_sponsorTimes_indexes.sql
Normal file
66
databases/_sponsorTimes_indexes.sql
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
-- sponsorTimes
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "sponsorTime_timeSubmitted"
|
||||||
|
ON public."sponsorTimes" USING btree
|
||||||
|
("timeSubmitted" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "sponsorTime_userID"
|
||||||
|
ON public."sponsorTimes" USING btree
|
||||||
|
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "sponsorTimes_UUID"
|
||||||
|
ON public."sponsorTimes" USING btree
|
||||||
|
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "sponsorTimes_hashedVideoID_gin"
|
||||||
|
ON public."sponsorTimes" USING gin
|
||||||
|
("hashedVideoID" COLLATE pg_catalog."default" gin_trgm_ops, category COLLATE pg_catalog."default" gin_trgm_ops)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "sponsorTimes_videoID"
|
||||||
|
ON public."sponsorTimes" USING btree
|
||||||
|
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
-- userNames
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "userNames_userID"
|
||||||
|
ON public."userNames" USING btree
|
||||||
|
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
-- vipUsers
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "vipUsers_index"
|
||||||
|
ON public."vipUsers" USING btree
|
||||||
|
("userID" COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
-- warnings
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "warnings_index"
|
||||||
|
ON public.warnings USING btree
|
||||||
|
("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "issueTime" DESC NULLS LAST, enabled DESC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "warnings_issueTime"
|
||||||
|
ON public.warnings USING btree
|
||||||
|
("issueTime" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
-- noSegments
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "noSegments_videoID"
|
||||||
|
ON public."noSegments" USING btree
|
||||||
|
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
|
|
||||||
|
-- categoryVotes
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "categoryVotes_UUID_public"
|
||||||
|
ON public."categoryVotes" USING btree
|
||||||
|
("UUID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
|
||||||
|
TABLESPACE pg_default;
|
||||||
29
databases/_upgrade_sponsorTimes_9.sql
Normal file
29
databases/_upgrade_sponsorTimes_9.sql
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
BEGIN TRANSACTION;
|
||||||
|
|
||||||
|
/* Add Service field */
|
||||||
|
CREATE TABLE "sqlb_temp_table_9" (
|
||||||
|
"videoID" TEXT NOT NULL,
|
||||||
|
"startTime" REAL NOT NULL,
|
||||||
|
"endTime" REAL NOT NULL,
|
||||||
|
"votes" INTEGER NOT NULL,
|
||||||
|
"locked" INTEGER NOT NULL default '0',
|
||||||
|
"incorrectVotes" INTEGER NOT NULL default '1',
|
||||||
|
"UUID" TEXT NOT NULL UNIQUE,
|
||||||
|
"userID" TEXT NOT NULL,
|
||||||
|
"timeSubmitted" INTEGER NOT NULL,
|
||||||
|
"views" INTEGER NOT NULL,
|
||||||
|
"category" TEXT NOT NULL DEFAULT 'sponsor',
|
||||||
|
"service" TEXT NOT NULL DEFAULT 'YouTube',
|
||||||
|
"videoDuration" REAL NOT NULL DEFAULT '0',
|
||||||
|
"shadowHidden" INTEGER NOT NULL,
|
||||||
|
"hashedVideoID" TEXT NOT NULL default ''
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO sqlb_temp_table_9 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
|
||||||
|
|
||||||
|
DROP TABLE "sponsorTimes";
|
||||||
|
ALTER TABLE sqlb_temp_table_9 RENAME TO "sponsorTimes";
|
||||||
|
|
||||||
|
UPDATE "config" SET value = 9 WHERE key = 'version';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -23,6 +23,13 @@ export class Postgres implements IDatabase {
|
|||||||
|
|
||||||
// Upgrade database if required
|
// Upgrade database if required
|
||||||
await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
|
await this.upgradeDB(this.config.fileNamePrefix, this.config.dbSchemaFolder);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
|
||||||
|
} catch (e) {
|
||||||
|
Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
|
||||||
|
Logger.warn(e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,6 +125,15 @@ export class Postgres implements IDatabase {
|
|||||||
Logger.debug('db update: no file ' + path);
|
Logger.debug('db update: no file ' + path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async applyIndexes(fileNamePrefix: string, schemaFolder: string) {
|
||||||
|
const path = schemaFolder + "/_" + fileNamePrefix + "_indexes.sql";
|
||||||
|
if (fs.existsSync(path)) {
|
||||||
|
await this.pool.query(fs.readFileSync(path).toString());
|
||||||
|
} else {
|
||||||
|
Logger.debug('failed to apply indexes to ' + fileNamePrefix);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private processUpgradeQuery(query: string): string {
|
private processUpgradeQuery(query: string): string {
|
||||||
let result = query;
|
let result = query;
|
||||||
result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
|
result = result.replace(/sha256\((.*?)\)/gm, "digest($1, 'sha256')");
|
||||||
|
|||||||
@@ -1,5 +1,13 @@
|
|||||||
import { Category, VideoID } from "../types/segments.model";
|
import { Service, VideoID, VideoIDHash } from "../types/segments.model";
|
||||||
|
import { Logger } from "../utils/logger";
|
||||||
|
|
||||||
export function skipSegmentsKey(videoID: VideoID): string {
|
export function skipSegmentsKey(videoID: VideoID): string {
|
||||||
return "segments-" + videoID;
|
return "segments-" + videoID;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
|
||||||
|
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
|
||||||
|
if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix);
|
||||||
|
|
||||||
|
return "segments." + service + "." + hashedVideoIDPrefix;
|
||||||
|
}
|
||||||
@@ -2,7 +2,7 @@ import { Request, Response } from 'express';
|
|||||||
import { RedisClient } from 'redis';
|
import { RedisClient } from 'redis';
|
||||||
import { config } from '../config';
|
import { config } from '../config';
|
||||||
import { db, privateDB } from '../databases/databases';
|
import { db, privateDB } from '../databases/databases';
|
||||||
import { skipSegmentsKey } from '../middleware/redisKeys';
|
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
|
||||||
import { SBRecord } from '../types/lib.model';
|
import { SBRecord } from '../types/lib.model';
|
||||||
import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
|
import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
|
||||||
import { getHash } from '../utils/getHash';
|
import { getHash } from '../utils/getHash';
|
||||||
@@ -92,13 +92,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
|
|||||||
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
|
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
|
||||||
if (categories.length === 0) return null;
|
if (categories.length === 0) return null;
|
||||||
|
|
||||||
const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
|
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
|
||||||
.prepare(
|
.filter((segment: DBSegment) => categories.includes(segment?.category))
|
||||||
'all',
|
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
|
||||||
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
|
|
||||||
WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
|
|
||||||
[hashedVideoIDPrefix + '%', service]
|
|
||||||
)).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
|
|
||||||
acc[segment.videoID] = acc[segment.videoID] || {
|
acc[segment.videoID] = acc[segment.videoID] || {
|
||||||
hash: segment.hashedVideoID,
|
hash: segment.hashedVideoID,
|
||||||
segmentPerCategory: {},
|
segmentPerCategory: {},
|
||||||
@@ -131,6 +127,37 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
|
||||||
|
const fetchFromDB = () => db
|
||||||
|
.prepare(
|
||||||
|
'all',
|
||||||
|
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
|
||||||
|
WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`,
|
||||||
|
[hashedVideoIDPrefix + '%', service]
|
||||||
|
);
|
||||||
|
|
||||||
|
if (hashedVideoIDPrefix.length === 4) {
|
||||||
|
const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
|
||||||
|
const {err, reply} = await redis.getAsync(key);
|
||||||
|
|
||||||
|
if (!err && reply) {
|
||||||
|
try {
|
||||||
|
Logger.debug("Got data from redis: " + reply);
|
||||||
|
return JSON.parse(reply);
|
||||||
|
} catch (e) {
|
||||||
|
// If all else, continue on to fetching from the database
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await fetchFromDB();
|
||||||
|
|
||||||
|
redis.setAsync(key, JSON.stringify(data));
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await fetchFromDB();
|
||||||
|
}
|
||||||
|
|
||||||
//gets a weighted random choice from the choices array based on their `votes` property.
|
//gets a weighted random choice from the choices array based on their `votes` property.
|
||||||
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
|
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
|
||||||
//choices are unique
|
//choices are unique
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import {getFormattedTime} from '../utils/getFormattedTime';
|
|||||||
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
|
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
|
||||||
import {dispatchEvent} from '../utils/webhookUtils';
|
import {dispatchEvent} from '../utils/webhookUtils';
|
||||||
import {Request, Response} from 'express';
|
import {Request, Response} from 'express';
|
||||||
import { skipSegmentsKey } from '../middleware/redisKeys';
|
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
|
||||||
import redis from '../utils/redis';
|
import redis from '../utils/redis';
|
||||||
import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
|
import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
|
||||||
|
|
||||||
@@ -423,7 +423,11 @@ export async function postSkipSegments(req: Request, res: Response) {
|
|||||||
if (service == Service.YouTube) {
|
if (service == Service.YouTube) {
|
||||||
apiVideoInfo = await getYouTubeVideoInfo(videoID);
|
apiVideoInfo = await getYouTubeVideoInfo(videoID);
|
||||||
}
|
}
|
||||||
videoDuration = getYouTubeVideoDuration(apiVideoInfo) || videoDuration;
|
const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
|
||||||
|
if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
|
||||||
|
// If api duration is far off, take that one instead (it is only precise to seconds, not millis)
|
||||||
|
videoDuration = apiVideoDuration || 0 as VideoDuration;
|
||||||
|
}
|
||||||
|
|
||||||
// Auto moderator check
|
// Auto moderator check
|
||||||
if (!isVIP && service == Service.YouTube) {
|
if (!isVIP && service == Service.YouTube) {
|
||||||
@@ -493,13 +497,14 @@ export async function postSkipSegments(req: Request, res: Response) {
|
|||||||
//it's better than generating an actual UUID like what was used before
|
//it's better than generating an actual UUID like what was used before
|
||||||
//also better for duplication checking
|
//also better for duplication checking
|
||||||
const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
|
const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
|
||||||
|
const hashedVideoID = getHash(videoID, 1);
|
||||||
|
|
||||||
const startingLocked = isVIP ? 1 : 0;
|
const startingLocked = isVIP ? 1 : 0;
|
||||||
try {
|
try {
|
||||||
await db.prepare('run', `INSERT INTO "sponsorTimes"
|
await db.prepare('run', `INSERT INTO "sponsorTimes"
|
||||||
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
|
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
|
||||||
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
||||||
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1),
|
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -508,6 +513,7 @@ export async function postSkipSegments(req: Request, res: Response) {
|
|||||||
|
|
||||||
// Clear redis cache for this video
|
// Clear redis cache for this video
|
||||||
redis.delAsync(skipSegmentsKey(videoID));
|
redis.delAsync(skipSegmentsKey(videoID));
|
||||||
|
redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
//a DB change probably occurred
|
//a DB change probably occurred
|
||||||
res.sendStatus(500);
|
res.sendStatus(500);
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import {getHash} from '../utils/getHash';
|
|||||||
import {config} from '../config';
|
import {config} from '../config';
|
||||||
import { UserID } from '../types/user.model';
|
import { UserID } from '../types/user.model';
|
||||||
import redis from '../utils/redis';
|
import redis from '../utils/redis';
|
||||||
import { skipSegmentsKey } from '../middleware/redisKeys';
|
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
|
||||||
import { VideoID } from '../types/segments.model';
|
import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
|
||||||
|
|
||||||
const voteTypes = {
|
const voteTypes = {
|
||||||
normal: 0,
|
normal: 0,
|
||||||
@@ -147,8 +147,8 @@ async function sendWebhooks(voteData: VoteData) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnSubmission: boolean, category: string
|
async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isOwnSubmission: boolean, category: Category
|
||||||
, hashedIP: string, finalResponse: FinalResponse, res: Response) {
|
, hashedIP: HashedIP, finalResponse: FinalResponse, res: Response) {
|
||||||
// Check if they've already made a vote
|
// Check if they've already made a vote
|
||||||
const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]);
|
const usersLastVoteInfo = await privateDB.prepare('get', `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]);
|
||||||
|
|
||||||
@@ -158,8 +158,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const currentCategory = await db.prepare('get', `select category from "sponsorTimes" where "UUID" = ?`, [UUID]);
|
const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
|
||||||
if (!currentCategory) {
|
[UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
|
||||||
|
if (!videoInfo) {
|
||||||
// Submission doesn't exist
|
// Submission doesn't exist
|
||||||
res.status(400).send("Submission doesn't exist.");
|
res.status(400).send("Submission doesn't exist.");
|
||||||
return;
|
return;
|
||||||
@@ -196,7 +197,7 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
|
|||||||
}
|
}
|
||||||
|
|
||||||
// See if the submissions category is ready to change
|
// See if the submissions category is ready to change
|
||||||
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, currentCategory.category]);
|
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
|
||||||
|
|
||||||
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
|
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
|
||||||
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
|
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
|
||||||
@@ -208,9 +209,9 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
|
|||||||
|
|
||||||
// Add submission as vote
|
// Add submission as vote
|
||||||
if (!currentCategoryInfo && submissionInfo) {
|
if (!currentCategoryInfo && submissionInfo) {
|
||||||
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, currentCategory.category, currentCategoryCount]);
|
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
|
||||||
|
|
||||||
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", currentCategory.category, submissionInfo.timeSubmitted]);
|
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
|
||||||
}
|
}
|
||||||
|
|
||||||
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
|
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
|
||||||
@@ -222,6 +223,8 @@ async function categoryVote(UUID: string, userID: string, isVIP: boolean, isOwnS
|
|||||||
await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
|
await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
clearRedisCache(videoInfo);
|
||||||
|
|
||||||
res.sendStatus(finalResponse.finalStatus);
|
res.sendStatus(finalResponse.finalStatus);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -230,10 +233,10 @@ export function getUserID(req: Request): UserID {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function voteOnSponsorTime(req: Request, res: Response) {
|
export async function voteOnSponsorTime(req: Request, res: Response) {
|
||||||
const UUID = req.query.UUID as string;
|
const UUID = req.query.UUID as SegmentUUID;
|
||||||
const paramUserID = getUserID(req);
|
const paramUserID = getUserID(req);
|
||||||
let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined;
|
let type = req.query.type !== undefined ? parseInt(req.query.type as string) : undefined;
|
||||||
const category = req.query.category as string;
|
const category = req.query.category as Category;
|
||||||
|
|
||||||
if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) {
|
if (UUID === undefined || paramUserID === undefined || (type === undefined && category === undefined)) {
|
||||||
//invalid request
|
//invalid request
|
||||||
@@ -255,7 +258,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
|
|||||||
const ip = getIP(req);
|
const ip = getIP(req);
|
||||||
|
|
||||||
//hash the ip 5000 times so no one can get it from the database
|
//hash the ip 5000 times so no one can get it from the database
|
||||||
const hashedIP = getHash(ip + config.globalSalt);
|
const hashedIP: HashedIP = getHash((ip + config.globalSalt) as IPAddress);
|
||||||
|
|
||||||
//check if this user is on the vip list
|
//check if this user is on the vip list
|
||||||
const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0;
|
const isVIP = (await db.prepare('get', `SELECT count(*) as "userCount" FROM "vipUsers" WHERE "userID" = ?`, [nonAnonUserID])).userCount > 0;
|
||||||
@@ -350,13 +353,13 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
//check if the increment amount should be multiplied (downvotes have more power if there have been many views)
|
//check if the increment amount should be multiplied (downvotes have more power if there have been many views)
|
||||||
const row = await db.prepare('get', `SELECT "videoID", votes, views FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
|
const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
|
||||||
{videoID: VideoID, votes: number, views: number};
|
{videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
|
||||||
|
|
||||||
if (voteTypeEnum === voteTypes.normal) {
|
if (voteTypeEnum === voteTypes.normal) {
|
||||||
if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
|
if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
|
||||||
//this user is a vip and a downvote
|
//this user is a vip and a downvote
|
||||||
incrementAmount = -(row.votes + 2 - oldIncrementAmount);
|
incrementAmount = -(videoInfo.votes + 2 - oldIncrementAmount);
|
||||||
type = incrementAmount;
|
type = incrementAmount;
|
||||||
}
|
}
|
||||||
} else if (voteTypeEnum == voteTypes.incorrect) {
|
} else if (voteTypeEnum == voteTypes.incorrect) {
|
||||||
@@ -399,8 +402,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
|
|||||||
await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
|
await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clear redis cache for this video
|
clearRedisCache(videoInfo);
|
||||||
redis.delAsync(skipSegmentsKey(row?.videoID));
|
|
||||||
|
|
||||||
//for each positive vote, see if a hidden submission can be shown again
|
//for each positive vote, see if a hidden submission can be shown again
|
||||||
if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
|
if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
|
||||||
@@ -437,7 +439,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
|
|||||||
voteTypeEnum,
|
voteTypeEnum,
|
||||||
isVIP,
|
isVIP,
|
||||||
isOwnSubmission,
|
isOwnSubmission,
|
||||||
row,
|
row: videoInfo,
|
||||||
category,
|
category,
|
||||||
incrementAmount,
|
incrementAmount,
|
||||||
oldIncrementAmount,
|
oldIncrementAmount,
|
||||||
@@ -449,4 +451,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
|
|||||||
|
|
||||||
res.status(500).json({error: 'Internal error creating segment vote'});
|
res.status(500).json({error: 'Internal error creating segment vote'});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
|
||||||
|
if (videoInfo) {
|
||||||
|
redis.delAsync(skipSegmentsKey(videoInfo.videoID));
|
||||||
|
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,8 +12,9 @@ export type HashedIP = IPAddress & HashedValue;
|
|||||||
// Uncomment as needed
|
// Uncomment as needed
|
||||||
export enum Service {
|
export enum Service {
|
||||||
YouTube = 'YouTube',
|
YouTube = 'YouTube',
|
||||||
// Nebula = 'Nebula',
|
|
||||||
PeerTube = 'PeerTube',
|
PeerTube = 'PeerTube',
|
||||||
|
// Twitch = 'Twitch',
|
||||||
|
// Nebula = 'Nebula',
|
||||||
// RSS = 'RSS',
|
// RSS = 'RSS',
|
||||||
// Corridor = 'Corridor',
|
// Corridor = 'Corridor',
|
||||||
// Lbry = 'Lbry'
|
// Lbry = 'Lbry'
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ describe('postSkipSegments', () => {
|
|||||||
.catch(err => done(err));
|
.catch(err => done(err));
|
||||||
});
|
});
|
||||||
|
|
||||||
it('Should be able to submit a single time with a duration (JSON method)', (done: Done) => {
|
it('Should be able to submit a single time with a duration from the YouTube API (JSON method)', (done: Done) => {
|
||||||
fetch(getbaseURL()
|
fetch(getbaseURL()
|
||||||
+ "/api/postVideoSponsorTimes", {
|
+ "/api/postVideoSponsorTimes", {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
@@ -129,7 +129,39 @@ describe('postSkipSegments', () => {
|
|||||||
.catch(err => done(err));
|
.catch(err => done(err));
|
||||||
});
|
});
|
||||||
|
|
||||||
it('Should be able to submit a single time with a duration from the API (JSON method)', (done: Done) => {
|
it('Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)', (done: Done) => {
|
||||||
|
fetch(getbaseURL()
|
||||||
|
+ "/api/postVideoSponsorTimes", {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
userID: "test",
|
||||||
|
videoID: "dQw4w9WgXZH",
|
||||||
|
videoDuration: 5010.20,
|
||||||
|
segments: [{
|
||||||
|
segment: [1, 10],
|
||||||
|
category: "sponsor",
|
||||||
|
}],
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.then(async res => {
|
||||||
|
if (res.status === 200) {
|
||||||
|
const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZH"]);
|
||||||
|
if (row.startTime === 1 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010.20) {
|
||||||
|
done();
|
||||||
|
} else {
|
||||||
|
done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
done("Status code was " + res.status);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(err => done(err));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('Should be able to submit a single time with a duration in the body (JSON method)', (done: Done) => {
|
||||||
fetch(getbaseURL()
|
fetch(getbaseURL()
|
||||||
+ "/api/postVideoSponsorTimes", {
|
+ "/api/postVideoSponsorTimes", {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
|||||||
Reference in New Issue
Block a user