Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-10 13:37:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Initial get branding

Browse Source
This commit is contained in:
Ajay
2022-12-23 16:56:27 -05:00
parent cff2325aef
commit cc24a4902f
7 changed files with 393 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
databases/_private.db.sql
View File

@@ -26,4 +26,30 @@ CREATE TABLE IF NOT EXISTS "config" (
"value" TEXT NOT NULL "value" TEXT NOT NULL
); );
CREATE TABLE IF NOT EXISTS "titles" (
"UUID" TEXT NOT NULL PRIMARY KEY,
"hashedIP" TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS "titleVotes" (
"id" SERIAL PRIMARY KEY,
"UUID" TEXT NOT NULL,
"userID" TEXT NOT NULL,
"hashedIP" TEXT NOT NULL,
"type" INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS "thumbnails" (
"UUID" TEXT NOT NULL PRIMARY KEY,
"hashedIP" TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS "thumbnailVotes" (
"id" SERIAL PRIMARY KEY,
"UUID" TEXT NOT NULL,
"userID" TEXT NOT NULL,
"hashedIP" TEXT NOT NULL,
"type" INTEGER NOT NULL
);
COMMIT; COMMIT;

43
databases/_sponsorTimes.db.sql
View File

@@ -37,6 +37,49 @@ CREATE TABLE IF NOT EXISTS "config" (
"value" TEXT NOT NULL "value" TEXT NOT NULL
); );
CREATE TABLE IF NOT EXISTS "titles" (
"videoID" TEXT NOT NULL,
"title" TEXT NOT NULL,
"original" BOOLEAN NOT NULL,
"userID" TEXT NOT NULL,
"service" TEXT NOT NULL,
"hashedVideoID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"UUID" TEXT NOT NULL PRIMARY KEY
);
CREATE TABLE IF NOT EXISTS "titleVotes" (
"UUID" TEXT NOT NULL PRIMARY KEY,
"votes" INTEGER NOT NULL default 0,
"locked" INTEGER NOT NULL default 0,
"shadowHidden" INTEGER NOT NULL default 0,
FOREIGN KEY("UUID") REFERENCES "titles"("UUID")
);
CREATE TABLE IF NOT EXISTS "thumbnails" (
"videoID" TEXT NOT NULL,
"original" INTEGER default 0,
"userID" TEXT NOT NULL,
"service" TEXT NOT NULL,
"hashedVideoID" TEXT NOT NULL,
"timeSubmitted" INTEGER NOT NULL,
"UUID" TEXT NOT NULL PRIMARY KEY
);
CREATE TABLE IF NOT EXISTS "thumbnailTimestamps" (
"UUID" TEXT NOT NULL PRIMARY KEY,
"timestamp" INTEGER NOT NULL default 0
FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID")
);
CREATE TABLE IF NOT EXISTS "thumbnailVotes" (
"UUID" TEXT NOT NULL PRIMARY KEY,
"votes" INTEGER NOT NULL default 0,
"locked" INTEGER NOT NULL default 0,
"shadowHidden" INTEGER NOT NULL default 0,
FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID")
);
CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore

34
databases/_sponsorTimes_indexes.sql
View File

@@ -116,3 +116,37 @@ CREATE INDEX IF NOT EXISTS "userFeatures_userID"
ON public."userFeatures" USING btree ON public."userFeatures" USING btree
("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "feature" ASC NULLS LAST) ("userID" COLLATE pg_catalog."default" ASC NULLS LAST, "feature" ASC NULLS LAST)
TABLESPACE pg_default; TABLESPACE pg_default;
-- titles
CREATE INDEX IF NOT EXISTS "titles_timeSubmitted"
ON public."titles" USING btree
("timeSubmitted" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "titles_videoID"
ON public."titles" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "titles_hashedVideoID"
ON public."titles" USING btree
("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
-- thumbnails
CREATE INDEX IF NOT EXISTS "thumbnails_timeSubmitted"
ON public."thumbnails" USING btree
("timeSubmitted" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "thumbnails_videoID"
ON public."thumbnails" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;
CREATE INDEX IF NOT EXISTS "thumbnails_hashedVideoID"
ON public."thumbnails" USING btree
("hashedVideoID" COLLATE pg_catalog."default" ASC NULLS LAST, "service" COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;

209
src/routes/getBranding.ts Normal file
View File

@@ -0,0 +1,209 @@
import { Request, Response } from "express";
import { isEmpty } from "lodash";
import { config } from "../config";
import { db, privateDB } from "../databases/databases";
import { BrandingDBSubmission, BrandingHashDBResult, BrandingHashResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model";
import { shuffleArray } from "../utils/array";
import { getHashCache } from "../utils/getHashCache";
import { getIP } from "../utils/getIP";
import { getService } from "../utils/getService";
import { hashPrefixTester } from "../utils/hashPrefixTester";
import { promiseOrTimeout } from "../utils/promise";
import { QueryCacher } from "../utils/queryCacher";
import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys";
enum BrandingSubmissionType {
Title = "title",
Thumbnail = "thumbnail"
}
export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise<BrandingResult> {
const getTitles = () => db.prepare(
"all",
`SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID
FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
[videoID, service],
{ useReplica: true }
) as Promise<TitleDBResult[]>;
const getThumbnails = () => db.prepare(
"all",
`SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
WHERE "thumbnails"."videoID" = ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
[videoID, service],
{ useReplica: true }
) as Promise<ThumbnailDBResult[]>;
// eslint-disable-next-line require-await
const getBranding = async () => ({
titles: getTitles(),
thumbnails: getThumbnails()
});
const branding = await QueryCacher.get(getBranding, brandingKey(videoID, service));
const cache = {
currentIP: null as Promise<HashedIP> | null
};
return filterAndSortBranding(await branding.titles, await branding.thumbnails, ip, cache);
}
export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise<Record<VideoID, BrandingHashResult>> {
const getTitles = () => db.prepare(
"all",
`SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID
FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
[`${videoHashPrefix}%`, service],
{ useReplica: true }
) as Promise<TitleDBResult[]>;
const getThumbnails = () => db.prepare(
"all",
`SELECT "thumbnailTimestamps"."timestamp", "thumbnails"."original", "thumbnailVotes"."votes", "thumbnailVotes"."locked", "thumbnailVotes"."shadowHidden", "thumbnails"."UUID", "thumbnails"."videoID", "thumbnails"."hashedVideoID"
FROM "thumbnails" LEFT JOIN "thumbnailVotes" ON "thumbnails"."UUID" = "thumbnailVotes"."UUID" LEFT JOIN "thumbnailTimestamps" ON "thumbnails"."UUID" = "thumbnailTimestamps"."UUID"
WHERE "thumbnails"."hashedVideoID" LIKE ? AND "thumbnails"."service" = ? AND "thumbnailVotes"."votes" > -2`,
[`${videoHashPrefix}%`, service],
{ useReplica: true }
) as Promise<ThumbnailDBResult[]>;
const branding = await QueryCacher.get(async () => {
// Make sure they are both called in parallel
const branding = {
titles: getTitles(),
thumbnails: getThumbnails()
};
const dbResult: Record<VideoID, BrandingHashDBResult> = {};
const initResult = (submission: BrandingDBSubmission) => {
dbResult[submission.videoID] = dbResult[submission.videoID] || {
hash: submission.hashedVideoID,
branding: {
titles: [],
thumbnails: []
}
};
};
(await branding.titles).map((title) => {
initResult(title);
dbResult[title.videoID].branding.titles.push(title);
});
(await branding.thumbnails).map((thumbnail) => {
initResult(thumbnail);
dbResult[thumbnail.videoID].branding.thumbnails.push(thumbnail);
});
return dbResult;
}, brandingHashKey(videoHashPrefix, service));
const cache = {
currentIP: null as Promise<HashedIP> | null
};
const processedResult: Record<VideoID, BrandingHashResult> = {};
await Promise.all(Object.keys(branding).map(async (key) => {
const castedKey = key as VideoID;
processedResult[castedKey] = {
hash: branding[castedKey].hash,
branding: await filterAndSortBranding(branding[castedKey].branding.titles, branding[castedKey].branding.thumbnails, ip, cache)
};
}));
return processedResult;
}
async function filterAndSortBranding(dbTitles: TitleDBResult[], dbThumbnails: ThumbnailDBResult[], ip: IPAddress, cache: { currentIP: Promise<HashedIP> | null}): Promise<BrandingResult> {
const shouldKeepTitles = shouldKeepSubmission(dbTitles, BrandingSubmissionType.Title, ip, cache);
const shouldKeepThumbnails = shouldKeepSubmission(dbThumbnails, BrandingSubmissionType.Thumbnail, ip, cache);
const titles = shuffleArray(dbTitles.filter(await shouldKeepTitles))
.sort((a, b) => b.votes - a.votes)
.sort((a, b) => b.locked - a.locked)
.map((r) => ({
title: r.title,
original: r.original === 1,
votes: r.votes,
locked: r.locked === 1,
UUID: r.UUID,
})) as TitleResult[];
const thumbnails = shuffleArray(dbThumbnails.filter(await shouldKeepThumbnails))
.sort((a, b) => b.votes - a.votes)
.sort((a, b) => b.locked - a.locked)
.map((r) => ({
timestamp: r.timestamp,
original: r.original === 1,
votes: r.votes,
locked: r.locked === 1,
UUID: r.UUID
})) as ThumbnailResult[];
return {
titles,
thumbnails
};
}
async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: BrandingSubmissionType, ip: IPAddress,
cache: { currentIP: Promise<HashedIP> | null }): Promise<(_: unknown, index: number) => boolean> {
const shouldKeep = await Promise.all(submissions.map(async (s) => {
if (s.shadowHidden != Visibility.HIDDEN) return true;
const table = type === BrandingSubmissionType.Title ? "titles" : "thumbnails";
const fetchData = () => privateDB.prepare("get", `SELECT "hashedIP" FROM "${table}" WHERE "UUID" = ?`,
[s.UUID], { useReplica: true }) as Promise<{ hashedIP: HashedIP }>;
try {
const submitterIP = await promiseOrTimeout(QueryCacher.get(fetchData, brandingIPKey(s.UUID)), 150);
if (cache.currentIP === null) cache.currentIP = getHashCache((ip + config.globalSalt) as IPAddress);
const hashedIP = await cache.currentIP;
return submitterIP.hashedIP !== hashedIP;
} catch (e) {
// give up on shadow hide for now
return true;
}
}));
return (_, index) => shouldKeep[index];
}
export async function getBranding(req: Request, res: Response) {
const videoID: VideoID = req.query.videoID as VideoID;
const service: Service = getService(req.query.service, req.body.service);
if (!videoID) {
return res.status(400).send("Missing parameter: videoID");
}
const ip = getIP(req);
const result = await getVideoBranding(videoID, service, ip);
const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404;
return res.status(status).json(result);
}
export async function getBrandingByHash(req: Request, res: Response) {
let hashPrefix = req.params.prefix as VideoIDHash;
if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) {
return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
}
hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
const service: Service = getService(req.query.service, req.body.service);
if (!hashPrefix || hashPrefix.length !== 4) {
return res.status(400).send("Hash prefix does not match format requirements.");
}
const ip = getIP(req);
const result = await getVideoBrandingByHash(hashPrefix, service, ip);
const status = !isEmpty(result) ? 200 : 404;
return res.status(status).json(result);
}

58
src/types/branding.model.ts Normal file
View File

@@ -0,0 +1,58 @@
import { VideoID, VideoIDHash } from "./segments.model";
export type BrandingUUID = string & { readonly __brandingUUID: unique symbol };
export interface BrandingDBSubmission {
shadowHidden: number,
UUID: BrandingUUID,
videoID: VideoID,
hashedVideoID: VideoIDHash
}
export interface TitleDBResult extends BrandingDBSubmission {
title: string,
original: number,
votes: number,
locked: number
}
export interface TitleResult {
title: string,
original: boolean,
votes: number,
locked: boolean,
UUID: BrandingUUID
}
export interface ThumbnailDBResult extends BrandingDBSubmission {
timestamp?: number,
original: number,
votes: number,
locked: number
}
export interface ThumbnailResult {
timestamp?: number,
original: boolean,
votes: number,
locked: boolean,
UUID: BrandingUUID
}
export interface BrandingResult {
titles: TitleResult[],
thumbnails: ThumbnailResult[]
}
export interface BrandingHashDBResult {
hash: VideoIDHash;
branding: {
titles: TitleDBResult[],
thumbnails: ThumbnailDBResult[]
};
}
export interface BrandingHashResult {
hash: VideoIDHash;
branding: BrandingResult;
}

8
src/utils/array.ts Normal file
View File

@@ -0,0 +1,8 @@
export function shuffleArray<T>(array: T[]): T[] {
for (let i = array.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[array[i], array[j]] = [array[j], array[i]];
}
return array;
}

15
src/utils/redisKeys.ts
View File

@@ -2,6 +2,7 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { Feature, HashedUserID, UserID } from "../types/user.model"; import { Feature, HashedUserID, UserID } from "../types/user.model";
import { HashedValue } from "../types/hash.model"; import { HashedValue } from "../types/hash.model";
import { Logger } from "./logger"; import { Logger } from "./logger";
import { BrandingUUID } from "../types/branding.model";
export const skipSegmentsKey = (videoID: VideoID, service: Service): string => export const skipSegmentsKey = (videoID: VideoID, service: Service): string =>
`segments.v4.${service}.videoID.${videoID}`; `segments.v4.${service}.videoID.${videoID}`;
@@ -16,6 +17,20 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
return `segments.v4.${service}.${hashedVideoIDPrefix}`; return `segments.v4.${service}.${hashedVideoIDPrefix}`;
} }
export const brandingKey = (videoID: VideoID, service: Service): string =>
`branding.${service}.videoID.${videoID}`;
export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash;
if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`);
return `branding.${service}.${hashedVideoIDPrefix}`;
}
export const brandingIPKey = (uuid: BrandingUUID): string =>
`branding.shadow.${uuid}`;
export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string => export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string =>
`segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`;