Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-16 08:26:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into fullVideoLabels

Browse Source
This commit is contained in:
Michael C
2022-05-09 02:42:18 -04:00
parent 65a661ca4d 60a3c017e5
commit 6a826b63c9
33 changed files with 653 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/app.ts
View File

@@ -27,7 +27,7 @@ import { loggerMiddleware } from "./middleware/logger";
import { corsMiddleware } from "./middleware/cors";
import { apiCspMiddleware } from "./middleware/apiCsp";
import { rateLimitMiddleware } from "./middleware/requestRateLimit";
import dumpDatabase, { appExportPath, redirectLink } from "./routes/dumpDatabase";
import dumpDatabase, { appExportPath, downloadFile } from "./routes/dumpDatabase";
import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
import { postClearCache } from "./routes/postClearCache";
import { addUnlistedVideo } from "./routes/addUnlistedVideo";
@@ -208,14 +208,14 @@ function setupRoutes(router: Router) {
router.get("/api/videoLabels", getVideoLabels);
router.get("/api/videoLabels/:prefix", getVideoLabelsByHash);
if (config.postgres) {
if (config.postgres?.enabled) {
router.get("/database", (req, res) => dumpDatabase(req, res, true));
router.get("/database.json", (req, res) => dumpDatabase(req, res, false));
router.get("/database/*", redirectLink);
router.get("/database/*", downloadFile);
router.use("/download", express.static(appExportPath));
} else {
router.get("/database.db", function (req: Request, res: Response) {
res.sendFile("./databases/sponsorTimes.db", { root: "./" });
});
}
}
}

102
src/config.ts
View File

@@ -1,6 +1,7 @@
import fs from "fs";
import { SBSConfig } from "./types/config.model";
import packageJson from "../package.json";
import { isBoolean, isNumber } from "lodash";
const isTestMode = process.env.npm_lifecycle_script === packageJson.scripts.test;
const configFile = process.env.TEST_POSTGRES ? "ci.json"
@@ -9,7 +10,7 @@ const configFile = process.env.TEST_POSTGRES ? "ci.json"
export const config: SBSConfig = JSON.parse(fs.readFileSync(configFile).toString("utf8"));
addDefaults(config, {
port: 80,
port: 8080,
behindProxy: "X-Forwarded-For",
db: "./databases/sponsorTimes.db",
privateDB: "./databases/private.db",
@@ -19,7 +20,7 @@ addDefaults(config, {
privateDBSchema: "./databases/_private.db.sql",
readOnly: false,
webhooks: [],
categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight", "chapter"],
categoryList: ["sponsor", "selfpromo", "exclusive_access", "interaction", "intro", "outro", "preview", "music_offtopic", "filler", "poi_highlight"],
categorySupport: {
sponsor: ["skip", "mute", "full"],
selfpromo: ["skip", "mute", "full"],
@@ -34,14 +35,14 @@ addDefaults(config, {
chapter: ["chapter"]
},
maxNumberOfActiveWarnings: 1,
hoursAfterWarningExpires: 24,
hoursAfterWarningExpires: 16300000,
adminUserID: "",
discordCompletelyIncorrectReportWebhookURL: null,
discordFirstTimeSubmissionsWebhookURL: null,
discordNeuralBlockRejectWebhookURL: null,
discordFailedReportChannelWebhookURL: null,
discordReportChannelWebhookURL: null,
getTopUsersCacheTimeMinutes: 0,
getTopUsersCacheTimeMinutes: 240,
globalSalt: null,
mode: "",
neuralBlockURL: null,
@@ -49,15 +50,15 @@ addDefaults(config, {
rateLimit: {
vote: {
windowMs: 900000,
max: 20,
message: "Too many votes, please try again later",
statusCode: 429,
max: 15,
message: "OK",
statusCode: 200,
},
view: {
windowMs: 900000,
max: 20,
max: 10,
statusCode: 200,
message: "Too many views, please try again later",
message: "OK",
},
rate: {
windowMs: 900000,
@@ -70,12 +71,17 @@ addDefaults(config, {
newLeafURLs: null,
maxRewardTimePerSegmentInSeconds: 600,
poiMinimumStartTime: 2,
postgres: null,
postgres: {
enabled: false,
user: "",
host: "",
password: "",
port: 5432
},
dumpDatabase: {
enabled: false,
minTimeBetweenMs: 60000,
minTimeBetweenMs: 180000,
appExportPath: "./docker/database-export",
postgresExportPath: "/opt/exports",
tables: [{
name: "sponsorTimes",
order: "timeSubmitted"
@@ -95,11 +101,32 @@ addDefaults(config, {
},
{
name: "vipUsers"
},
{
name: "unlistedVideos"
},
{
name: "videoInfo"
},
{
name: "ratings"
}]
},
diskCache: null,
crons: null
diskCache: {
max: 10737418240
},
crons: null,
redis: {
enabled: false,
socket: {
host: "",
port: 0
},
disableOfflineQueue: true
}
});
loadFromEnv(config);
migrate(config);
// Add defaults
function addDefaults(config: SBSConfig, defaults: SBSConfig) {
@@ -109,3 +136,50 @@ function addDefaults(config: SBSConfig, defaults: SBSConfig) {
}
}
}
function migrate(config: SBSConfig) {
// Redis change
if (config.redis) {
const redisConfig = config.redis as any;
if (redisConfig.host || redisConfig.port) {
config.redis.socket = {
host: redisConfig.host,
port: redisConfig.port
};
}
if (redisConfig.enable_offline_queue !== undefined) {
config.disableOfflineQueue = !redisConfig.enable_offline_queue;
}
if (redisConfig.socket?.host && redisConfig.enabled === undefined) {
redisConfig.enabled = true;
}
}
if (config.postgres && config.postgres.user && config.postgres.enabled === undefined) {
config.postgres.enabled = true;
}
}
function loadFromEnv(config: SBSConfig, prefix = "") {
for (const key in config) {
const fullKey = (prefix ? `${prefix}_` : "") + key;
const data = config[key];
if (typeof data === "object" && !Array.isArray(data)) {
loadFromEnv(data, fullKey);
} else if (process.env[fullKey]) {
const value = process.env[fullKey];
if (isNumber(value)) {
config[key] = parseInt(value, 10);
} else if (value.toLowerCase() === "true" || value.toLowerCase() === "false") {
config[key] = value === "true";
} else if (key === "newLeafURLs") {
config[key] = [value];
} else {
config[key] = value;
}
}
}
}

4
src/databases/Postgres.ts
View File

@@ -94,10 +94,6 @@ export class Postgres implements IDatabase {
await client.query(`CREATE DATABASE "${this.config.postgres.database}"
WITH
OWNER = ${this.config.postgres.user}
ENCODING = 'UTF8'
LC_COLLATE = 'en_US.utf8'
LC_CTYPE = 'en_US.utf8'
TABLESPACE = pg_default
CONNECTION LIMIT = -1;`
);
}

4
src/databases/databases.ts
View File

@@ -9,7 +9,7 @@ let privateDB: IDatabase;
if (config.mysql) {
db = new Mysql(config.mysql);
privateDB = new Mysql(config.privateMysql);
} else if (config.postgres) {
} else if (config.postgres?.enabled) {
db = new Postgres({
dbSchemaFileName: config.dbSchema,
dbSchemaFolder: config.schemaFolder,
@@ -72,7 +72,7 @@ async function initDb(): Promise<void> {
const tables = config?.dumpDatabase?.tables ?? [];
const tableNames = tables.map(table => table.name);
for (const table of tableNames) {
const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`;
const filePath = `${config?.dumpDatabase?.appExportPath}/${table}.csv`;
await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
}
}

15
src/middleware/requestRateLimit.ts
View File

@@ -1,19 +1,23 @@
import { getIP } from "../utils/getIP";
import { getHash } from "../utils/getHash";
import { getHashCache } from "../utils/getHashCache";
import rateLimit from "express-rate-limit";
import rateLimit, { RateLimitRequestHandler } from "express-rate-limit";
import { RateLimitConfig } from "../types/config.model";
import { Request } from "express";
import { isUserVIP } from "../utils/isUserVIP";
import { UserID } from "../types/user.model";
import RedisStore from "rate-limit-redis";
import redis from "../utils/redis";
import { config } from "../config";
export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): rateLimit.RateLimit {
export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (req: Request) => UserID): RateLimitRequestHandler {
return rateLimit({
windowMs: limitConfig.windowMs,
max: limitConfig.max,
message: limitConfig.message,
statusCode: limitConfig.statusCode,
headers: false,
legacyHeaders: false,
standardHeaders: false,
keyGenerator: (req) => {
return getHash(getIP(req), 1);
},
@@ -23,6 +27,9 @@ export function rateLimitMiddleware(limitConfig: RateLimitConfig, getUserID?: (r
} else {
return next();
}
}
},
store: config.redis?.enabled ? new RedisStore({
sendCommand: (...args: string[]) => redis.sendCommand(args),
}) : null,
});
}

25
src/routes/addUserAsTempVIP.ts
View File

@@ -9,6 +9,7 @@ import { isUserVIP } from "../utils/isUserVIP";
import { HashedUserID } from "../types/user.model";
import redis from "../utils/redis";
import { tempVIPKey } from "../utils/redisKeys";
import { Logger } from "../utils/logger";
interface AddUserAsTempVIPRequest extends Request {
query: {
@@ -65,12 +66,22 @@ export async function addUserAsTempVIP(req: AddUserAsTempVIPRequest, res: Respon
if (!channelInfo?.id) {
return res.status(404).send(`No channel found for videoID ${channelVideoID}`);
}
await redis.setAsyncEx(tempVIPKey(userID), channelInfo?.id, dayInSeconds);
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`);
}
await redis.delAsync(tempVIPKey(userID));
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP removed`);
try {
await redis.setEx(tempVIPKey(userID), dayInSeconds, channelInfo?.id);
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP added on channel ${channelInfo?.name}`);
} catch (e) {
Logger.error(e as string);
return res.status(500).send();
}
}
try {
await redis.del(tempVIPKey(userID));
await privateDB.prepare("run", `INSERT INTO "tempVipLog" VALUES (?, ?, ?, ?)`, [adminUserID, userID, + enabled, startTime]);
return res.status(200).send(`Temp VIP removed`);
} catch (e) {
Logger.error(e as string);
return res.status(500).send();
}
}

37
src/routes/dumpDatabase.ts
View File

@@ -5,6 +5,7 @@ import { config } from "../config";
import util from "util";
import fs from "fs";
import path from "path";
import { ChildProcess, exec, ExecOptions, spawn } from "child_process";
const unlink = util.promisify(fs.unlink);
const ONE_MINUTE = 1000 * 60;
@@ -32,9 +33,19 @@ const licenseHeader = `<p>The API and database follow <a href="https://creativec
const tables = config?.dumpDatabase?.tables ?? [];
const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
export const appExportPath = config?.dumpDatabase?.appExportPath ?? "./docker/database-export";
const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? "/opt/exports";
const tableNames = tables.map(table => table.name);
const credentials: ExecOptions = {
env: {
...process.env,
PGHOST: config.postgres.host,
PGPORT: String(config.postgres.port),
PGUSER: config.postgres.user,
PGPASSWORD: String(config.postgres.password),
PGDATABASE: "sponsorTimes",
}
}
interface TableDumpList {
fileName: string;
tableName: string;
@@ -100,7 +111,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) {
if (!config.postgres?.enabled) {
res.status(404).send("Not supported on this instance");
return;
}
@@ -170,12 +181,12 @@ async function getDbVersion(): Promise<number> {
return row.value;
}
export async function redirectLink(req: Request, res: Response): Promise<void> {
export async function downloadFile(req: Request, res: Response): Promise<void> {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) {
if (!config.postgres?.enabled) {
res.status(404).send("Not supported on this instance");
return;
}
@@ -183,7 +194,7 @@ export async function redirectLink(req: Request, res: Response): Promise<void> {
const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path);
if (file) {
res.redirect(`/download/${file.fileName}`);
res.sendFile(file.fileName, { root: appExportPath });
} else {
res.sendStatus(404);
}
@@ -210,9 +221,19 @@ async function queueDump(): Promise<void> {
for (const table of tables) {
const fileName = `${table.name}_${startTime}.csv`;
const file = `${postgresExportPath}/${fileName}`;
await db.prepare("run", `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '${file}' WITH (FORMAT CSV, HEADER true);`);
const file = `${appExportPath}/${fileName}`;
await new Promise<string>((resolve) => {
exec(`psql -c "\\copy (SELECT * FROM \\"${table.name}\\"${table.order ? ` ORDER BY \\"${table.order}\\"` : ``})`
+ ` TO '${file}' WITH (FORMAT CSV, HEADER true);"`, credentials, (error, stdout, stderr) => {
if (error) {
Logger.error(`[dumpDatabase] Failed to dump ${table.name} to ${file} due to ${stderr}`);
}
resolve(error ? stderr : stdout);
});
})
dumpFiles.push({
fileName,
tableName: table.name,

3
src/routes/getSkipSegments.ts
View File

@@ -223,7 +223,8 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
//The 3 makes -2 the minimum votes before being ignored completely
//this can be changed if this system increases in popularity.
const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost);
const repFactor = choice.votes > 0 ? Math.max(1, choice.reputation + 1) : 1;
const weight = Math.exp(choice.votes * repFactor + 3 + boost);
totalWeight += Math.max(weight, 0);
return { ...choice, weight };

8
src/routes/getStatus.ts
View File

@@ -10,8 +10,12 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
value = Array.isArray(value) ? value[0] : value;
try {
const dbVersion = (await db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"])).value;
const numberRequests = await redis.increment("statusRequest");
const statusRequests = numberRequests?.replies?.[0];
let statusRequests: unknown = 0;
try {
const numberRequests = await redis.increment("statusRequest");
statusRequests = numberRequests?.[0];
} catch (error) { } // eslint-disable-line no-empty
const statusValues: Record<string, any> = {
uptime: process.uptime(),
commit: (global as any).HEADCOMMIT || "unknown",

13
src/routes/getTopCategoryUsers.ts
View File

@@ -28,13 +28,16 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
for (const row of rows) {
userNames.push(row.userName);
viewCounts.push(row.viewCount);
totalSubmissions.push(row.totalSubmissions);
minutesSaved.push(row.minutesSaved);
if (rows) {
for (const row of rows) {
userNames.push(row.userName);
viewCounts.push(row.viewCount);
totalSubmissions.push(row.totalSubmissions);
minutesSaved.push(row.minutesSaved);
}
}
return {
userNames,
viewCounts,

238
src/routes/postSkipSegments.ts
View File

@@ -16,6 +16,7 @@ import { getReputation } from "../utils/reputation";
import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
import { HashedUserID, UserID } from "../types/user.model";
import { isUserVIP } from "../utils/isUserVIP";
import { isUserTempVIP } from "../utils/isUserTempVIP";
import { parseUserAgent } from "../utils/userAgent";
import { getService } from "../utils/getService";
import axios from "axios";
@@ -81,19 +82,19 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return;
axios.post(config.discordFirstTimeSubmissionsWebhookURL, {
"embeds": [{
"title": data?.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`,
"description": `Submission ID: ${UUID}\
embeds: [{
title: data?.title,
url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`,
description: `Submission ID: ${UUID}\
\n\nTimestamp: \
${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n\nCategory: ${segmentInfo.category}`,
"color": 10813440,
"author": {
"name": userID,
color: 10813440,
author: {
name: userID,
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
thumbnail: {
url: getMaxResThumbnail(data) || "",
},
}],
})
@@ -112,55 +113,6 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
}
}
async function sendWebhooksNB(userID: string, videoID: string, UUID: string, startTime: number, endTime: number, category: string, probability: number, ytData: any) {
const submissionInfoRow = await db.prepare("get", `SELECT
(select count(1) from "sponsorTimes" where "userID" = ?) count,
(select count(1) from "sponsorTimes" where "userID" = ? and "votes" <= -2) disregarded,
coalesce((select "userName" FROM "userNames" WHERE "userID" = ?), ?) "userName"`,
[userID, userID, userID, userID]);
let submittedBy: string;
// If a userName was created then show both
if (submissionInfoRow.userName !== userID) {
submittedBy = `${submissionInfoRow.userName}\n${userID}`;
} else {
submittedBy = userID;
}
// Send discord message
if (config.discordNeuralBlockRejectWebhookURL === null) return;
axios.post(config.discordNeuralBlockRejectWebhookURL, {
"embeds": [{
"title": ytData.items[0].snippet.title,
"url": `https://www.youtube.com/watch?v=${videoID}&t=${(parseFloat(startTime.toFixed(0)) - 2)}`,
"description": `**Submission ID:** ${UUID}\
\n**Timestamp:** ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\
\n**Predicted Probability:** ${probability}\
\n**Category:** ${category}\
\n**Submitted by:** ${submittedBy}\
\n**Total User Submissions:** ${submissionInfoRow.count}\
\n**Ignored User Submissions:** ${submissionInfoRow.disregarded}`,
"color": 10813440,
"thumbnail": {
"url": ytData.items[0].snippet.thumbnails.maxres ? ytData.items[0].snippet.thumbnails.maxres.url : "",
},
}]
})
.then(res => {
if (res.status >= 400) {
Logger.error("Error sending NeuralBlock Discord hook");
Logger.error(JSON.stringify(res));
Logger.error("\n");
}
})
.catch(err => {
Logger.error("Failed to send NeuralBlock Discord hook.");
Logger.error(JSON.stringify(err));
Logger.error("\n");
});
}
// callback: function(reject: "String containing reason the submission was rejected")
// returns: string when an error, false otherwise
@@ -168,98 +120,46 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem.
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service }) {
if (apiVideoInfo) {
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
const apiVideoDuration = (apiVideoInfo: APIVideoInfo) => {
if (!apiVideoInfo) return undefined;
const { err, data } = apiVideoInfo;
if (err) return false;
// return undefined if API error
if (err) return undefined;
return data?.lengthSeconds;
};
// get duration from API
const apiDuration = apiVideoDuration(apiVideoInfo);
// if API fail or returns 0, get duration from client
const duration = apiDuration || submission.videoDuration;
// return false on undefined or 0
if (!duration) return false;
const duration = apiVideoInfo?.data?.lengthSeconds;
const segments = submission.segments;
let nbString = "";
for (let i = 0; i < segments.length; i++) {
if (duration == 0) {
// Allow submission if the duration is 0 (bug in youtube api)
return false;
} else {
if (segments[i].category === "sponsor") {
//Prepare timestamps to send to NB all at once
nbString = `${nbString}${segments[i].segment[0]},${segments[i].segment[1]};`;
}
}
}
const segments = submission.segments;
// map all times to float array
const allSegmentTimes = segments.map(segment => [parseFloat(segment.segment[0]), parseFloat(segment.segment[1])]);
// Get all submissions for this user
const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]);
const allSegmentTimes = [];
if (allSubmittedByUser !== undefined) {
//add segments the user has previously submitted
for (const segmentInfo of allSubmittedByUser) {
allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
}
}
// add previous submissions by this user
const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? AND "videoID" = ? AND "votes" > -1 AND "hidden" = 0`, [submission.userID, submission.videoID]);
//add segments they are trying to add in this submission
for (let i = 0; i < segments.length; i++) {
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
allSegmentTimes.push([startTime, endTime]);
}
//merge all the times into non-overlapping arrays
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
return a[0] - b[0] || a[1] - b[1];
}));
const videoDuration = data?.lengthSeconds;
if (videoDuration != 0) {
let allSegmentDuration = 0;
//sum all segment times together
allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
if (allSegmentDuration > (videoDuration / 100) * 80) {
// Reject submission if all segments combine are over 80% of the video
return "Total length of your submitted segments are over 80% of the video.";
}
}
// Check NeuralBlock
const neuralBlockURL = config.neuralBlockURL;
if (!neuralBlockURL) return false;
const response = await axios.get(`${neuralBlockURL}/api/checkSponsorSegments?vid=${submission.videoID}
&segments=${nbString.substring(0, nbString.length - 1)}`, { validateStatus: () => true });
if (response.status !== 200) return false;
const nbPredictions = response.data;
let nbDecision = false;
let predictionIdx = 0; //Keep track because only sponsor categories were submitted
for (let i = 0; i < segments.length; i++) {
if (segments[i].category === "sponsor") {
if (nbPredictions.probabilities[predictionIdx] < 0.70) {
nbDecision = true; // At least one bad entry
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
const UUID = getSubmissionUUID(submission.videoID, segments[i].category, segments[i].actionType, submission.userID, startTime, endTime, submission.service);
// Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
}
predictionIdx++;
}
}
if (nbDecision) {
return "Rejected based on NeuralBlock predictions.";
} else {
return false;
}
} else {
Logger.debug("Skipped YouTube API");
// Can't moderate the submission without calling the youtube API
// so allow by default.
return false;
if (allSubmittedByUser) {
//add segments the user has previously submitted
const allSubmittedTimes = allSubmittedByUser.map((segment: { startTime: string, endTime: string }) => [parseFloat(segment.startTime), parseFloat(segment.endTime)]);
allSegmentTimes.push(...allSubmittedTimes);
}
//merge all the times into non-overlapping arrays
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort((a, b) => a[0] - b[0] || a[1] - b[1]));
//sum all segment times together
const allSegmentDuration = allSegmentsSorted.reduce((acc, curr) => acc + (curr[1] - curr[0]), 0);
if (allSegmentDuration > (duration / 100) * 80) {
// Reject submission if all segments combine are over 80% of the video
return "Total length of your submitted segments are over 80% of the video.";
}
return false;
}
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
@@ -310,7 +210,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming
invalidFields.push("userID");
if (userID?.length < 30) errors.push(`userID must be at least 30 characters long`);
}
if (!Array.isArray(segments) || segments.length < 1) {
if (!Array.isArray(segments) || segments.length == 0) {
invalidFields.push("segments");
}
// validate start and end times (no : marks)
@@ -323,7 +223,7 @@ function checkInvalidFields(videoID: VideoID, userID: UserID, segments: Incoming
}
if (typeof segmentPair.description !== "string"
|| (segmentPair.description.length > 60 && segmentPair.actionType === ActionType.Chapter)
|| (segmentPair.actionType === ActionType.Chapter && segmentPair.description.length > 60 )
|| (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) {
invalidFields.push("segment description");
}
@@ -402,7 +302,7 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
}
if (!isVIP && segments[i].category === "sponsor"
&& segments[i].actionType !== ActionType.Full && Math.abs(startTime - endTime) < 1) {
&& segments[i].actionType !== ActionType.Full && (endTime - startTime) < 1) {
// Too short
return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 };
}
@@ -425,32 +325,19 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
return CHECK_PASS;
}
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, isVIP: boolean, service:string, apiVideoInfo: APIVideoInfo, decreaseVotes: number): Promise<CheckResult & { decreaseVotes: number; } > {
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoInfo: APIVideoInfo, videoDuration: number): Promise<CheckResult> {
// Auto moderator check
if (!isVIP && service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service });//startTime, endTime, category: segments[i].category});
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
// If NB automod rejects, the submission will start with -2 votes.
// Note, if one submission is bad all submissions will be affected.
// However, this behavior is consistent with other automod functions
// already in place.
//decreaseVotes = -2; //Disable for now
} else if (autoModerateResult) {
//Normal automod behavior
if (service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service, videoDuration });
if (autoModerateResult) {
return {
pass: false,
errorCode: 403,
errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`,
decreaseVotes
errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
};
}
}
return {
...CHECK_PASS,
decreaseVotes
};
return CHECK_PASS;
}
async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
@@ -601,11 +488,12 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
const userWarningCheckResult = await checkUserActiveWarning(userID);
if (!userWarningCheckResult.pass) {
Logger.warn(`Caught a submission for for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`);
Logger.warn(`Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`);
return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage);
}
const isVIP = await isUserVIP(userID);
const isTempVIP = await isUserTempVIP(userID, videoID);
const rawIP = getIP(req);
const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam);
@@ -618,13 +506,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
return res.status(segmentCheckResult.errorCode).send(segmentCheckResult.errorMessage);
}
let decreaseVotes = 0;
// Auto check by NB
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, isVIP, service, apiVideoInfo, decreaseVotes);
if (!autoModerateCheckResult.pass) {
return res.status(autoModerateCheckResult.errorCode).send(autoModerateCheckResult.errorMessage);
} else {
decreaseVotes = autoModerateCheckResult.decreaseVotes;
if (!isVIP && !isTempVIP) {
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoInfo, videoDurationParam);
if (!autoModerateCheckResult.pass) {
return res.status(autoModerateCheckResult.errorCode).send(autoModerateCheckResult.errorMessage);
}
}
// Will be filled when submitting
@@ -645,7 +531,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//check to see if this user is shadowbanned
const shadowBanRow = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ? LIMIT 1`, [userID]);
const startingVotes = 0 + decreaseVotes;
const startingVotes = 0;
const reputation = await getReputation(userID);
for (const segmentInfo of segments) {

26
src/routes/voteOnSponsorTime.ts
View File

@@ -1,6 +1,7 @@
import { Request, Response } from "express";
import { Logger } from "../utils/logger";
import { isUserVIP } from "../utils/isUserVIP";
import { isUserTempVIP } from "../utils/isUserTempVIP";
import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi";
import { APIVideoInfo } from "../types/youtubeApi.model";
import { db, privateDB } from "../databases/databases";
@@ -9,12 +10,10 @@ import { getFormattedTime } from "../utils/getFormattedTime";
import { getIP } from "../utils/getIP";
import { getHashCache } from "../utils/getHashCache";
import { config } from "../config";
import { HashedUserID, UserID } from "../types/user.model";
import { UserID } from "../types/user.model";
import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, VideoDuration, ActionType } from "../types/segments.model";
import { QueryCacher } from "../utils/queryCacher";
import axios from "axios";
import redis from "../utils/redis";
import { tempVIPKey } from "../utils/redisKeys";
const voteTypes = {
normal: 0,
@@ -44,6 +43,7 @@ interface VoteData {
row: {
votes: number;
views: number;
locked: boolean;
};
category: string;
incrementAmount: number;
@@ -55,14 +55,6 @@ function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<API
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
}
const isUserTempVIP = async (nonAnonUserID: HashedUserID, videoID: VideoID): Promise<boolean> => {
const apiVideoInfo = await getYouTubeVideoInfo(videoID);
const channelID = apiVideoInfo?.data?.authorId;
const { err, reply } = await redis.getAsync(tempVIPKey(nonAnonUserID));
return err || !reply ? false : (reply == channelID);
};
const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2);
async function updateSegmentVideoDuration(UUID: SegmentUUID) {
@@ -178,7 +170,7 @@ async function sendWebhooks(voteData: VoteData) {
"url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}s#requiredSegment=${voteData.UUID}`,
"description": `**${voteData.row.votes} Votes Prior | \
${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \
Views**\n\n**Submission ID:** ${voteData.UUID}\
Views**\n\n**Locked**: ${voteData.row.locked}\n\n**Submission ID:** ${voteData.UUID}\
\n**Category:** ${submissionInfoRow.category}\
\n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\
\n\n**Total User Submissions:** ${submissionInfoRow.count}\
@@ -189,7 +181,7 @@ async function sendWebhooks(voteData: VoteData) {
"author": {
"name": voteData.finalResponse?.webhookMessage ??
voteData.finalResponse?.finalMessage ??
getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission),
`${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`,
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
@@ -476,11 +468,11 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
// Only change the database if they have made a submission before and haven't voted recently
const userAbleToVote = (!(isOwnSubmission && incrementAmount > 0 && oldIncrementAmount >= 0)
&& !finalResponse.blockVote
&& finalResponse.finalStatus === 200
&& (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
&& !finalResponse.blockVote
&& finalResponse.finalStatus === 200;
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined);
const ableToVote = isVIP || isTempVIP || userAbleToVote;
@@ -534,4 +526,4 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
Logger.error(err as string);
return { status: 500, message: finalResponse.finalMessage ?? undefined, json: { error: "Internal error creating segment vote" } };
}
}
}

13
src/types/config.model.ts
View File

@@ -2,6 +2,14 @@ import { PoolConfig } from "pg";
import * as redis from "redis";
import { CacheOptions } from "@ajayyy/lru-diskcache";
interface RedisConfig extends redis.RedisClientOptions {
enabled: boolean;
}
interface CustomPostgresConfig extends PoolConfig {
enabled: boolean;
}
export interface SBSConfig {
[index: string]: any
port: number;
@@ -41,9 +49,9 @@ export interface SBSConfig {
privateMysql?: any;
minimumPrefix?: string;
maximumPrefix?: string;
redis?: redis.ClientOpts;
redis?: RedisConfig;
maxRewardTimePerSegmentInSeconds?: number;
postgres?: PoolConfig;
postgres?: CustomPostgresConfig;
dumpDatabase?: DumpDatabase;
diskCache: CacheOptions;
crons: CronJobOptions;
@@ -76,7 +84,6 @@ export interface DumpDatabase {
enabled: boolean;
minTimeBetweenMs: number;
appExportPath: string;
postgresExportPath: string;
tables: DumpDatabaseTable[];
}

17
src/utils/getHashCache.ts
View File

@@ -18,18 +18,19 @@ export async function getHashCache<T extends string>(value: T, times = defaulted
async function getFromRedis<T extends string>(key: HashedValue): Promise<T & HashedValue> {
const redisKey = shaHashKey(key);
const { err, reply } = await redis.getAsync(redisKey);
if (!err && reply) {
try {
try {
const reply = await redis.get(redisKey);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
return reply as T & HashedValue;
} catch (e) {
// If all else, continue on hashing
}
}
const data = getHash(key, cachedHashTimes);
} catch (e) {} // eslint-disable-line no-empty
// Otherwise, calculate it
const data = getHash(key, cachedHashTimes);
redis.set(key, data);
redis.setAsync(key, data);
return data as T & HashedValue;
}

24
src/utils/isUserTempVIP.ts Normal file
View File

@@ -0,0 +1,24 @@
import redis from "../utils/redis";
import { tempVIPKey } from "../utils/redisKeys";
import { HashedUserID } from "../types/user.model";
import { YouTubeAPI } from "../utils/youtubeApi";
import { APIVideoInfo } from "../types/youtubeApi.model";
import { VideoID } from "../types/segments.model";
import { config } from "../config";
import { Logger } from "./logger";
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
}
export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID): Promise<boolean> => {
const apiVideoInfo = await getYouTubeVideoInfo(videoID);
const channelID = apiVideoInfo?.data?.authorId;
try {
const reply = await redis.get(tempVIPKey(hashedUserID));
return reply && reply == channelID;
} catch (e) {
Logger.error(e as string);
return false;
}
};

36
src/utils/queryCacher.ts
View File

@@ -5,21 +5,18 @@ import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { UserID } from "../types/user.model";
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
const { err, reply } = await redis.getAsync(key);
if (!err && reply) {
try {
try {
const reply = await redis.get(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
}
}
} catch (e) { } //eslint-disable-line no-empty
const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data));
redis.set(key, JSON.stringify(data));
return data;
}
@@ -30,18 +27,17 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Promise<Array<T>>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise<Array<T>> {
const cachedValues = await Promise.all(values.map(async (value) => {
const key = keyGenerator(value);
const { err, reply } = await redis.getAsync(key);
if (!err && reply) {
try {
try {
const reply = await redis.get(key);
if (reply) {
Logger.debug(`Got data from redis: ${reply}`);
return {
value,
result: JSON.parse(reply)
};
} catch (e) { } //eslint-disable-line no-empty
}
}
} catch (e) { } //eslint-disable-line no-empty
return {
value,
@@ -71,7 +67,7 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
}
for (const key in newResults) {
redis.setAsync(key, JSON.stringify(newResults[key]));
redis.set(key, JSON.stringify(newResults[key]));
}
});
}
@@ -81,16 +77,16 @@ async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Pr
function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void {
if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID));
redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service));
redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
if (videoInfo.userID) redis.del(reputationKey(videoInfo.userID));
}
}
function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
if (videoInfo) {
redis.delAsync(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service));
redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service));
}
}

71
src/utils/redis.ts
View File

@@ -1,59 +1,56 @@
import { config } from "../config";
import { Logger } from "./logger";
import redis, { Callback } from "redis";
import { createClient } from "redis";
import { RedisCommandArgument, RedisCommandArguments, RedisCommandRawReply } from "@node-redis/client/dist/lib/commands";
import { ClientCommandOptions } from "@node-redis/client/dist/lib/client";
import { RedisReply } from "rate-limit-redis";
interface RedisSB {
get(key: string, callback?: Callback<string | null>): void;
getAsync?(key: string): Promise<{err: Error | null, reply: string | null}>;
set(key: string, value: string, callback?: Callback<string | null>): void;
setAsync?(key: string, value: string): Promise<{err: Error | null, reply: string | null}>;
setAsyncEx?(key: string, value: string, seconds: number): Promise<{err: Error | null, reply: string | null}>;
delAsync?(...keys: [string]): Promise<Error | null>;
close?(flush?: boolean): void;
increment?(key: string): Promise<{err: Error| null, replies: any[] | null}>;
get(key: RedisCommandArgument): Promise<string>;
set(key: RedisCommandArgument, value: RedisCommandArgument): Promise<string>;
setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise<string>;
del(...keys: [RedisCommandArgument]): Promise<number>;
increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
sendCommand(args: RedisCommandArguments, options?: ClientCommandOptions): Promise<RedisReply>;
quit(): Promise<void>;
}
let exportObject: RedisSB = {
get: (key, callback?) => callback(null, undefined),
getAsync: () =>
new Promise((resolve) => resolve({ err: null, reply: undefined })),
set: (key, value, callback) => callback(null, undefined),
setAsync: () =>
new Promise((resolve) => resolve({ err: null, reply: undefined })),
setAsyncEx: () =>
new Promise((resolve) => resolve({ err: null, reply: undefined })),
delAsync: () =>
new Promise((resolve) => resolve(null)),
increment: () =>
new Promise((resolve) => resolve({ err: null, replies: undefined })),
let exportClient: RedisSB = {
get: () => new Promise((resolve) => resolve(null)),
set: () => new Promise((resolve) => resolve(null)),
setEx: () => new Promise((resolve) => resolve(null)),
del: () => new Promise((resolve) => resolve(null)),
increment: () => new Promise((resolve) => resolve(null)),
sendCommand: () => new Promise((resolve) => resolve(null)),
quit: () => new Promise((resolve) => resolve(null)),
};
if (config.redis) {
if (config.redis?.enabled) {
Logger.info("Connected to redis");
const client = redis.createClient(config.redis);
exportObject = client;
const client = createClient(config.redis);
client.connect();
exportClient = client;
const timeoutDuration = 200;
exportObject.getAsync = (key) => new Promise((resolve) => {
const timeout = setTimeout(() => resolve({ err: null, reply: undefined }), timeoutDuration);
client.get(key, (err, reply) => {
const get = client.get.bind(client);
exportClient.get = (key) => new Promise((resolve, reject) => {
const timeout = setTimeout(() => reject(), timeoutDuration);
get(key).then((reply) => {
clearTimeout(timeout);
resolve({ err, reply });
});
resolve(reply);
}).catch((err) => reject(err));
});
exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({ err, reply })));
exportObject.setAsyncEx = (key, value, seconds) => new Promise((resolve) => client.setex(key, seconds, value, (err, reply) => resolve({ err, reply })));
exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
exportObject.close = (flush) => client.end(flush);
exportObject.increment = (key) => new Promise((resolve) =>
exportClient.increment = (key) => new Promise((resolve, reject) =>
client.multi()
.incr(key)
.expire(key, 60)
.exec((err, replies) => resolve({ err, replies }))
.exec()
.then((reply) => resolve(reply))
.catch((err) => reject(err))
);
client.on("error", function(error) {
Logger.error(error);
});
}
export default exportObject;
export default exportClient;