Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-24 16:38:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into pr/FlorianZahn/376

Browse Source
This commit is contained in:
Ajay Ramachandran
2021-10-03 13:11:25 -04:00
parent 69e321c405 b5d9c02d9e
commit 658e391f50
21 changed files with 232 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/routes/addUnlistedVideo.ts
View File

@@ -1,5 +1,6 @@
import { Request, Response } from "express";
import { db } from "../databases/databases";
import { getService } from "../utils/getService";
import { Logger } from "../utils/logger";
/**
@@ -14,6 +15,7 @@ export function addUnlistedVideo(req: Request, res: Response): Response {
const year = req.body.year || 0;
const views = req.body.views || 0;
const channelID = req.body.channelID || "Unknown";
const service = getService(req.body.service);
if (videoID === undefined || typeof(videoID) !== "string" || videoID.length !== 11) {
return res.status(400).send("Invalid parameters");
@@ -21,7 +23,7 @@ export function addUnlistedVideo(req: Request, res: Response): Response {
try {
const timeSubmitted = Date.now();
db.prepare("run", `INSERT INTO "unlistedVideos" ("videoID", "year", "views", "channelID", "timeSubmitted") values (?, ?, ?, ?, ?)`, [videoID, year, views, channelID, timeSubmitted]);
db.prepare("run", `INSERT INTO "unlistedVideos" ("videoID", "year", "views", "channelID", "timeSubmitted", "service") values (?, ?, ?, ?, ?, ?)`, [videoID, year, views, channelID, timeSubmitted, service]);
} catch (err) {
Logger.error(err as string);
return res.sendStatus(500);

23
src/routes/deleteLockCategories.ts
View File

@@ -2,14 +2,16 @@ import { Request, Response } from "express";
import { isUserVIP } from "../utils/isUserVIP";
import { getHash } from "../utils/getHash";
import { db } from "../databases/databases";
import { Category, VideoID } from "../types/segments.model";
import { Category, Service, VideoID } from "../types/segments.model";
import { UserID } from "../types/user.model";
import { getService } from "../utils/getService";
export async function deleteLockCategoriesEndpoint(req: Request, res: Response): Promise<Response> {
// Collect user input data
const videoID = req.body.videoID as VideoID;
const userID = req.body.userID as UserID;
const categories = req.body.categories as Category[];
const service = getService(req.body.service);
// Check input data is valid
if (!videoID
@@ -33,7 +35,7 @@ export async function deleteLockCategoriesEndpoint(req: Request, res: Response):
});
}
await deleteLockCategories(videoID, categories);
await deleteLockCategories(videoID, categories, service);
return res.status(200).json({ message: `Removed lock categories entrys for video ${videoID}` });
}
@@ -42,13 +44,20 @@ export async function deleteLockCategoriesEndpoint(req: Request, res: Response):
*
* @param videoID
* @param categories If null, will remove all
* @param service
*/
export async function deleteLockCategories(videoID: VideoID, categories: Category[]): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
return categories === null || categories.indexOf(entry.category) !== -1;
});
export async function deleteLockCategories(videoID: VideoID, categories: Category[], service: Service): Promise<void> {
const entries = (
await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ? AND "service" = ?', [videoID, service]))
.filter((entry: any) => {
return categories === null || categories.indexOf(entry.category) !== -1;
});
for (const entry of entries) {
await db.prepare("run", 'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
await db.prepare(
"run",
'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?',
[videoID, service, entry.category]
);
}
}

4
src/routes/getLockCategories.ts
View File

@@ -2,9 +2,11 @@ import { db } from "../databases/databases";
import { Logger } from "../utils/logger";
import { Request, Response } from "express";
import { Category, VideoID } from "../types/segments.model";
import { getService } from "../utils/getService";
export async function getLockCategories(req: Request, res: Response): Promise<Response> {
const videoID = req.query.videoID as VideoID;
const service = getService(req.query.service as string);
if (videoID == undefined) {
//invalid request
@@ -13,7 +15,7 @@ export async function getLockCategories(req: Request, res: Response): Promise<Re
try {
// Get existing lock categories markers
const row = await db.prepare("all", 'SELECT "category", "reason" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string}[];
const row = await db.prepare("all", 'SELECT "category", "reason" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]) as {category: Category, reason: string}[];
// map categories to array in JS becaues of SQL incompatibilities
const categories = row.map(item => item.category);
if (categories.length === 0 || !categories[0]) return res.sendStatus(404);

5
src/routes/getSkipSegments.ts
View File

@@ -27,8 +27,9 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
if (cache.shadowHiddenSegmentIPs[videoID] === undefined) cache.shadowHiddenSegmentIPs[videoID] = {};
if (cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] === undefined) {
cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ?',
[videoID, segment.timeSubmitted]) as { hashedIP: HashedIP }[];
const service = getService(req?.query?.service as string);
cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ? AND "service" = ?',
[videoID, segment.timeSubmitted, service]) as { hashedIP: HashedIP }[];
}
//if this isn't their ip, don't send it to them

14
src/routes/postLockCategories.ts
View File

@@ -4,6 +4,7 @@ import { isUserVIP } from "../utils/isUserVIP";
import { db } from "../databases/databases";
import { Request, Response } from "express";
import { VideoIDHash } from "../types/segments.model";
import { getService } from "../utils/getService";
export async function postLockCategories(req: Request, res: Response): Promise<string[]> {
// Collect user input data
@@ -11,6 +12,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
let userID = req.body.userID;
const categories = req.body.categories;
const reason: string = req.body.reason ?? "";
const service = getService(req.body.service);
// Check input data is valid
if (!videoID
@@ -37,7 +39,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
}
// Get existing lock categories markers
let noCategoryList = await db.prepare("all", 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]);
let noCategoryList = await db.prepare("all", 'SELECT "category" from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
if (!noCategoryList || noCategoryList.length === 0) {
noCategoryList = [];
} else {
@@ -65,9 +67,9 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
// create database entry
for (const category of categoriesToMark) {
try {
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason") VALUES(?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason]);
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason", "service") VALUES(?, ?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason, service]);
} catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`);
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}' (${service})`);
Logger.error(err as string);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",
@@ -85,10 +87,10 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
for (const category of overlapCategories) {
try {
await db.prepare("run",
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ?',
[reason, userID, videoID, category]);
'UPDATE "lockCategories" SET "reason" = ?, "userID" = ? WHERE "videoID" = ? AND "category" = ? AND "service" = ?',
[reason, userID, videoID, category, service]);
} catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`);
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID} (${service})'`);
Logger.error(err as string);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",

23
src/routes/postSkipSegments.ts
View File

@@ -168,7 +168,7 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
// false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem.
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[] }) {
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service }) {
if (apiVideoInfo) {
const { err, data } = apiVideoInfo;
if (err) return false;
@@ -238,7 +238,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
const startTime = parseFloat(segments[i].segment[0]);
const endTime = parseFloat(segments[i].segment[1]);
const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime);
const UUID = getSubmissionUUID(submission.videoID, segments[i].actionType, submission.userID, startTime, endTime, submission.service);
// Send to Discord
// Note, if this is too spammy. Consider sending all the segments as one Webhook
sendWebhooksNB(submission.userID, submission.videoID, UUID, startTime, endTime, segments[i].category, nbPredictions.probabilities[predictionIdx], data);
@@ -407,7 +407,8 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID,
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, isVIP: boolean, service:string, apiVideoInfo: APIVideoInfo, decreaseVotes: number): Promise<CheckResult & { decreaseVotes: number; } > {
// Auto moderator check
if (!isVIP && service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments });//startTime, endTime, category: segments[i].category});
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service });//startTime, endTime, category: segments[i].category});
if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
// If NB automod rejects, the submission will start with -2 votes.
// Note, if one submission is bad all submissions will be affected.
@@ -431,8 +432,8 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
};
}
async function updateDataIfVideoDurationChange(videoID: VideoID, service: string, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
let lockedCategoryList = await db.prepare("all", 'SELECT category, reason from "lockCategories" where "videoID" = ?', [videoID]);
async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) {
let lockedCategoryList = await db.prepare("all", 'SELECT category, reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
const previousSubmissions = await db.prepare("all",
`SELECT "videoDuration", "UUID"
@@ -465,7 +466,7 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: string
await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
}
lockedCategoryList = [];
deleteLockCategories(videoID, null);
deleteLockCategories(videoID, null, service);
}
return {
@@ -478,7 +479,7 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: string
// Disable max submissions for now
// Disable IP ratelimiting for now
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function checkRateLimit(userID:string, videoID: VideoID, timeSubmitted: number, hashedIP: string, options: {
async function checkRateLimit(userID:string, videoID: VideoID, service: Service, timeSubmitted: number, hashedIP: string, options: {
enableCheckByIP: boolean;
enableCheckByUserID: boolean;
} = {
@@ -489,7 +490,7 @@ async function checkRateLimit(userID:string, videoID: VideoID, timeSubmitted: nu
if (options.enableCheckByIP) {
//check to see if this ip has submitted too many sponsors today
const rateLimitCheckRow = await privateDB.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "hashedIP" = ? AND "videoID" = ? AND "timeSubmitted" > ?`, [hashedIP, videoID, yesterday]);
const rateLimitCheckRow = await privateDB.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "hashedIP" = ? AND "videoID" = ? AND "timeSubmitted" > ? AND "service" = ?`, [hashedIP, videoID, yesterday, service]);
if (rateLimitCheckRow.count >= 10) {
//too many sponsors for the same video from the same ip address
@@ -613,7 +614,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//get current time
const timeSubmitted = Date.now();
// const rateLimitCheckResult = checkRateLimit(userID, videoID, timeSubmitted, hashedIP);
// const rateLimitCheckResult = checkRateLimit(userID, videoID, service, timeSubmitted, hashedIP);
// if (!rateLimitCheckResult.pass) {
// return res.status(rateLimitCheckResult.errorCode).send(rateLimitCheckResult.errorMessage);
// }
@@ -635,7 +636,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//this can just be a hash of the data
//it's better than generating an actual UUID like what was used before
//also better for duplication checking
const UUID = getSubmissionUUID(videoID, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
const UUID = getSubmissionUUID(videoID, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service);
const hashedVideoID = getHash(videoID, 1);
const startingLocked = isVIP ? 1 : 0;
@@ -648,7 +649,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
);
//add to private db as well
await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?)`, [videoID, hashedIP, timeSubmitted]);
await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?, ?)`, [videoID, hashedIP, timeSubmitted, service]);
// Clear redis cache for this video
QueryCacher.clearVideoCache({

4
src/routes/shadowBanUser.ts
View File

@@ -57,7 +57,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
//find all previous submissions and unhide them
if (unHideOldSubmissions) {
const segmentsToIgnore = (await db.prepare("all", `SELECT "UUID" FROM "sponsorTimes" st
JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?`
JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category AND "st"."service" = "ns"."service" WHERE "st"."userID" = ?`
, [userID])).map((item: {UUID: string}) => item.UUID);
const allSegments = (await db.prepare("all", `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID]))
.map((item: {UUID: string}) => item.UUID);
@@ -120,7 +120,7 @@ export async function shadowBanUser(req: Request, res: Response): Promise<Respon
async function unHideSubmissions(categories: string[], userID: UserID) {
await db.prepare("run", `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})
AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
"sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
"sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."service" = "lockCategories"."service" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
// clear cache for all old videos
(await db.prepare("all", `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]))

3
src/routes/voteOnSponsorTime.ts
View File

@@ -301,7 +301,8 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
if (!isVIP && type != 1) {
const isSegmentLocked = async () => !!(await db.prepare("get", `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked;
const isVideoLocked = async () => !!(await db.prepare("get", `SELECT "lockCategories".category from "lockCategories" left join "sponsorTimes"
on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and "lockCategories".category = "sponsorTimes".category)
on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and
"lockCategories"."service" = "sponsorTimes"."service" and "lockCategories".category = "sponsorTimes".category)
where "UUID" = ?`, [UUID]));
if (await isSegmentLocked() || await isVideoLocked()) {