Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-25 08:58:23 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/general-fixes

Browse Source
This commit is contained in:
Nishant Arora
2021-10-27 23:51:45 -06:00
committed by GitHub
parent 70d0387356 19f7bbcde5
commit 246ec7c3c3
18 changed files with 220 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/databases/databases.ts
View File

@@ -4,7 +4,6 @@ import { Mysql } from "./Mysql";
import { Postgres } from "./Postgres";
import { IDatabase } from "./IDatabase";
let db: IDatabase;
let privateDB: IDatabase;
if (config.mysql) {
@@ -68,6 +67,15 @@ async function initDb(): Promise<void> {
// Attach private db to main db
(db as Sqlite).attachDatabase(config.privateDB, "privateDB");
}
if (config.mode === "mirror" && db instanceof Postgres) {
const tables = config?.dumpDatabase?.tables ?? [];
const tableNames = tables.map(table => table.name);
for (const table of tableNames) {
const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`;
await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
}
}
}
export {

4
src/index.ts
View File

@@ -7,11 +7,15 @@ import { getCommit } from "./utils/getCommit";
async function init() {
process.on("unhandledRejection", (error: any) => {
// eslint-disable-next-line no-console
console.dir(error?.stack);
process.exit(1);
});
await initDb();
// edge case clause for creating compatible .db files, do not enable
if (config.mode === "init-db-and-exit") process.exit(0);
// do not enable init-db-only mode for usage.
(global as any).HEADCOMMIT = config.mode === "development" ? "development"
: config.mode === "test" ? "test"
: getCommit() as string;

8
src/routes/dumpDatabase.ts
View File

@@ -118,6 +118,12 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
Then, you can download the csv files below, or use the links returned from the JSON request.
A dump will also be triggered by making a request to one of these urls.
<h3>Keeping your dump up to date</h3>
If you want a live dump, please do not continually fetch this url.
Please instead use the <a href="https://github.com/mchangrh/sb-mirror">sb-mirror</a> project.
This can automatically fetch new data and will not require a redownload each time, saving bandwith.
<h3>Links</h3>
<table>
<thead>
@@ -184,7 +190,7 @@ export async function redirectLink(req: Request, res: Response): Promise<void> {
res.sendStatus(404);
}
await queueDump();
if (req.query.generate !== "false") await queueDump();
}
function updateQueueTime(): void {

52
src/routes/getLockReason.ts
View File

@@ -8,7 +8,9 @@ const possibleCategoryList = config.categoryList;
interface lockArray {
category: Category;
locked: number,
reason: string
reason: string,
userID: string,
userName: string,
}
export async function getLockReason(req: Request, res: Response): Promise<Response> {
@@ -38,31 +40,49 @@ export async function getLockReason(req: Request, res: Response): Promise<Respon
try {
// Get existing lock categories markers
const row = await db.prepare("all", 'SELECT "category", "reason" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string}[];
const row = await db.prepare("all", 'SELECT "category", "reason", "userID" from "lockCategories" where "videoID" = ?', [videoID]) as {category: Category, reason: string, userID: string }[];
// map to object array
const locks = [];
const lockedCategories = [] as string[];
const userIDs = new Set();
// get all locks for video, check if requested later
for (const lock of row) {
locks.push({
category: lock.category,
locked: 1,
reason: lock.reason
reason: lock.reason,
userID: lock?.userID || "",
userName: "",
} as lockArray);
lockedCategories.push(lock.category);
userIDs.add(lock.userID);
}
// add empty locks for categories requested but not locked
const noLockCategories = searchCategories.filter(x => !lockedCategories.includes(x));
for (const noLock of noLockCategories) {
locks.push({
category: noLock,
locked: 0,
reason: ""
} as lockArray);
// all userName from userIDs
const userNames = await db.prepare(
"all",
`SELECT "userName", "userID" FROM "userNames" WHERE "userID" IN (${Array.from("?".repeat(userIDs.size)).join()}) LIMIT ?`,
[...userIDs, userIDs.size]
) as { userName: string, userID: string }[];
const results = [];
for (const category of searchCategories) {
const lock = locks.find(l => l.category === category);
if (lock?.userID) {
// mapping userName to locks
const user = userNames.find(u => u.userID === lock.userID);
lock.userName = user?.userName || "";
results.push(lock);
} else {
// add empty locks for categories requested but not locked
results.push({
category,
locked: 0,
reason: "",
userID: "",
userName: "",
} as lockArray);
}
}
// return real and fake locks that were requested
const filtered = locks.filter(lock => searchCategories.includes(lock.category));
return res.send(filtered);
return res.send(results);
} catch (err) {
Logger.error(err as string);
return res.sendStatus(500);

4
src/routes/getStatus.ts
View File

@@ -1,6 +1,7 @@
import { db } from "../databases/databases";
import { Logger } from "../utils/logger";
import { Request, Response } from "express";
import os from "os";
export async function getStatus(req: Request, res: Response): Promise<Response> {
const startTime = Date.now();
@@ -14,8 +15,9 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
db: Number(dbVersion),
startTime,
processTime: Date.now() - startTime,
loadavg: os.loadavg().slice(1) // only return 5 & 15 minute load average
};
return value ? res.send(String(statusValues[value])) : res.send(statusValues);
return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues);
} catch (err) {
Logger.error(err as string);
return res.sendStatus(500);

10
src/routes/postSkipSegments.ts
View File

@@ -6,7 +6,6 @@ import { getSubmissionUUID } from "../utils/getSubmissionUUID";
import { getHash } from "../utils/getHash";
import { getIP } from "../utils/getIP";
import { getFormattedTime } from "../utils/getFormattedTime";
import { isUserTrustworthy } from "../utils/isUserTrustworthy";
import { dispatchEvent } from "../utils/webhookUtils";
import { Request, Response } from "express";
import { ActionType, Category, CategoryActionType, IncomingSegment, SegmentUUID, Service, VideoDuration, VideoID } from "../types/segments.model";
@@ -622,13 +621,6 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//check to see if this user is shadowbanned
const shadowBanRow = await db.prepare("get", `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ? LIMIT 1`, [userID]);
let shadowBanned = shadowBanRow.userCount;
if (!(await isUserTrustworthy(userID))) {
//hide this submission as this user is untrustworthy
shadowBanned = 1;
}
const startingVotes = 0 + decreaseVotes;
const reputation = await getReputation(userID);
@@ -644,7 +636,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
await db.prepare("run", `INSERT INTO "sponsorTimes"
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "actionType", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "userAgent")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, segmentInfo.actionType, service, videoDuration, reputation, shadowBanned, hashedVideoID, userAgent
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, segmentInfo.actionType, service, videoDuration, reputation, 0, hashedVideoID, userAgent
],
);

20
src/utils/isUserTrustworthy.ts
View File

@@ -1,20 +0,0 @@
import { db } from "../databases/databases";
/**
* Returns true if the user is considered trustworthy. This happens after a user has made 5 submissions and has less than 60% downvoted submissions
* @param userID
*/
export async function isUserTrustworthy(userID: string): Promise<boolean> {
//check to see if this user how many submissions this user has submitted
const totalSubmissionsRow = await db.prepare("get", `SELECT count(*) as "totalSubmissions", sum(votes) as "voteSum" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
if (totalSubmissionsRow.totalSubmissions > 5) {
//check if they have a high downvote ratio
const downvotedSubmissionsRow = await db.prepare("get", `SELECT count(*) as "downvotedSubmissions" FROM "sponsorTimes" WHERE "userID" = ? AND (votes < 0 OR "shadowHidden" > 0)`, [userID]);
return (downvotedSubmissionsRow.downvotedSubmissions / totalSubmissionsRow.totalSubmissions) < 0.6 ||
(totalSubmissionsRow.voteSum > downvotedSubmissionsRow.downvotedSubmissions);
}
return true;
}

1
src/utils/logger.ts
View File

@@ -79,6 +79,7 @@ class Logger {
if (levelStr.length === 4) {
levelStr += " "; // ensure logs are aligned
}
// eslint-disable-next-line no-console
console.log(colors.Dim, `${levelStr} ${new Date().toISOString()}: `, color, str, colors.Reset);
}
}