Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-17 13:08:49 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into fix/prepare-statements

Browse Source
This commit is contained in:
Nanobyte
2021-06-21 18:19:50 +02:00
parent 1122681c4f 9351bef61c
commit 11e6c93932
53 changed files with 2764 additions and 2133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/app.ts
View File

@@ -25,9 +25,11 @@ import {endpoint as getSkipSegments} from './routes/getSkipSegments';
import {userCounter} from './middleware/userCounter';
import {loggerMiddleware} from './middleware/logger';
import {corsMiddleware} from './middleware/cors';
import {apiCspMiddleware} from './middleware/apiCsp';
import {rateLimitMiddleware} from './middleware/requestRateLimit';
import dumpDatabase, {redirectLink} from './routes/dumpDatabase';
import {endpoint as getSegmentInfo} from './routes/getSegmentInfo';
import {postClearCache} from './routes/postClearCache';
export function createServer(callback: () => void) {
// Create a service (the app object is just a callback).
@@ -36,6 +38,7 @@ export function createServer(callback: () => void) {
//setup CORS correctly
app.use(corsMiddleware);
app.use(loggerMiddleware);
app.use("/api/", apiCspMiddleware);
app.use(express.json());
if (config.userCounterURL) app.use(userCounter);
@@ -110,6 +113,7 @@ function setupRoutes(app: Express) {
app.get('/api/getTotalStats', getTotalStats);
app.get('/api/getUserInfo', getUserInfo);
app.get('/api/userInfo', getUserInfo);
//send out a formatted time saved total
app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
@@ -130,6 +134,12 @@ function setupRoutes(app: Express) {
//get if user is a vip
app.post('/api/segmentShift', postSegmentShift);
//get segment info
app.get('/api/segmentInfo', getSegmentInfo);
//clear cache as VIP
app.post('/api/clearCache', postClearCache)
if (config.postgres) {
app.get('/database', (req, res) => dumpDatabase(req, res, true));
app.get('/database.json', (req, res) => dumpDatabase(req, res, false));

13
src/config.ts
View File

@@ -16,14 +16,15 @@ addDefaults(config, {
privateDBSchema: "./databases/_private.db.sql",
readOnly: false,
webhooks: [],
categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic"],
categoryList: ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"],
maxNumberOfActiveWarnings: 3,
hoursAfterWarningExpires: 24,
adminUserID: "",
discordCompletelyIncorrectReportWebhookURL: "",
discordFirstTimeSubmissionsWebhookURL: "",
discordNeuralBlockRejectWebhookURL: "",
discordReportChannelWebhookURL: "",
discordCompletelyIncorrectReportWebhookURL: null,
discordFirstTimeSubmissionsWebhookURL: null,
discordNeuralBlockRejectWebhookURL: null,
discordFailedReportChannelWebhookURL: null,
discordReportChannelWebhookURL: null,
getTopUsersCacheTimeMinutes: 0,
globalSalt: null,
mode: "",
@@ -44,7 +45,7 @@ addDefaults(config, {
},
},
userCounterURL: null,
youtubeAPIKey: null,
newLeafURLs: null,
maxRewardTimePerSegmentInSeconds: 86400,
postgres: null,
dumpDatabase: {

2
src/databases/IDatabase.ts
View File

@@ -1,5 +1,5 @@
export interface IDatabase {
async init(): Promise<void>;
init(): Promise<void>;
prepare(type: QueryType, query: string, params?: any[]): Promise<any | any[] | void>;
}

6
src/middleware/apiCsp.ts Normal file
View File

@@ -0,0 +1,6 @@
import {NextFunction, Request, Response} from 'express';
export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) {
res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
next();
}

3
src/middleware/cors.ts
View File

@@ -2,6 +2,7 @@ import {NextFunction, Request, Response} from 'express';
export function corsMiddleware(req: Request, res: Response, next: NextFunction) {
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
res.header("Access-Control-Allow-Headers", "X-Requested-With, Content-Type, Accept");
res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE")
next();
}

9
src/routes/dumpDatabase.ts
View File

@@ -143,6 +143,7 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
} else {
res.send({
dbVersion: await getDbVersion(),
lastUpdated: lastUpdate,
updateQueued,
links: latestDumpFiles.map((item:any) => {
@@ -158,6 +159,12 @@ export default async function dumpDatabase(req: Request, res: Response, showPage
await queueDump();
}
async function getDbVersion(): Promise<number> {
const row = await db.prepare('get', `SELECT "value" FROM "config" WHERE "key" = 'version'`);
if (row === undefined) return 0;
return row.value;
}
export async function redirectLink(req: Request, res: Response): Promise<void> {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
@@ -210,4 +217,4 @@ async function queueDump(): Promise<void> {
updateRunning = false;
lastUpdate = startTime;
}
}
}

79
src/routes/getSegmentInfo.ts Normal file
View File

@@ -0,0 +1,79 @@
import { Request, Response } from 'express';
import { db } from '../databases/databases';
import { DBSegment, SegmentUUID } from "../types/segments.model";
const isValidSegmentUUID = (str: string): Boolean => /^([a-f0-9]{64}|[a-f0-9]{8}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{12})/.test(str)
async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise<DBSegment> {
try {
return await db.prepare('get',
`SELECT "videoID", "startTime", "endTime", "votes", "locked",
"UUID", "userID", "timeSubmitted", "views", "category",
"service", "videoDuration", "hidden", "reputation", "shadowHidden" FROM "sponsorTimes"
WHERE "UUID" = ?`, [UUID]);
} catch (err) {
return null;
}
}
async function getSegmentsByUUID(UUIDs: SegmentUUID[]): Promise<DBSegment[]> {
const DBSegments: DBSegment[] = [];
for (let UUID of UUIDs) {
// if UUID is invalid, skip
if (!isValidSegmentUUID(UUID)) continue;
DBSegments.push(await getSegmentFromDBByUUID(UUID as SegmentUUID));
}
return DBSegments;
}
async function handleGetSegmentInfo(req: Request, res: Response) {
// If using params instead of JSON, only one UUID can be pulled
let UUIDs = req.query.UUIDs
? JSON.parse(req.query.UUIDs as string)
: req.query.UUID
? [req.query.UUID]
: null;
// deduplicate with set
UUIDs = [ ...new Set(UUIDs)];
// if more than 10 entries, slice
if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
if (!Array.isArray(UUIDs) || !UUIDs) {
res.status(400).send("UUIDs parameter does not match format requirements.");
return false;
}
const DBSegments = await getSegmentsByUUID(UUIDs);
// all uuids failed lookup
if (!DBSegments?.length) {
res.sendStatus(400);
return false;
}
// uuids valid but not found
if (DBSegments[0] === null || DBSegments[0] === undefined) {
res.sendStatus(400);
return false;
}
return DBSegments;
}
async function endpoint(req: Request, res: Response): Promise<void> {
try {
const DBSegments = await handleGetSegmentInfo(req, res);
// If false, res.send has already been called
if (DBSegments) {
//send result
res.send(DBSegments);
}
} catch (err) {
if (err instanceof SyntaxError) { // catch JSON.parse error
res.status(400).send("UUIDs parameter does not match format requirements.");
} else res.status(500).send();
}
}
export {
getSegmentFromDBByUUID,
getSegmentsByUUID,
handleGetSegmentInfo,
endpoint
};

111
src/routes/getSkipSegments.ts
View File

@@ -1,14 +1,15 @@
import { Request, Response } from 'express';
import { RedisClient } from 'redis';
import { config } from '../config';
import { db, privateDB } from '../databases/databases';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys';
import { SBRecord } from '../types/lib.model';
import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { Category, CategoryActionType, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
import { getCategoryActionType } from '../utils/categoryInfo';
import { getHash } from '../utils/getHash';
import { getIP } from '../utils/getIP';
import { Logger } from '../utils/logger';
import redis from '../utils/redis';
import { QueryCacher } from '../utils/queryCacher'
import { getReputation } from '../utils/reputation';
async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
@@ -40,7 +41,8 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
return chooseSegments(filteredSegments).map((chosenSegment) => ({
const maxSegments = getCategoryActionType(category) === CategoryActionType.Skippable ? 32 : 1
return (await chooseSegments(filteredSegments, maxSegments)).map((chosenSegment) => ({
category,
segment: [chosenSegment.startTime, chosenSegment.endTime],
UUID: chosenSegment.UUID,
@@ -48,7 +50,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
}));
}
async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise<Segment[]> {
async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories: Category[], service: Service): Promise<Segment[]> {
const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
const segments: Segment[] = [];
@@ -56,13 +58,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
categories = categories.filter((category) => !/[^a-z|_|-]/.test(category));
if (categories.length === 0) return null;
const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service]
)).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await getSegmentsFromDBByVideoID(videoID, service))
.filter((segment: DBSegment) => categories.includes(segment?.category))
.reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
acc[segment.category] = acc[segment.category] || [];
acc[segment.category].push(segment);
@@ -70,7 +68,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
}, {});
for (const [category, categorySegments] of Object.entries(segmentsByCategory)) {
segments.push(...(await prepareCategorySegments(req, videoID as VideoID, category as Category, categorySegments, cache)));
segments.push(...(await prepareCategorySegments(req, videoID, category as Category, categorySegments, cache)));
}
return segments;
@@ -92,7 +90,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) return null;
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDB(hashedVideoIDPrefix, service))
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
.filter((segment: DBSegment) => categories.includes(segment?.category))
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || {
@@ -127,37 +125,34 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}
}
async function getSegmentsFromDB(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
const fetchFromDB = () => db
.prepare(
'all',
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[hashedVideoIDPrefix + '%', service]
);
) as Promise<DBSegment[]>;
if (hashedVideoIDPrefix.length === 4) {
const key = skipSegmentsHashKey(hashedVideoIDPrefix, service);
const {err, reply} = await redis.getAsync(key);
if (!err && reply) {
try {
Logger.debug("Got data from redis: " + reply);
return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
}
}
const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data));
return data;
return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service))
}
return await fetchFromDB();
}
async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
const fetchFromDB = () => db
.prepare(
'all',
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "videoDuration", "reputation", "shadowHidden" FROM "sponsorTimes"
WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service]
) as Promise<DBSegment[]>;
return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service))
}
//gets a weighted random choice from the choices array based on their `votes` property.
//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
//choices are unique
@@ -174,10 +169,12 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
//assign a weight to each choice
let totalWeight = 0;
let choicesWithWeights: TWithWeight[] = choices.map(choice => {
const boost = Math.min(choice.reputation, 4);
//The 3 makes -2 the minimum votes before being ignored completely
//this can be changed if this system increases in popularity.
const weight = Math.exp((choice.votes + 3));
totalWeight += weight;
const weight = Math.exp(choice.votes * Math.max(1, choice.reputation + 1) + 3 + boost);
totalWeight += Math.max(weight, 0);
return {...choice, weight};
});
@@ -206,7 +203,7 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
//Only one similar time will be returned, randomly generated based on the sqrt of votes.
//This allows new less voted items to still sometimes appear to give them a chance at getting votes.
//Segments with less than -1 votes are already ignored before this function is called
function chooseSegments(segments: DBSegment[]): DBSegment[] {
async function chooseSegments(segments: DBSegment[], max: number): Promise<DBSegment[]> {
//Create groups of segments that are similar to eachother
//Segments must be sorted by their startTime so that we can build groups chronologically:
//1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
@@ -215,9 +212,9 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] {
const overlappingSegmentsGroups: OverlappingSegmentGroup[] = [];
let currentGroup: OverlappingSegmentGroup;
let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
segments.forEach(segment => {
for (const segment of segments) {
if (segment.startTime > cursor) {
currentGroup = {segments: [], votes: 0, locked: false};
currentGroup = {segments: [], votes: 0, reputation: 0, locked: false};
overlappingSegmentsGroups.push(currentGroup);
}
@@ -227,21 +224,28 @@ function chooseSegments(segments: DBSegment[]): DBSegment[] {
currentGroup.votes += segment.votes;
}
if (segment.userID) segment.reputation = Math.min(segment.reputation, await getReputation(segment.userID));
if (segment.reputation > 0) {
currentGroup.reputation += segment.reputation;
}
if (segment.locked) {
currentGroup.locked = true;
}
cursor = Math.max(cursor, segment.endTime);
});
};
overlappingSegmentsGroups.forEach((group) => {
if (group.locked) {
group.segments = group.segments.filter((segment) => segment.locked);
}
group.reputation = group.reputation / group.segments.length;
});
//if there are too many groups, find the best 8
return getWeightedRandomChoice(overlappingSegmentsGroups, 32).map(
//if there are too many groups, find the best ones
return getWeightedRandomChoice(overlappingSegmentsGroups, max).map(
//randomly choose 1 good segment per group and return them
group => getWeightedRandomChoice(group.segments, 1)[0],
);
@@ -266,24 +270,16 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
: req.query.category
? [req.query.category]
: ['sponsor'];
if (!Array.isArray(categories)) {
res.status(400).send("Categories parameter does not match format requirements.");
return false;
}
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {
service = Service.YouTube;
}
// Only 404s are cached at the moment
const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
if (redisResult.reply) {
const redisSegments = JSON.parse(redisResult.reply);
if (redisSegments?.length === 0) {
res.sendStatus(404);
Logger.debug("Using segments from cache for " + videoID);
return false;
}
}
const segments = await getSegmentsByVideoID(req, videoID, categories, service);
if (segments === null || segments === undefined) {
@@ -294,9 +290,6 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
if (segments.length === 0) {
res.sendStatus(404);
// Save in cache
if (categories.length == 7) redis.setAsync(skipSegmentsKey(videoID), JSON.stringify(segments));
return false;
}
@@ -313,7 +306,9 @@ async function endpoint(req: Request, res: Response): Promise<void> {
res.send(segments);
}
} catch (err) {
res.status(500).send();
if (err instanceof SyntaxError) {
res.status(400).send("Categories parameter does not match format requirements.");
} else res.status(500).send();
}
}

14
src/routes/getTopUsers.ts
View File

@@ -21,17 +21,18 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as "categorySumOutro",
SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as "categorySumInteraction",
SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as "categorySelfpromo",
SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic", `;
SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as "categoryMusicOfftopic",
SUM(CASE WHEN category = 'preview' THEN 1 ELSE 0 END) as "categorySumPreview", `;
}
const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
SUM("votes") as "userVotes", ` +
additionalFields +
`IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
`COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID"
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL
GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
ORDER BY ? DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, sortBy]);
for (let i = 0; i < rows.length; i++) {
@@ -48,6 +49,7 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
rows[i].categorySumInteraction,
rows[i].categorySelfpromo,
rows[i].categoryMusicOfftopic,
rows[i].categorySumPreview
];
}
}
@@ -71,10 +73,6 @@ export async function getTopUsers(req: Request, res: Response) {
return;
}
//TODO: remove. This is broken for now
res.status(200).send();
return;
//setup which sort type to use
let sortBy = '';
if (sortType == 0) {

43
src/routes/getUserInfo.ts
View File

@@ -4,6 +4,8 @@ import {isUserVIP} from '../utils/isUserVIP';
import {Request, Response} from 'express';
import {Logger} from '../utils/logger';
import { HashedUserID, UserID } from '../types/user.model';
import { getReputation } from '../utils/reputation';
import { SegmentUUID } from "../types/segments.model";
async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> {
try {
@@ -26,6 +28,15 @@ async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ min
}
}
async function dbGetIgnoredSegmentCount(userID: HashedUserID): Promise<number> {
try {
let row = await db.prepare("get", `SELECT COUNT(*) as "ignoredSegmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID]);
return row?.ignoredSegmentCount ?? 0
} catch (err) {
return null;
}
}
async function dbGetUsername(userID: HashedUserID) {
try {
let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -49,6 +60,15 @@ async function dbGetViewsForUser(userID: HashedUserID) {
}
}
async function dbGetIgnoredViewsForUser(userID: HashedUserID) {
try {
let row = await db.prepare('get', `SELECT SUM("views") as "ignoredViewCount" FROM "sponsorTimes" WHERE "userID" = ? AND ( "votes" <= -2 OR "shadowHidden" = 1 )`, [userID]);
return row?.ignoredViewCount ?? 0;
} catch (err) {
return false;
}
}
async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
try {
let row = await db.prepare('get', `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]);
@@ -59,18 +79,25 @@ async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
}
}
export async function getUserInfo(req: Request, res: Response) {
let userID = req.query.userID as UserID;
async function dbGetLastSegmentForUser(userID: HashedUserID): Promise<SegmentUUID> {
try {
let row = await db.prepare('get', `SELECT "UUID" FROM "sponsorTimes" WHERE "userID" = ? ORDER BY "timeSubmitted" DESC LIMIT 1`, [userID]);
return row?.UUID ?? null;
} catch (err) {
return null;
}
}
if (userID == undefined) {
export async function getUserInfo(req: Request, res: Response) {
const userID = req.query.userID as UserID;
const hashedUserID: HashedUserID = userID ? getHash(userID) : req.query.publicUserID as HashedUserID;
if (hashedUserID == undefined) {
//invalid request
res.status(400).send('Parameters are not valid');
return;
}
//hash the userID
const hashedUserID: HashedUserID = getHash(userID);
const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID);
if (segmentsSummary) {
res.send({
@@ -78,9 +105,13 @@ export async function getUserInfo(req: Request, res: Response) {
userName: await dbGetUsername(hashedUserID),
minutesSaved: segmentsSummary.minutesSaved,
segmentCount: segmentsSummary.segmentCount,
ignoredSegmentCount: await dbGetIgnoredSegmentCount(hashedUserID),
viewCount: await dbGetViewsForUser(hashedUserID),
ignoredViewCount: await dbGetIgnoredViewsForUser(hashedUserID),
warnings: await dbGetWarningsForUser(hashedUserID),
reputation: await getReputation(hashedUserID),
vip: await isUserVIP(hashedUserID),
lastSegmentID: await dbGetLastSegmentForUser(hashedUserID),
});
} else {
res.status(400).send();

55
src/routes/postClearCache.ts Normal file
View File

@@ -0,0 +1,55 @@
import { Logger } from '../utils/logger';
import { HashedUserID, UserID } from '../types/user.model';
import { getHash } from '../utils/getHash';
import { Request, Response } from 'express';
import { Service, VideoID } from '../types/segments.model';
import { QueryCacher } from '../utils/queryCacher';
import { isUserVIP } from '../utils/isUserVIP';
import { VideoIDHash } from "../types/segments.model";
export async function postClearCache(req: Request, res: Response) {
const videoID = req.query.videoID as VideoID;
let userID = req.query.userID as UserID;
const service = req.query.service as Service ?? Service.YouTube;
const invalidFields = [];
if (typeof videoID !== 'string') {
invalidFields.push('videoID');
}
if (typeof userID !== 'string') {
invalidFields.push('userID');
}
if (invalidFields.length !== 0) {
// invalid request
const fields = invalidFields.reduce((p, c, i) => p + (i !== 0 ? ', ' : '') + c, '');
res.status(400).send(`No valid ${fields} field(s) provided`);
return false;
}
// hash the userID as early as possible
const hashedUserID: HashedUserID = getHash(userID);
// hash videoID
const hashedVideoID: VideoIDHash = getHash(videoID, 1);
// Ensure user is a VIP
if (!(await isUserVIP(hashedUserID))){
Logger.warn("Permission violation: User " + hashedUserID + " attempted to clear cache for video " + videoID + ".");
res.status(403).json({"message": "Not a VIP"});
return false;
}
try {
QueryCacher.clearVideoCache({
videoID,
hashedVideoID,
service
});
res.status(200).json({
message: "Cache cleared on video " + videoID
});
} catch(err) {
res.status(500).send()
return false;
}
}

100
src/routes/postSkipSegments.ts
View File

@@ -1,30 +1,28 @@
import {config} from '../config';
import {Logger} from '../utils/logger';
import {db, privateDB} from '../databases/databases';
import {YouTubeAPI} from '../utils/youtubeApi';
import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi';
import {getSubmissionUUID} from '../utils/getSubmissionUUID';
import fetch from 'node-fetch';
import isoDurations from 'iso8601-duration';
import isoDurations, { end } from 'iso8601-duration';
import {getHash} from '../utils/getHash';
import {getIP} from '../utils/getIP';
import {getFormattedTime} from '../utils/getFormattedTime';
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
import {dispatchEvent} from '../utils/webhookUtils';
import {Request, Response} from 'express';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { skipSegmentsHashKey, skipSegmentsKey } from '../utils/redisKeys';
import redis from '../utils/redis';
import { Category, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { deleteLockCategories } from './deleteLockCategories';
import { getCategoryActionType } from '../utils/categoryInfo';
import { QueryCacher } from '../utils/queryCacher';
import { getReputation } from '../utils/reputation';
import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
interface APIVideoInfo {
err: string | boolean,
data: any
}
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
const userName = row !== undefined ? row.userName : null;
const video = youtubeData.items[0];
let scopeName = "submissions.other";
if (submissionCount <= 1) {
@@ -34,8 +32,8 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
dispatchEvent(scopeName, {
"video": {
"id": videoID,
"title": video.snippet.title,
"thumbnail": video.snippet.thumbnails.maxres ? video.snippet.thumbnails.maxres : null,
"title": youtubeData?.title,
"thumbnail": getMaxResThumbnail(youtubeData) || null,
"url": "https://www.youtube.com/watch?v=" + videoID,
},
"submission": {
@@ -73,7 +71,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
method: 'POST',
body: JSON.stringify({
"embeds": [{
"title": data.items[0].snippet.title,
"title": data?.title,
"url": "https://www.youtube.com/watch?v=" + videoID + "&t=" + (parseInt(startTime.toFixed(0)) - 2),
"description": "Submission ID: " + UUID +
"\n\nTimestamp: " +
@@ -84,7 +82,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
"name": userID,
},
"thumbnail": {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
"url": getMaxResThumbnail(data) || "",
},
}],
}),
@@ -174,10 +172,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
const {err, data} = apiVideoInfo;
if (err) return false;
// Check to see if video exists
if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID;
const duration = getYouTubeVideoDuration(apiVideoInfo);
const duration = apiVideoInfo?.data?.lengthSeconds;
const segments = submission.segments;
let nbString = "";
for (let i = 0; i < segments.length; i++) {
@@ -217,8 +212,7 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
return a[0] - b[0] || a[1] - b[1];
}));
let videoDuration = data.items[0].contentDetails.duration;
videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
const videoDuration = data?.lengthSeconds;
if (videoDuration != 0) {
let allSegmentDuration = 0;
//sum all segment times together
@@ -270,16 +264,9 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
}
}
function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration;
return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
}
async function getYouTubeVideoInfo(videoID: VideoID): Promise<APIVideoInfo> {
if (config.youtubeAPIKey !== null) {
return new Promise((resolve) => {
YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
});
async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
if (config.newLeafURLs !== null) {
return YouTubeAPI.listVideos(videoID, ignoreCache);
} else {
return null;
}
@@ -291,14 +278,10 @@ function proxySubmission(req: Request) {
body: req.body,
})
.then(async res => {
if (config.mode === 'development') {
Logger.debug('Proxy Submission: ' + res.status + ' (' + (await res.text()) + ')');
}
Logger.debug('Proxy Submission: ' + res.status + ' (' + (await res.text()) + ')');
})
.catch(err => {
if (config.mode === 'development') {
Logger.error("Proxy Submission: Failed to make call");
}
Logger.error("Proxy Submission: Failed to make call");
});
}
@@ -367,22 +350,24 @@ export async function postSkipSegments(req: Request, res: Response) {
const decreaseVotes = 0;
const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0
AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as
{videoDuration: VideoDuration, UUID: SegmentUUID}[];
// If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
const videoDurationChanged = (videoDuration: number) => videoDuration != 0 && previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
let apiVideoInfo: APIVideoInfo = null;
if (service == Service.YouTube) {
apiVideoInfo = await getYouTubeVideoInfo(videoID);
// Don't use cache if we don't know the video duraton, or the client claims that it has changed
apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDuration || videoDurationChanged(videoDuration));
}
const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
// If api duration is far off, take that one instead (it is only precise to seconds, not millis)
videoDuration = apiVideoDuration || 0 as VideoDuration;
}
const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0
AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as
{videoDuration: VideoDuration, UUID: SegmentUUID}[];
// If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
if (videoDurationChanged) {
if (videoDurationChanged(videoDuration)) {
// Hide all previous submissions
for (const submission of previousSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
@@ -411,10 +396,10 @@ export async function postSkipSegments(req: Request, res: Response) {
// TODO: Do something about the fradulent submission
Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
res.status(403).send(
"Request rejected by auto moderator: New submissions are not allowed for the following category: '"
"New submissions are not allowed for the following category: '"
+ segments[i].category + "'. A moderator has decided that no new segments are needed and that all current segments of this category are timed perfectly.\n\n "
+ (segments[i].category === "sponsor" ? "Maybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " +
"Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n " : "")
"Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n\n" : "")
+ "If you believe this is incorrect, please contact someone on discord.gg/SponsorBlock or matrix.to/#/+sponsorblock:ajay.app",
);
return;
@@ -425,7 +410,9 @@ export async function postSkipSegments(req: Request, res: Response) {
let endTime = parseFloat(segments[i].segment[1]);
if (isNaN(startTime) || isNaN(endTime)
|| startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) {
|| startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime
|| (getCategoryActionType(segments[i].category) === CategoryActionType.Skippable && startTime === endTime)
|| (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime !== endTime)) {
//invalid request
res.status(400).send("One of your segments times are invalid (too short, startTime before endTime, etc.)");
return;
@@ -498,7 +485,7 @@ export async function postSkipSegments(req: Request, res: Response) {
}
//check to see if this user is shadowbanned
const shadowBanRow = await privateDB.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
const shadowBanRow = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
let shadowBanned = shadowBanRow.userCount;
@@ -508,6 +495,7 @@ export async function postSkipSegments(req: Request, res: Response) {
}
let startingVotes = 0 + decreaseVotes;
const reputation = await getReputation(userID);
for (const segmentInfo of segments) {
//this can just be a hash of the data
@@ -519,9 +507,9 @@ export async function postSkipSegments(req: Request, res: Response) {
const startingLocked = isVIP ? 1 : 0;
try {
await db.prepare('run', `INSERT INTO "sponsorTimes"
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, hashedVideoID,
("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "reputation", "shadowHidden", "hashedVideoID")
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, reputation, shadowBanned, hashedVideoID,
],
);
@@ -529,8 +517,12 @@ export async function postSkipSegments(req: Request, res: Response) {
await privateDB.prepare('run', `INSERT INTO "sponsorTimes" VALUES(?, ?, ?)`, [videoID, hashedIP, timeSubmitted]);
// Clear redis cache for this video
redis.delAsync(skipSegmentsKey(videoID));
redis.delAsync(skipSegmentsHashKey(hashedVideoID, service));
QueryCacher.clearVideoCache({
videoID,
hashedVideoID,
service,
userID
});
} catch (err) {
//a DB change probably occurred
res.sendStatus(500);

2
src/routes/postWarning.ts
View File

@@ -32,7 +32,7 @@ export async function postWarning(req: Request, res: Response) {
return;
}
} else {
await db.prepare('run', 'UPDATE "warnings" SET "enabled" = 0 WHERE "userID" = ? AND "issuerUserID" = ?', [userID, issuerUserID]);
await db.prepare('run', 'UPDATE "warnings" SET "enabled" = 0 WHERE "userID" = ?', [userID]);
resultStatus = "removed from";
}

15
src/routes/setUsername.ts
View File

@@ -40,6 +40,19 @@ export async function setUsername(req: Request, res: Response) {
userID = getHash(userID);
}
try {
const row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ? AND "locked" = '1'`, [userID]);
if (adminUserIDInput === undefined && row.count > 0) {
res.sendStatus(200);
return;
}
}
catch (error) {
Logger.error(error);
res.sendStatus(500);
return;
}
try {
//check if username is already set
let row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -49,7 +62,7 @@ export async function setUsername(req: Request, res: Response) {
await db.prepare('run', `UPDATE "userNames" SET "userName" = ? WHERE "userID" = ?`, [userName, userID]);
} else {
//add to the db
await db.prepare('run', `INSERT INTO "userNames" VALUES(?, ?)`, [userID, userName]);
await db.prepare('run', `INSERT INTO "userNames"("userID", "userName") VALUES(?, ?)`, [userID, userName]);
}
res.sendStatus(200);

39
src/routes/shadowBanUser.ts
View File

@@ -1,6 +1,10 @@
import {db, privateDB} from '../databases/databases';
import {db} from '../databases/databases';
import {getHash} from '../utils/getHash';
import {Request, Response} from 'express';
import { config } from '../config';
import { Category, Service, VideoID, VideoIDHash } from '../types/segments.model';
import { UserID } from '../types/user.model';
import { QueryCacher } from '../utils/queryCacher';
export async function shadowBanUser(req: Request, res: Response) {
const userID = req.query.userID as string;
@@ -8,12 +12,15 @@ export async function shadowBanUser(req: Request, res: Response) {
let adminUserIDInput = req.query.adminUserID as string;
const enabled = req.query.enabled === undefined
? false
? true
: req.query.enabled === 'true';
//if enabled is false and the old submissions should be made visible again
const unHideOldSubmissions = req.query.unHideOldSubmissions !== "false";
const categories: string[] = req.query.categories ? JSON.parse(req.query.categories as string) : config.categoryList;
categories.filter((category) => typeof category === "string" && !(/[^a-z|_|-]/.test(category)));
if (adminUserIDInput == undefined || (userID == undefined && hashedIP == undefined)) {
//invalid request
res.sendStatus(400);
@@ -32,23 +39,30 @@ export async function shadowBanUser(req: Request, res: Response) {
if (userID) {
//check to see if this user is already shadowbanned
const row = await privateDB.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
const row = await db.prepare('get', `SELECT count(*) as "userCount" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
if (enabled && row.userCount == 0) {
//add them to the shadow ban list
//add it to the table
await privateDB.prepare('run', `INSERT INTO "shadowBannedUsers" VALUES(?)`, [userID]);
await db.prepare('run', `INSERT INTO "shadowBannedUsers" VALUES(?)`, [userID]);
//find all previous submissions and hide them
if (unHideOldSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ?
await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})
AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
"sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
// clear cache for all old videos
(await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]))
.forEach((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => {
QueryCacher.clearVideoCache(videoInfo);
}
);
}
} else if (!enabled && row.userCount > 0) {
//remove them from the shadow ban list
await privateDB.prepare('run', `DELETE FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
await db.prepare('run', `DELETE FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID]);
//find all previous submissions and unhide them
if (unHideOldSubmissions) {
@@ -60,8 +74,15 @@ export async function shadowBanUser(req: Request, res: Response) {
await Promise.all(allSegments.filter((item: {uuid: string}) => {
return segmentsToIgnore.indexOf(item) === -1;
}).map((UUID: string) => {
return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ?`, [UUID]);
}).map(async (UUID: string) => {
// collect list for unshadowbanning
(await db.prepare('all', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ? AND "shadowHidden" = 1 AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID]))
.forEach((videoInfo: {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID}) => {
QueryCacher.clearVideoCache(videoInfo);
}
);
return db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE "UUID" = ? AND "category" in (${categories.map((c) => `'${c}'`).join(",")})`, [UUID]);
}));
}
}
@@ -85,7 +106,7 @@ export async function shadowBanUser(req: Request, res: Response) {
}
} /*else if (!enabled && row.userCount > 0) {
// //remove them from the shadow ban list
// await privateDB.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]);
// await db.prepare('run', "DELETE FROM shadowBannedUsers WHERE userID = ?", [userID]);
// //find all previous submissions and unhide them
// if (unHideOldSubmissions) {

259
src/routes/voteOnSponsorTime.ts
View File

@@ -2,27 +2,34 @@ import {Request, Response} from 'express';
import {Logger} from '../utils/logger';
import {isUserVIP} from '../utils/isUserVIP';
import fetch from 'node-fetch';
import {YouTubeAPI} from '../utils/youtubeApi';
import {getMaxResThumbnail, YouTubeAPI} from '../utils/youtubeApi';
import {db, privateDB} from '../databases/databases';
import {dispatchEvent, getVoteAuthor, getVoteAuthorRaw} from '../utils/webhookUtils';
import {isUserTrustworthy} from '../utils/isUserTrustworthy';
import {getFormattedTime} from '../utils/getFormattedTime';
import {getIP} from '../utils/getIP';
import {getHash} from '../utils/getHash';
import {config} from '../config';
import { UserID } from '../types/user.model';
import redis from '../utils/redis';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import { Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
import { Category, CategoryActionType, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash } from '../types/segments.model';
import { getCategoryActionType } from '../utils/categoryInfo';
import { QueryCacher } from '../utils/queryCacher';
const voteTypes = {
normal: 0,
incorrect: 1,
};
enum VoteWebhookType {
Normal,
Rejected
}
interface FinalResponse {
blockVote: boolean,
finalStatus: number
finalMessage: string
finalMessage: string,
webhookType: VoteWebhookType,
webhookMessage: string
}
interface VoteData {
@@ -53,96 +60,102 @@ async function sendWebhooks(voteData: VoteData) {
if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) {
let webhookURL: string = null;
if (voteData.voteTypeEnum === voteTypes.normal) {
webhookURL = config.discordReportChannelWebhookURL;
switch (voteData.finalResponse.webhookType) {
case VoteWebhookType.Normal:
webhookURL = config.discordReportChannelWebhookURL;
break;
case VoteWebhookType.Rejected:
webhookURL = config.discordFailedReportChannelWebhookURL;
break;
}
} else if (voteData.voteTypeEnum === voteTypes.incorrect) {
webhookURL = config.discordCompletelyIncorrectReportWebhookURL;
}
if (config.youtubeAPIKey !== null) {
YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => {
if (err || data.items.length === 0) {
err && Logger.error(err.toString());
return;
}
const isUpvote = voteData.incrementAmount > 0;
// Send custom webhooks
dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
if (config.newLeafURLs !== null) {
const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID);
if (err) return;
const isUpvote = voteData.incrementAmount > 0;
// Send custom webhooks
dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
"user": {
"status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"video": {
"id": submissionInfoRow.videoID,
"title": data?.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
"thumbnail": getMaxResThumbnail(data) || null,
},
"submission": {
"UUID": voteData.UUID,
"views": voteData.row.views,
"category": voteData.category,
"startTime": submissionInfoRow.startTime,
"endTime": submissionInfoRow.endTime,
"user": {
"status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"video": {
"id": submissionInfoRow.videoID,
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
"thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
},
"submission": {
"UUID": voteData.UUID,
"views": voteData.row.views,
"category": voteData.category,
"startTime": submissionInfoRow.startTime,
"endTime": submissionInfoRow.endTime,
"user": {
"UUID": submissionInfoRow.userID,
"username": submissionInfoRow.userName,
"submissions": {
"total": submissionInfoRow.count,
"ignored": submissionInfoRow.disregarded,
},
"UUID": submissionInfoRow.userID,
"username": submissionInfoRow.userName,
"submissions": {
"total": submissionInfoRow.count,
"ignored": submissionInfoRow.disregarded,
},
},
"votes": {
"before": voteData.row.votes,
"after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount),
},
});
// Send discord message
if (webhookURL !== null && !isUpvote) {
fetch(webhookURL, {
method: 'POST',
body: JSON.stringify({
"embeds": [{
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID
+ "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
"description": "**" + voteData.row.votes + " Votes Prior | " +
(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views
+ " Views**\n\n**Submission ID:** " + voteData.UUID
+ "\n**Category:** " + submissionInfoRow.category
+ "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID
+ "\n\n**Total User Submissions:** " + submissionInfoRow.count
+ "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded
+ "\n\n**Timestamp:** " +
getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
"color": 10813440,
"author": {
"name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"thumbnail": {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
},
}],
}),
headers: {
'Content-Type': 'application/json'
}
})
.then(async res => {
if (res.status >= 400) {
Logger.error("Error sending reported submission Discord hook");
Logger.error(JSON.stringify((await res.text())));
Logger.error("\n");
}
})
.catch(err => {
Logger.error("Failed to send reported submission Discord hook.");
Logger.error(JSON.stringify(err));
Logger.error("\n");
});
}
},
"votes": {
"before": voteData.row.votes,
"after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount),
},
});
// Send discord message
if (webhookURL !== null && !isUpvote) {
fetch(webhookURL, {
method: 'POST',
body: JSON.stringify({
"embeds": [{
"title": data?.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID
+ "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
"description": "**" + voteData.row.votes + " Votes Prior | " +
(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views
+ " Views**\n\n**Submission ID:** " + voteData.UUID
+ "\n**Category:** " + submissionInfoRow.category
+ "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID
+ "\n\n**Total User Submissions:** " + submissionInfoRow.count
+ "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded
+ "\n\n**Timestamp:** " +
getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
"color": 10813440,
"author": {
"name": voteData.finalResponse?.webhookMessage ??
voteData.finalResponse?.finalMessage ??
getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
},
}],
}),
headers: {
'Content-Type': 'application/json'
}
})
.then(async res => {
if (res.status >= 400) {
Logger.error("Error sending reported submission Discord hook");
Logger.error(JSON.stringify((await res.text())));
Logger.error("\n");
}
})
.catch(err => {
Logger.error("Failed to send reported submission Discord hook.");
Logger.error(JSON.stringify(err));
Logger.error("\n");
});
}
}
}
}
@@ -158,8 +171,8 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
return;
}
const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service" FROM "sponsorTimes" WHERE "UUID" = ?`,
[UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service};
const videoInfo = (await db.prepare('get', `SELECT "category", "videoID", "hashedVideoID", "service", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`,
[UUID])) as {category: Category, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID};
if (!videoInfo) {
// Submission doesn't exist
res.status(400).send("Submission doesn't exist.");
@@ -170,6 +183,10 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
res.status(400).send("Category doesn't exist.");
return;
}
if (getCategoryActionType(category) !== CategoryActionType.Skippable) {
res.status(400).send("Cannot vote for this category");
return;
}
const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]);
@@ -226,7 +243,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
}
}
clearRedisCache(videoInfo);
QueryCacher.clearVideoCache(videoInfo);
res.sendStatus(finalResponse.finalStatus);
}
@@ -253,8 +270,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
// To force a non 200, change this early
let finalResponse: FinalResponse = {
blockVote: false,
finalStatus: 200,
finalMessage: null
finalMessage: null,
webhookType: VoteWebhookType.Normal,
webhookMessage: null
}
//x-forwarded-for if this server is behind a proxy
@@ -277,8 +297,9 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
' where "UUID" = ?', [UUID]));
if (await isSegmentLocked() || await isVideoLocked()) {
finalResponse.finalStatus = 403;
finalResponse.finalMessage = "Vote rejected: A moderator has decided that this segment is correct"
finalResponse.blockVote = true;
finalResponse.webhookType = VoteWebhookType.Rejected
finalResponse.webhookMessage = "Vote rejected: A moderator has decided that this segment is correct"
}
}
@@ -312,7 +333,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
return res.status(403).send('Vote rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
}
const voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect;
const voteTypeEnum = (type == 0 || type == 1 || type == 20) ? voteTypes.normal : voteTypes.incorrect;
try {
//check if vote has already happened
@@ -362,8 +383,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
}
//check if the increment amount should be multiplied (downvotes have more power if there have been many views)
const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
{videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number};
const videoInfo = await db.prepare('get', `SELECT "videoID", "hashedVideoID", "service", "votes", "views", "userID" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]) as
{videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, votes: number, views: number, userID: UserID};
if (voteTypeEnum === voteTypes.normal) {
if ((isVIP || isOwnSubmission) && incrementAmount < 0) {
@@ -381,9 +402,11 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
// Only change the database if they have made a submission before and haven't voted recently
const ableToVote = isVIP
|| ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
|| (!(isOwnSubmission && incrementAmount > 0)
&& (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
&& !finalResponse.blockVote
&& finalResponse.finalStatus === 200;
if (ableToVote) {
@@ -403,7 +426,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
//update the vote count on this sponsorTime
//oldIncrementAmount will be zero is row is null
await db.prepare('run', 'UPDATE "sponsorTimes" SET ' + columnName + ' = ' + columnName + ' + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]);
await db.prepare('run', 'UPDATE "sponsorTimes" SET "' + columnName + '" = "' + columnName + '" + ? WHERE "UUID" = ?', [incrementAmount - oldIncrementAmount, UUID]);
if (isVIP && incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
// Lock this submission
await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 1 WHERE "UUID" = ?', [UUID]);
@@ -412,32 +435,7 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
await db.prepare('run', 'UPDATE "sponsorTimes" SET locked = 0 WHERE "UUID" = ?', [UUID]);
}
clearRedisCache(videoInfo);
//for each positive vote, see if a hidden submission can be shown again
if (incrementAmount > 0 && voteTypeEnum === voteTypes.normal) {
//find the UUID that submitted the submission that was voted on
const submissionUserIDInfo = await db.prepare('get', 'SELECT "userID" FROM "sponsorTimes" WHERE "UUID" = ?', [UUID]);
if (!submissionUserIDInfo) {
// They are voting on a non-existent submission
res.status(400).send("Voting on a non-existent submission");
return;
}
const submissionUserID = submissionUserIDInfo.userID;
//check if any submissions are hidden
const hiddenSubmissionsRow = await db.prepare('get', 'SELECT count(*) as "hiddenSubmissions" FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" > 0', [submissionUserID]);
if (hiddenSubmissionsRow.hiddenSubmissions > 0) {
//see if some of this users submissions should be visible again
if (await isUserTrustworthy(submissionUserID)) {
//they are trustworthy again, show 2 of their submissions again, if there are two to show
await db.prepare('run', 'UPDATE "sponsorTimes" SET "shadowHidden" = 0 WHERE ROWID IN (SELECT ROWID FROM "sponsorTimes" WHERE "userID" = ? AND "shadowHidden" = 1 LIMIT 2)', [submissionUserID]);
}
}
}
QueryCacher.clearVideoCache(videoInfo);
}
res.status(finalResponse.finalStatus).send(finalResponse.finalMessage ?? undefined);
@@ -461,11 +459,4 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
res.status(500).json({error: 'Internal error creating segment vote'});
}
}
function clearRedisCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; }) {
if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
}
}
}

3
src/types/config.model.ts
View File

@@ -6,8 +6,9 @@ export interface SBSConfig {
mockPort?: number;
globalSalt: string;
adminUserID: string;
youtubeAPIKey?: string;
newLeafURLs?: string[];
discordReportChannelWebhookURL?: string;
discordFailedReportChannelWebhookURL?: string;
discordFirstTimeSubmissionsWebhookURL?: string;
discordCompletelyIncorrectReportWebhookURL?: string;
neuralBlockURL?: string;

12
src/types/segments.model.ts
View File

@@ -1,10 +1,11 @@
import { HashedValue } from "./hash.model";
import { SBRecord } from "./lib.model";
import { UserID } from "./user.model";
export type SegmentUUID = string & { __segmentUUIDBrand: unknown };
export type VideoID = string & { __videoIDBrand: unknown };
export type VideoDuration = number & { __videoDurationBrand: unknown };
export type Category = string & { __categoryBrand: unknown };
export type Category = ("sponsor" | "selfpromo" | "interaction" | "intro" | "outro" | "preview" | "music_offtopic" | "highlight") & { __categoryBrand: unknown };
export type VideoIDHash = VideoID & HashedValue;
export type IPAddress = string & { __ipAddressBrand: unknown };
export type HashedIP = IPAddress & HashedValue;
@@ -42,11 +43,13 @@ export interface DBSegment {
startTime: number;
endTime: number;
UUID: SegmentUUID;
userID: UserID;
votes: number;
locked: boolean;
shadowHidden: Visibility;
videoID: VideoID;
videoDuration: VideoDuration;
reputation: number;
hashedVideoID: VideoIDHash;
}
@@ -54,10 +57,12 @@ export interface OverlappingSegmentGroup {
segments: DBSegment[],
votes: number;
locked: boolean; // Contains a locked segment
reputation: number;
}
export interface VotableObject {
votes: number;
reputation: number;
}
export interface VotableObjectWithWeight extends VotableObject {
@@ -72,4 +77,9 @@ export interface VideoData {
export interface SegmentCache {
shadowHiddenSegmentIPs: SBRecord<VideoID, {hashedIP: HashedIP}[]>,
userHashedIP?: HashedIP
}
export enum CategoryActionType {
Skippable,
POI
}

111
src/types/youtubeApi.model.ts Normal file
View File

@@ -0,0 +1,111 @@
export interface APIVideoData {
"title": string,
"videoId": string,
"videoThumbnails": [
{
"quality": string,
"url": string,
second__originalUrl: string,
"width": number,
"height": number
}
],
"description": string,
"descriptionHtml": string,
"published": number,
"publishedText": string,
"keywords": string[],
"viewCount": number,
"likeCount": number,
"dislikeCount": number,
"paid": boolean,
"premium": boolean,
"isFamilyFriendly": boolean,
"allowedRegions": string[],
"genre": string,
"genreUrl": string,
"author": string,
"authorId": string,
"authorUrl": string,
"authorThumbnails": [
{
"url": string,
"width": number,
"height": number
}
],
"subCountText": string,
"lengthSeconds": number,
"allowRatings": boolean,
"rating": number,
"isListed": boolean,
"liveNow": boolean,
"isUpcoming": boolean,
"premiereTimestamp"?: number,
"hlsUrl"?: string,
"adaptiveFormats": [
{
"index": string,
"bitrate": string,
"init": string,
"url": string,
"itag": string,
"type": string,
"clen": string,
"lmt": string,
"projectionType": number,
"container": string,
"encoding": string,
"qualityLabel"?: string,
"resolution"?: string
}
],
"formatStreams": [
{
"url": string,
"itag": string,
"type": string,
"quality": string,
"container": string,
"encoding": string,
"qualityLabel": string,
"resolution": string,
"size": string
}
],
"captions": [
{
"label": string,
"languageCode": string,
"url": string
}
],
"recommendedVideos": [
{
"videoId": string,
"title": string,
"videoThumbnails": [
{
"quality": string,
"url": string,
"width": number,
"height": number
}
],
"author": string,
"lengthSeconds": number,
"viewCountText": string
}
]
}
export interface APIVideoInfo {
err: string | boolean,
data?: APIVideoData
}

10
src/utils/categoryInfo.ts Normal file
View File

@@ -0,0 +1,10 @@
import { Category, CategoryActionType } from "../types/segments.model";
export function getCategoryActionType(category: Category): CategoryActionType {
switch (category) {
case "highlight":
return CategoryActionType.POI;
default:
return CategoryActionType.Skippable;
}
}

36
src/utils/queryCacher.ts Normal file
View File

@@ -0,0 +1,36 @@
import redis from "../utils/redis";
import { Logger } from "../utils/logger";
import { skipSegmentsHashKey, skipSegmentsKey, reputationKey } from "./redisKeys";
import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { UserID } from "../types/user.model";
async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
const {err, reply} = await redis.getAsync(key);
if (!err && reply) {
try {
Logger.debug("Got data from redis: " + reply);
return JSON.parse(reply);
} catch (e) {
// If all else, continue on to fetching from the database
}
}
const data = await fetchFromDB();
redis.setAsync(key, JSON.stringify(data));
return data;
}
function clearVideoCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }) {
if (videoInfo) {
redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
redis.delAsync(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service));
if (videoInfo.userID) redis.delAsync(reputationKey(videoInfo.userID));
}
}
export const QueryCacher = {
get,
clearVideoCache
}

13
src/middleware/redisKeys.ts → src/utils/redisKeys.ts
View File

@@ -1,8 +1,9 @@
import { Service, VideoID, VideoIDHash } from "../types/segments.model";
import { Logger } from "../utils/logger";
import { UserID } from "../types/user.model";
import { Logger } from "./logger";
export function skipSegmentsKey(videoID: VideoID): string {
return "segments-" + videoID;
export function skipSegmentsKey(videoID: VideoID, service: Service): string {
return "segments." + service + ".videoID." + videoID;
}
export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string {
@@ -10,4 +11,8 @@ export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: S
if (hashedVideoIDPrefix.length !== 4) Logger.warn("Redis skip segment hash-prefix key is not length 4! " + hashedVideoIDPrefix);
return "segments." + service + "." + hashedVideoIDPrefix;
}
}
export function reputationKey(userID: UserID): string {
return "reputation.user." + userID;
}

59
src/utils/reputation.ts Normal file
View File

@@ -0,0 +1,59 @@
import { db } from "../databases/databases";
import { UserID } from "../types/user.model";
import { QueryCacher } from "./queryCacher";
import { reputationKey } from "./redisKeys";
interface ReputationDBResult {
totalSubmissions: number,
downvotedSubmissions: number,
nonSelfDownvotedSubmissions: number,
upvotedSum: number,
lockedSum: number,
oldUpvotedSubmissions: number
}
export async function getReputation(userID: UserID): Promise<number> {
const pastDate = Date.now() - 1000 * 60 * 60 * 24 * 45; // 45 days ago
// 1596240000000 is August 1st 2020, a little after auto upvote was disabled
const fetchFromDB = () => db.prepare("get",
`SELECT COUNT(*) AS "totalSubmissions",
SUM(CASE WHEN "votes" < 0 THEN 1 ELSE 0 END) AS "downvotedSubmissions",
SUM(CASE WHEN "votes" < 0 AND "videoID" NOT IN
(SELECT b."videoID" FROM "sponsorTimes" as b
WHERE b."userID" = ?
AND b."votes" > 0 AND b."category" = "a"."category" AND b."videoID" = "a"."videoID" LIMIT 1)
THEN 1 ELSE 0 END) AS "nonSelfDownvotedSubmissions",
SUM(CASE WHEN "votes" > 0 AND "timeSubmitted" > 1596240000000 THEN "votes" ELSE 0 END) AS "upvotedSum",
SUM(locked) AS "lockedSum",
SUM(CASE WHEN "timeSubmitted" < ? AND "timeSubmitted" > 1596240000000 AND "votes" > 0 THEN 1 ELSE 0 END) AS "oldUpvotedSubmissions"
FROM "sponsorTimes" as "a" WHERE "userID" = ?`, [userID, pastDate, userID]) as Promise<ReputationDBResult>;
const result = await QueryCacher.get(fetchFromDB, reputationKey(userID));
// Grace period
if (result.totalSubmissions < 5) {
return 0;
}
const downvoteRatio = result.downvotedSubmissions / result.totalSubmissions;
if (downvoteRatio > 0.3) {
return convertRange(Math.min(downvoteRatio, 0.7), 0.3, 0.7, -0.5, -2.5);
}
const nonSelfDownvoteRatio = result.nonSelfDownvotedSubmissions / result.totalSubmissions;
if (nonSelfDownvoteRatio > 0.05) {
return convertRange(Math.min(nonSelfDownvoteRatio, 0.4), 0.05, 0.4, -0.5, -2.5);
}
if (result.oldUpvotedSubmissions < 3 || result.upvotedSum < 5) {
return 0;
}
return convertRange(Math.min(result.upvotedSum, 150), 5, 150, 0, 7) + convertRange(Math.min(result.lockedSum ?? 0, 50), 0, 50, 0, 20);
}
function convertRange(value: number, currentMin: number, currentMax: number, targetMin: number, targetMax: number): number {
const currentRange = currentMax - currentMin;
const targetRange = targetMax - targetMin;
return ((value - currentMin) / currentRange) * targetRange + targetMin;
}

8
src/utils/webhookUtils.ts
View File

@@ -1,6 +1,7 @@
import {config} from '../config';
import {Logger} from '../utils/logger';
import fetch from 'node-fetch';
import AbortController from "abort-controller";
function getVoteAuthorRaw(submissionCount: number, isVIP: boolean, isOwnSubmission: boolean): string {
if (isOwnSubmission) {
@@ -30,7 +31,8 @@ function dispatchEvent(scope: string, data: any): void {
let webhooks = config.webhooks;
if (webhooks === undefined || webhooks.length === 0) return;
Logger.debug("Dispatching webhooks");
webhooks.forEach(webhook => {
for (const webhook of webhooks) {
let webhookURL = webhook.url;
let authKey = webhook.key;
let scopes = webhook.scopes || [];
@@ -43,13 +45,13 @@ function dispatchEvent(scope: string, data: any): void {
"Authorization": authKey,
"Event-Type": scope, // Maybe change this in the future?
'Content-Type': 'application/json'
},
}
})
.catch(err => {
Logger.warn('Couldn\'t send webhook to ' + webhook.url);
Logger.warn(err);
});
});
}
}
export {

78
src/utils/youtubeApi.ts
View File

@@ -1,52 +1,56 @@
import fetch from 'node-fetch';
import {config} from '../config';
import {Logger} from './logger';
import redis from './redis';
// @ts-ignore
import _youTubeAPI from 'youtube-api';
_youTubeAPI.authenticate({
type: "key",
key: config.youtubeAPIKey,
});
import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
export class YouTubeAPI {
static listVideos(videoID: string, callback: (err: string | boolean, data: any) => void) {
const part = 'contentDetails,snippet';
static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
if (!videoID || videoID.length !== 11 || videoID.includes(".")) {
callback("Invalid video ID", undefined);
return;
return { err: "Invalid video ID" };
}
const redisKey = "youtube.video." + videoID;
redis.get(redisKey, (getErr, result) => {
if (getErr || !result) {
const redisKey = "yt.newleaf.video." + videoID;
if (!ignoreCache) {
const {err, reply} = await redis.getAsync(redisKey);
if (!err && reply) {
Logger.debug("redis: no cache for video information: " + videoID);
_youTubeAPI.videos.list({
part,
id: videoID,
}, (ytErr: boolean | string, { data }: any) => {
if (!ytErr) {
// Only set cache if data returned
if (data.items.length > 0) {
redis.set(redisKey, JSON.stringify(data), (setErr) => {
if (setErr) {
Logger.warn(setErr.message);
} else {
Logger.debug("redis: video information cache set for: " + videoID);
}
callback(false, data); // don't fail
});
} else {
callback(false, data); // don't fail
}
return { err: err?.message, data: JSON.parse(reply) }
}
}
if (!config.newLeafURLs || config.newLeafURLs.length <= 0) return {err: "NewLeaf URL not found", data: null};
try {
const result = await fetch(config.newLeafURLs[Math.floor(Math.random() * config.newLeafURLs.length)] + "/api/v1/videos/" + videoID, { method: "GET" });
if (result.ok) {
const data = await result.json();
if (data.error) {
Logger.warn("NewLeaf API Error for " + videoID + ": " + data.error)
return { err: data.error, data: null };
}
redis.setAsync(redisKey, JSON.stringify(data)).then((result) => {
if (result?.err) {
Logger.warn(result?.err.message);
} else {
callback(ytErr, data);
Logger.debug("redis: video information cache set for: " + videoID);
}
});
return { err: false, data };
} else {
Logger.debug("redis: fetched video information from cache: " + videoID);
callback(getErr?.message, JSON.parse(result));
return { err: result.statusText, data: null };
}
});
};
} catch (err) {
return {err, data: null}
}
}
}
export function getMaxResThumbnail(apiInfo: APIVideoData): string | void {
return apiInfo?.videoThumbnails?.find((elem) => elem.quality === "maxres")?.second__originalUrl;
}