14 Commits

Author SHA1 Message Date
Ajay
a860b89ef0 Add config for min 2022-05-27 23:23:45 -04:00
Ajay
4650316067 FIx max not being used 2022-05-27 22:46:35 -04:00
Ajay
08d458bdd6 Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into k8s-test 2022-05-27 19:07:53 -04:00
Ajay
6abfba1b12 Add max to postgres config 2022-05-27 19:07:25 -04:00
Ajay
d038279d79 more logs 2022-05-27 12:41:09 -04:00
Ajay
35d3627760 more logs 2022-05-27 02:35:32 -04:00
Ajay
dcffb83e62 Logs for submiting 2022-05-27 01:24:34 -04:00
Ajay
a0465a44ae More logging 2022-05-26 22:59:59 -04:00
Ajay
db55d314ee Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into k8s-test 2022-05-26 22:49:27 -04:00
Ajay
b2af4fc7b0 Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into k8s-test 2022-05-26 22:47:17 -04:00
Ajay
4f28d92eb8 Fix params check 2022-05-26 22:21:00 -04:00
Ajay
c822a37a6e Add more logging for debugging 2022-05-26 22:15:09 -04:00
Ajay
fd636a2770 Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into k8s-test 2022-05-24 19:15:58 -04:00
Ajay
c2e0d5a98f logs 2022-05-20 04:27:39 -04:00
5 changed files with 207 additions and 10 deletions

View File

@@ -76,7 +76,9 @@ addDefaults(config, {
user: "", user: "",
host: "", host: "",
password: "", password: "",
port: 5432 port: 5432,
max: 150,
min: 10
}, },
dumpDatabase: { dumpDatabase: {
enabled: false, enabled: false,

View File

@@ -22,6 +22,8 @@ if (config.mysql) {
database: "sponsorTimes", database: "sponsorTimes",
password: config.postgres?.password, password: config.postgres?.password,
port: config.postgres?.port, port: config.postgres?.port,
max: config.postgres?.max,
min: config.postgres?.min,
} }
}); });
@@ -37,6 +39,8 @@ if (config.mysql) {
database: "privateDB", database: "privateDB",
password: config.postgres?.password, password: config.postgres?.password,
port: config.postgres?.port, port: config.postgres?.port,
max: config.postgres?.max,
min: config.postgres?.min,
} }
}); });
} else { } else {

View File

@@ -13,7 +13,7 @@ import { getReputation } from "../utils/reputation";
import { getService } from "../utils/getService"; import { getService } from "../utils/getService";
async function prepareCategorySegments(req: Request, videoID: VideoID, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, useCache: boolean): Promise<Segment[]> { async function prepareCategorySegments(req: Request, videoID: VideoID, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, useCache: boolean, logData: any): Promise<Segment[]> {
const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => { const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => {
if (segment.votes < -1 && !segment.required) { if (segment.votes < -1 && !segment.required) {
return false; //too untrustworthy, just ignore it return false; //too untrustworthy, just ignore it
@@ -25,6 +25,11 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, service:
return true; return true;
} }
if (logData.extraLogging) {
Logger.error(`Starting ip fetch: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
if (cache.shadowHiddenSegmentIPs[videoID] === undefined) cache.shadowHiddenSegmentIPs[videoID] = {}; if (cache.shadowHiddenSegmentIPs[videoID] === undefined) cache.shadowHiddenSegmentIPs[videoID] = {};
if (cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] === undefined) { if (cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] === undefined) {
const service = getService(req?.query?.service as string); const service = getService(req?.query?.service as string);
@@ -37,7 +42,15 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, service:
if (ipList?.length > 0 && cache.userHashedIP === undefined) { if (ipList?.length > 0 && cache.userHashedIP === undefined) {
//hash the IP only if it's strictly necessary //hash the IP only if it's strictly necessary
if (logData.extraLogging) {
Logger.error(`Starting hash: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
cache.userHashedIP = await getHashCache((getIP(req) + config.globalSalt) as IPAddress); cache.userHashedIP = await getHashCache((getIP(req) + config.globalSalt) as IPAddress);
if (logData.extraLogging) {
Logger.error(`Ending hash: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
} }
//if this isn't their ip, don't send it to them //if this isn't their ip, don't send it to them
const shouldShadowHide = cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted]?.some( const shouldShadowHide = cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted]?.some(
@@ -47,9 +60,19 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, service:
return shouldShadowHide; return shouldShadowHide;
})); }));
if (logData.extraLogging) {
Logger.error(`Preparing segments: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const filteredSegments = segments.filter((_, index) => shouldFilter[index]); const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
return (await chooseSegments(videoID, service, filteredSegments, useCache)).map((chosenSegment) => ({ if (logData.extraLogging) {
Logger.error(`Filter complete: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
return (await chooseSegments(videoID, service, filteredSegments, useCache, logData)).map((chosenSegment) => ({
category: chosenSegment.category, category: chosenSegment.category,
actionType: chosenSegment.actionType, actionType: chosenSegment.actionType,
segment: [chosenSegment.startTime, chosenSegment.endTime], segment: [chosenSegment.startTime, chosenSegment.endTime],
@@ -83,7 +106,7 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
}, {}); }, {});
const canUseCache = requiredSegments.length === 0; const canUseCache = requiredSegments.length === 0;
let processedSegments: Segment[] = (await prepareCategorySegments(req, videoID, service, segments, cache, canUseCache)) let processedSegments: Segment[] = (await prepareCategorySegments(req, videoID, service, segments, cache, canUseCache, {}))
.filter((segment: Segment) => categories.includes(segment?.category) && (actionTypes.includes(segment?.actionType))); .filter((segment: Segment) => categories.includes(segment?.category) && (actionTypes.includes(segment?.actionType)));
if (forcePoiAsSkip) { if (forcePoiAsSkip) {
@@ -113,12 +136,24 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
actionTypes.push(ActionType.Poi); actionTypes.push(ActionType.Poi);
} }
const logData = {
extraLogging: req.query.extraLogging,
startTime: Date.now(),
lastTime: Date.now()
};
try { try {
type SegmentWithHashPerVideoID = SBRecord<VideoID, { hash: VideoIDHash, segments: DBSegment[] }>; type SegmentWithHashPerVideoID = SBRecord<VideoID, { hash: VideoIDHash, segments: DBSegment[] }>;
categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category))); categories = categories.filter((category) => !(/[^a-z|_|-]/.test(category)));
if (categories.length === 0) return null; if (categories.length === 0) return null;
if (logData.extraLogging) {
Logger.warn(`IP: ${getIP(req)}, request con ip: ${req.socket?.remoteAddress}`);
Logger.error(`About to fetch: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service)) const segmentPerVideoID: SegmentWithHashPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service))
.reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => { .reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
acc[segment.videoID] = acc[segment.videoID] || { acc[segment.videoID] = acc[segment.videoID] || {
@@ -133,6 +168,11 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
return acc; return acc;
}, {}); }, {});
if (logData.extraLogging) {
Logger.error(`Fetch complete: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
for (const [videoID, videoData] of Object.entries(segmentPerVideoID)) { for (const [videoID, videoData] of Object.entries(segmentPerVideoID)) {
const data: VideoData = { const data: VideoData = {
hash: videoData.hash, hash: videoData.hash,
@@ -140,7 +180,7 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
}; };
const canUseCache = requiredSegments.length === 0; const canUseCache = requiredSegments.length === 0;
data.segments = (await prepareCategorySegments(req, videoID as VideoID, service, videoData.segments, cache, canUseCache)) data.segments = (await prepareCategorySegments(req, videoID as VideoID, service, videoData.segments, cache, canUseCache, logData))
.filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType)); .filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType));
if (forcePoiAsSkip) { if (forcePoiAsSkip) {
@@ -153,6 +193,11 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
if (data.segments.length > 0) { if (data.segments.length > 0) {
segments[videoID] = data; segments[videoID] = data;
} }
if (logData.extraLogging) {
Logger.error(`Done one video: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
} }
return segments; return segments;
@@ -255,13 +300,18 @@ function getWeightedRandomChoice<T extends VotableObject>(choices: T[], amountOf
return chosen; return chosen;
} }
async function chooseSegments(videoID: VideoID, service: Service, segments: DBSegment[], useCache: boolean): Promise<DBSegment[]> { async function chooseSegments(videoID: VideoID, service: Service, segments: DBSegment[], useCache: boolean, logData: any): Promise<DBSegment[]> {
const fetchData = async () => await buildSegmentGroups(segments); const fetchData = async () => await buildSegmentGroups(segments);
const groups = useCache const groups = useCache
? await QueryCacher.get(fetchData, skipSegmentGroupsKey(videoID, service)) ? await QueryCacher.get(fetchData, skipSegmentGroupsKey(videoID, service))
: await fetchData(); : await fetchData();
if (logData.extraLogging) {
Logger.error(`Built groups: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
// Filter for only 1 item for POI categories and Full video // Filter for only 1 item for POI categories and Full video
let chosenGroups = getWeightedRandomChoice(groups, 1, true, (choice) => choice.segments[0].actionType === ActionType.Full); let chosenGroups = getWeightedRandomChoice(groups, 1, true, (choice) => choice.segments[0].actionType === ActionType.Full);
chosenGroups = getWeightedRandomChoice(chosenGroups, 1, true, (choice) => choice.segments[0].actionType === ActionType.Poi); chosenGroups = getWeightedRandomChoice(chosenGroups, 1, true, (choice) => choice.segments[0].actionType === ActionType.Poi);

View File

@@ -340,9 +340,14 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
return CHECK_PASS; return CHECK_PASS;
} }
async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) { async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration, logData: any) {
let lockedCategoryList = await db.prepare("all", 'SELECT category, "actionType", reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]); let lockedCategoryList = await db.prepare("all", 'SELECT category, "actionType", reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]);
if (logData.extraLogging) {
Logger.error(`got locked categories: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const previousSubmissions = await db.prepare("all", const previousSubmissions = await db.prepare("all",
`SELECT "videoDuration", "UUID" `SELECT "videoDuration", "UUID"
FROM "sponsorTimes" FROM "sponsorTimes"
@@ -353,6 +358,11 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic
[videoID, service] [videoID, service]
) as {videoDuration: VideoDuration, UUID: SegmentUUID}[]; ) as {videoDuration: VideoDuration, UUID: SegmentUUID}[];
if (logData.extraLogging) {
Logger.error(`got previous submissions: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
// If the video's duration is changed, then the video should be unlocked and old submissions should be hidden // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
const videoDurationChanged = (videoDuration: number) => videoDuration != 0 const videoDurationChanged = (videoDuration: number) => videoDuration != 0
&& previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); && previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
@@ -368,6 +378,11 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic
videoDuration = apiVideoDuration || 0 as VideoDuration; videoDuration = apiVideoDuration || 0 as VideoDuration;
} }
if (logData.extraLogging) {
Logger.error(`got api response: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
// Only treat as difference if both the api duration and submitted duration have changed // Only treat as difference if both the api duration and submitted duration have changed
if (videoDurationChanged(videoDuration) && (!videoDurationParam || videoDurationChanged(videoDurationParam))) { if (videoDurationChanged(videoDuration) && (!videoDurationParam || videoDurationChanged(videoDurationParam))) {
// Hide all previous submissions // Hide all previous submissions
@@ -378,6 +393,11 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic
deleteLockCategories(videoID, null, null, service); deleteLockCategories(videoID, null, null, service);
} }
if (logData.extraLogging) {
Logger.error(`updated old submissions: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
return { return {
videoDuration, videoDuration,
apiVideoInfo, apiVideoInfo,
@@ -475,37 +495,73 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
proxySubmission(req); proxySubmission(req);
} }
const logData = {
extraLogging: req.query.extraLogging,
startTime: Date.now(),
lastTime: Date.now()
};
// eslint-disable-next-line prefer-const // eslint-disable-next-line prefer-const
let { videoID, userID: paramUserID, service, videoDuration, videoDurationParam, segments, userAgent } = preprocessInput(req); let { videoID, userID: paramUserID, service, videoDuration, videoDurationParam, segments, userAgent } = preprocessInput(req);
if (logData.extraLogging) {
Logger.error(`preprocess: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const invalidCheckResult = checkInvalidFields(videoID, paramUserID, segments); const invalidCheckResult = checkInvalidFields(videoID, paramUserID, segments);
if (!invalidCheckResult.pass) { if (!invalidCheckResult.pass) {
return res.status(invalidCheckResult.errorCode).send(invalidCheckResult.errorMessage); return res.status(invalidCheckResult.errorCode).send(invalidCheckResult.errorMessage);
} }
if (logData.extraLogging) {
Logger.error(`invalid fields: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
//hash the userID //hash the userID
const userID = await getHashCache(paramUserID); const userID = await getHashCache(paramUserID);
if (logData.extraLogging) {
Logger.error(`userid: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const userWarningCheckResult = await checkUserActiveWarning(userID); const userWarningCheckResult = await checkUserActiveWarning(userID);
if (!userWarningCheckResult.pass) { if (!userWarningCheckResult.pass) {
Logger.warn(`Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`); Logger.warn(`Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce<string>((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce<string>((prev, val) => `${prev} ${val.segment}`, "")}`);
return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage); return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage);
} }
if (logData.extraLogging) {
Logger.error(`warning done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const isVIP = await isUserVIP(userID); const isVIP = await isUserVIP(userID);
const isTempVIP = await isUserTempVIP(userID, videoID); const isTempVIP = await isUserTempVIP(userID, videoID);
const rawIP = getIP(req); const rawIP = getIP(req);
const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam); const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam, logData);
videoDuration = newData.videoDuration; videoDuration = newData.videoDuration;
const { lockedCategoryList, apiVideoInfo } = newData; const { lockedCategoryList, apiVideoInfo } = newData;
if (logData.extraLogging) {
Logger.error(`duration change done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
// Check if all submissions are correct // Check if all submissions are correct
const segmentCheckResult = await checkEachSegmentValid(rawIP, paramUserID, userID, videoID, segments, service, isVIP, lockedCategoryList); const segmentCheckResult = await checkEachSegmentValid(rawIP, paramUserID, userID, videoID, segments, service, isVIP, lockedCategoryList);
if (!segmentCheckResult.pass) { if (!segmentCheckResult.pass) {
return res.status(segmentCheckResult.errorCode).send(segmentCheckResult.errorMessage); return res.status(segmentCheckResult.errorCode).send(segmentCheckResult.errorMessage);
} }
if (logData.extraLogging) {
Logger.error(`valid segmenrs done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
if (!isVIP && !isTempVIP) { if (!isVIP && !isTempVIP) {
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoInfo, videoDurationParam); const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoInfo, videoDurationParam);
if (!autoModerateCheckResult.pass) { if (!autoModerateCheckResult.pass) {
@@ -513,6 +569,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
} }
} }
if (logData.extraLogging) {
Logger.error(`Checks done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
// Will be filled when submitting // Will be filled when submitting
const UUIDs = []; const UUIDs = [];
const newSegments = []; const newSegments = [];
@@ -520,6 +581,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//hash the ip 5000 times so no one can get it from the database //hash the ip 5000 times so no one can get it from the database
const hashedIP = await getHashCache(rawIP + config.globalSalt); const hashedIP = await getHashCache(rawIP + config.globalSalt);
if (logData.extraLogging) {
Logger.error(`IP hash done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
try { try {
//get current time //get current time
const timeSubmitted = Date.now(); const timeSubmitted = Date.now();
@@ -534,6 +600,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
const startingVotes = 0; const startingVotes = 0;
const reputation = await getReputation(userID); const reputation = await getReputation(userID);
if (logData.extraLogging) {
Logger.error(`Ban check complete: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
for (const segmentInfo of segments) { for (const segmentInfo of segments) {
// Full segments are always rejected since there can only be one, so shadow hide wouldn't work // Full segments are always rejected since there can only be one, so shadow hide wouldn't work
if (segmentInfo.ignoreSegment if (segmentInfo.ignoreSegment
@@ -547,6 +618,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
const UUID = getSubmissionUUID(videoID, segmentInfo.category, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service); const UUID = getSubmissionUUID(videoID, segmentInfo.category, segmentInfo.actionType, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]), service);
const hashedVideoID = getHash(videoID, 1); const hashedVideoID = getHash(videoID, 1);
if (logData.extraLogging) {
Logger.error(`Submission prep done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const startingLocked = isVIP ? 1 : 0; const startingLocked = isVIP ? 1 : 0;
try { try {
await db.prepare("run", `INSERT INTO "sponsorTimes" await db.prepare("run", `INSERT INTO "sponsorTimes"
@@ -557,14 +633,29 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
], ],
); );
if (logData.extraLogging) {
Logger.error(`Normal DB done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
//add to private db as well //add to private db as well
await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?, ?)`, [videoID, hashedIP, timeSubmitted, service]); await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?, ?)`, [videoID, hashedIP, timeSubmitted, service]);
if (logData.extraLogging) {
Logger.error(`Private db: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published", "genreUrl") await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published", "genreUrl")
SELECT ?, ?, ?, ?, ? SELECT ?, ?, ?, ?, ?
WHERE NOT EXISTS (SELECT 1 FROM "videoInfo" WHERE "videoID" = ?)`, [ WHERE NOT EXISTS (SELECT 1 FROM "videoInfo" WHERE "videoID" = ?)`, [
videoID, apiVideoInfo?.data?.authorId || "", apiVideoInfo?.data?.title || "", apiVideoInfo?.data?.published || 0, apiVideoInfo?.data?.genreUrl || "", videoID]); videoID, apiVideoInfo?.data?.authorId || "", apiVideoInfo?.data?.title || "", apiVideoInfo?.data?.published || 0, apiVideoInfo?.data?.genreUrl || "", videoID]);
if (logData.extraLogging) {
Logger.error(`Video info: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
// Clear redis cache for this video // Clear redis cache for this video
QueryCacher.clearSegmentCache({ QueryCacher.clearSegmentCache({
videoID, videoID,
@@ -590,6 +681,11 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
return res.sendStatus(500); return res.sendStatus(500);
} }
if (logData.extraLogging) {
Logger.error(`Sending webhooks: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
for (let i = 0; i < segments.length; i++) { for (let i = 0; i < segments.length; i++) {
sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service); sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service);
} }

View File

@@ -102,6 +102,7 @@ async function checkVideoDuration(UUID: SegmentUUID) {
} }
async function sendWebhooks(voteData: VoteData) { async function sendWebhooks(voteData: VoteData) {
Logger.error(`about to send webhook ${JSON.stringify(voteData)}`);
const submissionInfoRow = await db.prepare("get", `SELECT "s"."videoID", "s"."userID", s."startTime", s."endTime", s."category", u."userName", const submissionInfoRow = await db.prepare("get", `SELECT "s"."videoID", "s"."userID", s."startTime", s."endTime", s."category", u."userName",
(select count(1) from "sponsorTimes" where "userID" = s."userID") count, (select count(1) from "sponsorTimes" where "userID" = s."userID") count,
(select count(1) from "sponsorTimes" where "userID" = s."userID" and votes <= -2) disregarded (select count(1) from "sponsorTimes" where "userID" = s."userID" and votes <= -2) disregarded
@@ -110,6 +111,8 @@ async function sendWebhooks(voteData: VoteData) {
const userSubmissionCountRow = await db.prepare("get", `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [voteData.nonAnonUserID]); const userSubmissionCountRow = await db.prepare("get", `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [voteData.nonAnonUserID]);
Logger.error(`Sending webhook ${config.discordReportChannelWebhookURL}, ${submissionInfoRow}, ${userSubmissionCountRow}`)
if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) { if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) {
let webhookURL: string = null; let webhookURL: string = null;
if (voteData.voteTypeEnum === voteTypes.normal) { if (voteData.voteTypeEnum === voteTypes.normal) {
@@ -307,7 +310,18 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
const category = req.query.category as Category; const category = req.query.category as Category;
const ip = getIP(req); const ip = getIP(req);
const result = await vote(ip, UUID, paramUserID, type, category); const logData = {
extraLogging: req.query.extraLogging,
startTime: Date.now(),
lastTime: Date.now()
};
const result = await vote(ip, UUID, paramUserID, type, category, logData);
if (logData.extraLogging) {
Logger.error(`Done vote: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const response = res.status(result.status); const response = res.status(result.status);
if (result.message) { if (result.message) {
@@ -319,7 +333,7 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
} }
} }
export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID, type: number, category?: Category): Promise<{ status: number, message?: string, json?: unknown }> { export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID, type: number, category?: Category, logData = {} as any): Promise<{ status: number, message?: string, json?: unknown }> {
// missing key parameters // missing key parameters
if (!UUID || !paramUserID || !(type !== undefined || category)) { if (!UUID || !paramUserID || !(type !== undefined || category)) {
return { status: 400 }; return { status: 400 };
@@ -351,6 +365,11 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
return { status: 404 }; return { status: 404 };
} }
if (logData.extraLogging) {
Logger.error(`Got segment info: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
const isTempVIP = await isUserTempVIP(nonAnonUserID, segmentInfo.videoID); const isTempVIP = await isUserTempVIP(nonAnonUserID, segmentInfo.videoID);
const isVIP = await isUserVIP(nonAnonUserID); const isVIP = await isUserVIP(nonAnonUserID);
@@ -412,10 +431,20 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
checkVideoDuration(UUID); checkVideoDuration(UUID);
} }
if (logData.extraLogging) {
Logger.error(`Main prep done: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
try { try {
// check if vote has already happened // check if vote has already happened
const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID]); const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID]);
if (logData.extraLogging) {
Logger.error(`Got previous vote info: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
// -1 for downvote, 1 for upvote. Maybe more depending on reputation in the future // -1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
// oldIncrementAmount will be zero if row is null // oldIncrementAmount will be zero if row is null
let incrementAmount = 0; let incrementAmount = 0;
@@ -477,6 +506,11 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
const ableToVote = isVIP || isTempVIP || userAbleToVote; const ableToVote = isVIP || isTempVIP || userAbleToVote;
if (logData.extraLogging) {
Logger.error(`About to run vote code ${ableToVote}: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
if (ableToVote) { if (ableToVote) {
//update the votes table //update the votes table
if (votesRow) { if (votesRow) {
@@ -504,8 +538,19 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
await db.prepare("run", 'UPDATE "sponsorTimes" SET "locked" = 0 WHERE "UUID" = ?', [UUID]); await db.prepare("run", 'UPDATE "sponsorTimes" SET "locked" = 0 WHERE "UUID" = ?', [UUID]);
} }
if (logData.extraLogging) {
Logger.error(`Did vote stuff: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
QueryCacher.clearSegmentCache(segmentInfo); QueryCacher.clearSegmentCache(segmentInfo);
} }
if (logData.extraLogging) {
Logger.error(`Ready to send webhook ${incrementAmount - oldIncrementAmount !== 0}: ${Date.now() - logData.lastTime}, ${Date.now() - logData.startTime}`);
logData.lastTime = Date.now();
}
if (incrementAmount - oldIncrementAmount !== 0) { if (incrementAmount - oldIncrementAmount !== 0) {
sendWebhooks({ sendWebhooks({
UUID, UUID,