Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 11:36:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Don't always use YouTube API cache

Browse Source
This commit is contained in:
Ajay Ramachandran
2021-04-08 20:37:19 -04:00
parent 8088f37632
commit 6a9b218e22
6 changed files with 183 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/routes/postSkipSegments.ts
View File

@@ -19,7 +19,7 @@ import { getCategoryActionType } from '../utils/categoryInfo';
interface APIVideoInfo {
err: string | boolean,
data: any
data?: any
}
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
@@ -276,11 +276,9 @@ function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
}
async function getYouTubeVideoInfo(videoID: VideoID): Promise<APIVideoInfo> {
async function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
if (config.youtubeAPIKey !== null) {
return new Promise((resolve) => {
YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
});
return YouTubeAPI.listVideos(videoID, ignoreCache);
} else {
return null;
}
@@ -368,9 +366,16 @@ export async function postSkipSegments(req: Request, res: Response) {
const decreaseVotes = 0;
const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0
AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as
{videoDuration: VideoDuration, UUID: SegmentUUID}[];
// If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
const videoDurationChanged = (videoDuration: number) => previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
let apiVideoInfo: APIVideoInfo = null;
if (service == Service.YouTube) {
apiVideoInfo = await getYouTubeVideoInfo(videoID);
// Don't use cache if we don't know the video duraton, or the client claims that it has changed
apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDuration || videoDurationChanged(videoDuration));
}
const apiVideoDuration = getYouTubeVideoDuration(apiVideoInfo);
if (!videoDuration || (apiVideoDuration && Math.abs(videoDuration - apiVideoDuration) > 2)) {
@@ -378,12 +383,7 @@ export async function postSkipSegments(req: Request, res: Response) {
videoDuration = apiVideoDuration || 0 as VideoDuration;
}
const previousSubmissions = await db.prepare('all', `SELECT "videoDuration", "UUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0
AND "shadowHidden" = 0 AND "votes" >= 0 AND "videoDuration" != 0`, [videoID, service]) as
{videoDuration: VideoDuration, UUID: SegmentUUID}[];
// If the video's duration is changed, then the video should be unlocked and old submissions should be hidden
const videoDurationChanged = previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
if (videoDurationChanged) {
if (videoDurationChanged(videoDuration)) {
// Hide all previous submissions
for (const submission of previousSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);

163
src/routes/voteOnSponsorTime.ts
View File

@@ -60,90 +60,89 @@ async function sendWebhooks(voteData: VoteData) {
}
if (config.youtubeAPIKey !== null) {
YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => {
if (err || data.items.length === 0) {
err && Logger.error(err.toString());
return;
}
const isUpvote = voteData.incrementAmount > 0;
// Send custom webhooks
dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
"user": {
"status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"video": {
"id": submissionInfoRow.videoID,
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
"thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
},
"submission": {
"UUID": voteData.UUID,
"views": voteData.row.views,
"category": voteData.category,
"startTime": submissionInfoRow.startTime,
"endTime": submissionInfoRow.endTime,
"user": {
"UUID": submissionInfoRow.userID,
"username": submissionInfoRow.userName,
"submissions": {
"total": submissionInfoRow.count,
"ignored": submissionInfoRow.disregarded,
},
},
},
"votes": {
"before": voteData.row.votes,
"after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount),
},
});
// Send discord message
if (webhookURL !== null && !isUpvote) {
fetch(webhookURL, {
method: 'POST',
body: JSON.stringify({
"embeds": [{
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID
+ "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
"description": "**" + voteData.row.votes + " Votes Prior | " +
(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views
+ " Views**\n\n**Submission ID:** " + voteData.UUID
+ "\n**Category:** " + submissionInfoRow.category
+ "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID
+ "\n\n**Total User Submissions:** " + submissionInfoRow.count
+ "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded
+ "\n\n**Timestamp:** " +
getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
"color": 10813440,
"author": {
"name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"thumbnail": {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
},
}],
}),
headers: {
'Content-Type': 'application/json'
}
})
.then(async res => {
if (res.status >= 400) {
Logger.error("Error sending reported submission Discord hook");
Logger.error(JSON.stringify((await res.text())));
Logger.error("\n");
}
})
.catch(err => {
Logger.error("Failed to send reported submission Discord hook.");
Logger.error(JSON.stringify(err));
Logger.error("\n");
});
}
const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID);
if (err || data.items.length === 0) {
if (err) Logger.error(err.toString());
return;
}
const isUpvote = voteData.incrementAmount > 0;
// Send custom webhooks
dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
// "user": {
// "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
// },
// "video": {
// "id": submissionInfoRow.videoID,
// "title": data.items[0].snippet.title,
// "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
// "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
// },
// "submission": {
// "UUID": voteData.UUID,
// "views": voteData.row.views,
// "category": voteData.category,
// "startTime": submissionInfoRow.startTime,
// "endTime": submissionInfoRow.endTime,
// "user": {
// "UUID": submissionInfoRow.userID,
// "username": submissionInfoRow.userName,
// "submissions": {
// "total": submissionInfoRow.count,
// "ignored": submissionInfoRow.disregarded,
// },
// },
// },
// "votes": {
// "before": voteData.row.votes,
// "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount),
// },
});
// Send discord message
if (webhookURL !== null && !isUpvote) {
fetch(webhookURL, {
method: 'POST',
body: JSON.stringify({
"embeds": [{
"title": data.items[0].snippet.title,
"url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID
+ "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
"description": "**" + voteData.row.votes + " Votes Prior | " +
(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views
+ " Views**\n\n**Submission ID:** " + voteData.UUID
+ "\n**Category:** " + submissionInfoRow.category
+ "\n\n**Submitted by:** " + submissionInfoRow.userName + "\n " + submissionInfoRow.userID
+ "\n\n**Total User Submissions:** " + submissionInfoRow.count
+ "\n**Ignored User Submissions:** " + submissionInfoRow.disregarded
+ "\n\n**Timestamp:** " +
getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
"color": 10813440,
"author": {
"name": voteData.finalResponse?.finalMessage ?? getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission),
},
"thumbnail": {
"url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
},
}],
}),
headers: {
'Content-Type': 'application/json'
}
})
.then(async res => {
if (res.status >= 400) {
Logger.error("Error sending reported submission Discord hook");
Logger.error(JSON.stringify((await res.text())));
Logger.error("\n");
}
})
.catch(err => {
Logger.error("Failed to send reported submission Discord hook.");
Logger.error(JSON.stringify(err));
Logger.error("\n");
});
}
}
}
}

8
src/utils/webhookUtils.ts
View File

@@ -1,6 +1,7 @@
import {config} from '../config';
import {Logger} from '../utils/logger';
import fetch from 'node-fetch';
import AbortController from "abort-controller";
function getVoteAuthorRaw(submissionCount: number, isVIP: boolean, isOwnSubmission: boolean): string {
if (isOwnSubmission) {
@@ -30,7 +31,8 @@ function dispatchEvent(scope: string, data: any): void {
let webhooks = config.webhooks;
if (webhooks === undefined || webhooks.length === 0) return;
Logger.debug("Dispatching webhooks");
webhooks.forEach(webhook => {
for (const webhook of webhooks) {
let webhookURL = webhook.url;
let authKey = webhook.key;
let scopes = webhook.scopes || [];
@@ -43,13 +45,13 @@ function dispatchEvent(scope: string, data: any): void {
"Authorization": authKey,
"Event-Type": scope, // Maybe change this in the future?
'Content-Type': 'application/json'
},
}
})
.catch(err => {
Logger.warn('Couldn\'t send webhook to ' + webhook.url);
Logger.warn(err);
});
});
}
}
export {

66
src/utils/youtubeApi.ts
View File

@@ -10,43 +10,45 @@ _youTubeAPI.authenticate({
});
export class YouTubeAPI {
static listVideos(videoID: string, callback: (err: string | boolean, data: any) => void) {
static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> {
const part = 'contentDetails,snippet';
if (!videoID || videoID.length !== 11 || videoID.includes(".")) {
callback("Invalid video ID", undefined);
return;
return { err: "Invalid video ID" };
}
const redisKey = "youtube.video." + videoID;
redis.get(redisKey, (getErr, result) => {
if (getErr || !result) {
if (!ignoreCache) {
const {err, reply} = await redis.getAsync(redisKey);
if (!err && reply) {
Logger.debug("redis: no cache for video information: " + videoID);
_youTubeAPI.videos.list({
part,
id: videoID,
}, (ytErr: boolean | string, { data }: any) => {
if (!ytErr) {
// Only set cache if data returned
if (data.items.length > 0) {
redis.set(redisKey, JSON.stringify(data), (setErr) => {
if (setErr) {
Logger.warn(setErr.message);
} else {
Logger.debug("redis: video information cache set for: " + videoID);
}
callback(false, data); // don't fail
});
} else {
callback(false, data); // don't fail
}
} else {
callback(ytErr, data);
}
});
} else {
Logger.debug("redis: fetched video information from cache: " + videoID);
callback(getErr?.message, JSON.parse(result));
return { err: err?.message, data: JSON.parse(reply) }
}
});
};
}
const { ytErr, data } = await new Promise((resolve) => _youTubeAPI.videos.list({
part,
id: videoID,
}, (ytErr: boolean | string, { data }: any) => resolve({ytErr, data})));
if (!ytErr) {
// Only set cache if data returned
if (data.items.length > 0) {
const { err: setErr } = await redis.setAsync(redisKey, JSON.stringify(data));
if (setErr) {
Logger.warn(setErr.message);
} else {
Logger.debug("redis: video information cache set for: " + videoID);
}
return { err: false, data }; // don't fail
} else {
return { err: false, data }; // don't fail
}
} else {
return { err: ytErr, data };
}
}
}

7
test.json
View File

@@ -40,13 +40,6 @@
"vote.up",
"vote.down"
]
}, {
"url": "http://unresolvable.host:8081/FailedWebhook",
"key": "superSecretKey",
"scopes": [
"vote.up",
"vote.down"
]
}
],
"categoryList": ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "highlight"],

92
test/youtubeMock.ts
View File

@@ -9,61 +9,71 @@ YouTubeAPI.videos.list({
export class YouTubeApiMock {
static listVideos(videoID: string, callback: (ytErr: any, data: any) => void) {
static async listVideos(videoID: string, ignoreCache = false): Promise<{err: string | boolean, data?: any}> {
const obj = {
id: videoID
};
if (obj.id === "knownWrongID") {
callback(undefined, {
pageInfo: {
totalResults: 0,
},
items: [],
});
return {
err: null,
data: {
pageInfo: {
totalResults: 0,
},
items: [],
}
};
}
if (obj.id === "noDuration") {
callback(undefined, {
pageInfo: {
totalResults: 1,
},
items: [
{
contentDetails: {
duration: "PT0S",
},
snippet: {
title: "Example Title",
thumbnails: {
maxres: {
url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
return {
err: null,
data: {
pageInfo: {
totalResults: 1,
},
items: [
{
contentDetails: {
duration: "PT0S",
},
snippet: {
title: "Example Title",
thumbnails: {
maxres: {
url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
},
},
},
},
},
],
});
],
}
};
} else {
callback(undefined, {
pageInfo: {
totalResults: 1,
},
items: [
{
contentDetails: {
duration: "PT1H23M30S",
},
snippet: {
title: "Example Title",
thumbnails: {
maxres: {
url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
return {
err: null,
data: {
pageInfo: {
totalResults: 1,
},
items: [
{
contentDetails: {
duration: "PT1H23M30S",
},
snippet: {
title: "Example Title",
thumbnails: {
maxres: {
url: "https://sponsor.ajay.app/LogoSponsorBlockSimple256px.png",
},
},
},
},
},
],
});
],
}
};
}
}
}