Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-26 01:18:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #503 from mchangrh/innerTubeDuration

Browse Source 
add innerTube API, types and tests
This commit is contained in:
Ajay Ramachandran
2022-09-29 16:45:55 -04:00
committed by GitHub
parent 54e53f098e 55aa33aa6e
commit ada32217d9
15 changed files with 251 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/routes/addUserAsTempVIP.ts
View File

@@ -1,7 +1,5 @@
import { VideoID } from "../types/segments.model";
import { YouTubeAPI } from "../utils/youtubeApi";
import { APIVideoInfo } from "../types/youtubeApi.model";
import { config } from "../config";
import { getVideoDetails } from "../utils/getVideoDetails";
import { getHashCache } from "../utils/getHashCache";
import { privateDB } from "../databases/databases";
import { Request, Response } from "express";
@@ -20,15 +18,11 @@ interface AddUserAsTempVIPRequest extends Request {
}
}
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
return (config.newLeafURLs) ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
}
const getChannelInfo = async (videoID: VideoID): Promise<{id: string | null, name: string | null }> => {
const videoInfo = await getYouTubeVideoInfo(videoID);
const videoInfo = await getVideoDetails(videoID);
return {
id: videoInfo?.data?.authorId,
name: videoInfo?.data?.author
id: videoInfo?.authorId,
name: videoInfo?.authorName
};
};

70
src/routes/postSkipSegments.ts
View File

@@ -1,7 +1,7 @@
import { config } from "../config";
import { Logger } from "../utils/logger";
import { db, privateDB } from "../databases/databases";
import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi";
import { getMaxResThumbnail } from "../utils/youtubeApi";
import { getSubmissionUUID } from "../utils/getSubmissionUUID";
import { getHash } from "../utils/getHash";
import { getHashCache } from "../utils/getHashCache";
@@ -13,7 +13,6 @@ import { ActionType, Category, IncomingSegment, IPAddress, SegmentUUID, Service,
import { deleteLockCategories } from "./deleteLockCategories";
import { QueryCacher } from "../utils/queryCacher";
import { getReputation } from "../utils/reputation";
import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
import { HashedUserID, UserID } from "../types/user.model";
import { isUserVIP } from "../utils/isUserVIP";
import { isUserTempVIP } from "../utils/isUserTempVIP";
@@ -22,6 +21,7 @@ import { getService } from "../utils/getService";
import axios from "axios";
import { vote } from "./voteOnSponsorTime";
import { canSubmit } from "../utils/permissions";
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
type CheckResult = {
pass: boolean,
@@ -35,7 +35,7 @@ const CHECK_PASS: CheckResult = {
errorCode: 0
};
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, { submissionStart, submissionEnd }: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: videoDetails, { submissionStart, submissionEnd }: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
const userName = row !== undefined ? row.userName : null;
@@ -48,7 +48,7 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
"video": {
"id": videoID,
"title": youtubeData?.title,
"thumbnail": getMaxResThumbnail(youtubeData) || null,
"thumbnail": getMaxResThumbnail(videoID),
"url": `https://www.youtube.com/watch?v=${videoID}`,
},
"submission": {
@@ -64,16 +64,13 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
});
}
async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
if (apiVideoInfo && service == Service.YouTube) {
async function sendWebhooks(apiVideoDetails: videoDetails, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
if (apiVideoDetails && service == Service.YouTube) {
const userSubmissionCountRow = await db.prepare("get", `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
const { data, err } = apiVideoInfo;
if (err) return;
const startTime = parseFloat(segmentInfo.segment[0]);
const endTime = parseFloat(segmentInfo.segment[1]);
sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, apiVideoDetails, {
submissionStart: startTime,
submissionEnd: endTime,
}, segmentInfo).catch(Logger.error);
@@ -84,7 +81,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
axios.post(config.discordFirstTimeSubmissionsWebhookURL, {
embeds: [{
title: data?.title,
title: apiVideoDetails.title,
url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`,
description: `Submission ID: ${UUID}\
\n\nTimestamp: \
@@ -95,7 +92,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
name: userID,
},
thumbnail: {
url: getMaxResThumbnail(data) || "",
url: getMaxResThumbnail(videoID),
},
}],
})
@@ -120,18 +117,10 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
// false for a pass - it was confusing and lead to this bug - any use of this function in
// the future could have the same problem.
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
async function autoModerateSubmission(apiVideoDetails: videoDetails,
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
const apiVideoDuration = (apiVideoInfo: APIVideoInfo) => {
if (!apiVideoInfo) return undefined;
const { err, data } = apiVideoInfo;
// return undefined if API error
if (err) return undefined;
return data?.lengthSeconds;
};
// get duration from API
const apiDuration = apiVideoDuration(apiVideoInfo);
const apiDuration = apiVideoDetails.duration;
// if API fail or returns 0, get duration from client
const duration = apiDuration || submission.videoDuration;
// return false on undefined or 0
@@ -165,14 +154,6 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
return false;
}
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
if (config.newLeafURLs !== null) {
return YouTubeAPI.listVideos(videoID, ignoreCache);
} else {
return null;
}
}
async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
const MILLISECONDS_IN_HOUR = 3600000;
const now = Date.now();
@@ -345,10 +326,10 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
return CHECK_PASS;
}
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoInfo: APIVideoInfo, videoDuration: number): Promise<CheckResult> {
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
// Auto moderator check
if (service == Service.YouTube) {
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service, videoDuration });
const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { userID, videoID, segments, service, videoDuration });
if (autoModerateResult) {
return {
pass: false,
@@ -377,12 +358,13 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic
const videoDurationChanged = (videoDuration: number) => videoDuration != 0
&& previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
let apiVideoInfo: APIVideoInfo = null;
let apiVideoDetails: videoDetails = null;
if (service == Service.YouTube) {
// Don't use cache if we don't know the video duration, or the client claims that it has changed
apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam));
const ignoreCache = !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam);
apiVideoDetails = await getVideoDetails(videoID, ignoreCache);
}
const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
if (!videoDurationParam || (apiVideoDuration && Math.abs(videoDurationParam - apiVideoDuration) > 2)) {
// If api duration is far off, take that one instead (it is only precise to seconds, not millis)
videoDuration = apiVideoDuration || 0 as VideoDuration;
@@ -400,7 +382,7 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic
return {
videoDuration,
apiVideoInfo,
apiVideoDetails,
lockedCategoryList
};
}
@@ -501,10 +483,6 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//hash the userID
const userID = await getHashCache(paramUserID || "");
if (userID === "a41d853c7328a86f8d712f910c4ef77f6c7a9e467f349781b1a7d405c37b681b") {
return res.status(200);
}
const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent);
if (!invalidCheckResult.pass) {
return res.status(invalidCheckResult.errorCode).send(invalidCheckResult.errorMessage);
@@ -521,7 +499,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam);
videoDuration = newData.videoDuration;
const { lockedCategoryList, apiVideoInfo } = newData;
const { lockedCategoryList, apiVideoDetails } = newData;
// Check if all submissions are correct
const segmentCheckResult = await checkEachSegmentValid(rawIP, paramUserID, userID, videoID, segments, service, isVIP, lockedCategoryList);
@@ -530,7 +508,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
}
if (!isVIP) {
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoInfo, videoDurationParam);
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoDetails, videoDurationParam);
if (!autoModerateCheckResult.pass) {
return res.status(autoModerateCheckResult.errorCode).send(autoModerateCheckResult.errorMessage);
}
@@ -583,10 +561,10 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
//add to private db as well
await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?, ?)`, [videoID, hashedIP, timeSubmitted, service]);
await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published", "genreUrl")
SELECT ?, ?, ?, ?, ?
await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published")
SELECT ?, ?, ?, ?
WHERE NOT EXISTS (SELECT 1 FROM "videoInfo" WHERE "videoID" = ?)`, [
videoID, apiVideoInfo?.data?.authorId || "", apiVideoInfo?.data?.title || "", apiVideoInfo?.data?.published || 0, apiVideoInfo?.data?.genreUrl || "", videoID]);
videoID, apiVideoDetails?.authorId || "", apiVideoDetails?.title || "", apiVideoDetails?.published || 0, videoID]);
// Clear redis cache for this video
QueryCacher.clearSegmentCache({
@@ -614,7 +592,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
}
for (let i = 0; i < segments.length; i++) {
sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service).catch(Logger.error);
sendWebhooks(apiVideoDetails, userID, videoID, UUIDs[i], segments[i], service).catch(Logger.error);
}
return res.json(newSegments);
}

27
src/routes/voteOnSponsorTime.ts
View File

@@ -3,7 +3,6 @@ import { Logger } from "../utils/logger";
import { isUserVIP } from "../utils/isUserVIP";
import { isUserTempVIP } from "../utils/isUserTempVIP";
import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi";
import { APIVideoInfo } from "../types/youtubeApi.model";
import { db, privateDB } from "../databases/databases";
import { dispatchEvent, getVoteAuthor, getVoteAuthorRaw } from "../utils/webhookUtils";
import { getFormattedTime } from "../utils/getFormattedTime";
@@ -14,6 +13,7 @@ import { UserID } from "../types/user.model";
import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, VideoDuration, ActionType, VoteType } from "../types/segments.model";
import { QueryCacher } from "../utils/queryCacher";
import axios from "axios";
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
const voteTypes = {
normal: 0,
@@ -52,20 +52,16 @@ interface VoteData {
finalResponse: FinalResponse;
}
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
}
const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2);
async function updateSegmentVideoDuration(UUID: SegmentUUID) {
const { videoDuration, videoID, service } = await db.prepare("get", `select "videoDuration", "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]);
let apiVideoInfo: APIVideoInfo = null;
let apiVideoDetails: videoDetails = null;
if (service == Service.YouTube) {
// don't use cache since we have no information about the video length
apiVideoInfo = await getYouTubeVideoInfo(videoID);
apiVideoDetails = await getVideoDetails(videoID);
}
const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
if (videoDurationChanged(videoDuration, apiVideoDuration)) {
Logger.info(`Video duration changed for ${videoID} from ${videoDuration} to ${apiVideoDuration}`);
await db.prepare("run", `UPDATE "sponsorTimes" SET "videoDuration" = ? WHERE "UUID" = ?`, [apiVideoDuration, UUID]);
@@ -74,12 +70,12 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) {
async function checkVideoDuration(UUID: SegmentUUID) {
const { videoID, service } = await db.prepare("get", `select "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]);
let apiVideoInfo: APIVideoInfo = null;
let apiVideoDetails: videoDetails = null;
if (service == Service.YouTube) {
// don't use cache since we have no information about the video length
apiVideoInfo = await getYouTubeVideoInfo(videoID, true);
apiVideoDetails = await getVideoDetails(videoID, true);
}
const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
// if no videoDuration return early
if (isNaN(apiVideoDuration)) return;
// fetch latest submission
@@ -129,7 +125,8 @@ async function sendWebhooks(voteData: VoteData) {
}
if (config.newLeafURLs !== null) {
const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID);
const videoID = submissionInfoRow.videoID;
const { err, data } = await YouTubeAPI.listVideos(videoID);
if (err) return;
const isUpvote = voteData.incrementAmount > 0;
@@ -141,8 +138,8 @@ async function sendWebhooks(voteData: VoteData) {
"video": {
"id": submissionInfoRow.videoID,
"title": data?.title,
"url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}`,
"thumbnail": getMaxResThumbnail(data) || null,
"url": `https://www.youtube.com/watch?v=${videoID}`,
"thumbnail": getMaxResThumbnail(videoID),
},
"submission": {
"UUID": voteData.UUID,
@@ -187,7 +184,7 @@ async function sendWebhooks(voteData: VoteData) {
`${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`,
},
"thumbnail": {
"url": getMaxResThumbnail(data) || "",
"url": getMaxResThumbnail(videoID),
},
}],
})