diff --git a/DatabaseSchema.md b/DatabaseSchema.md index ab11b53..535e667 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -126,7 +126,6 @@ | channelID | TEXT | not null | | title | TEXT | not null | | published | REAL | not null | -| genreUrl | TEXT | not null | | index | field | | -- | :--: | diff --git a/databases/_upgrade_sponsorTimes_34.sql b/databases/_upgrade_sponsorTimes_34.sql new file mode 100644 index 0000000..e8d41a4 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_34.sql @@ -0,0 +1,7 @@ +BEGIN TRANSACTION; + +ALTER TABLE "videoInfo" DROP COLUMN "genreUrl"; + +UPDATE "config" SET value = 34 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/routes/addUserAsTempVIP.ts b/src/routes/addUserAsTempVIP.ts index 3001a0e..774d661 100644 --- a/src/routes/addUserAsTempVIP.ts +++ b/src/routes/addUserAsTempVIP.ts @@ -1,7 +1,5 @@ import { VideoID } from "../types/segments.model"; -import { YouTubeAPI } from "../utils/youtubeApi"; -import { APIVideoInfo } from "../types/youtubeApi.model"; -import { config } from "../config"; +import { getVideoDetails } from "../utils/getVideoDetails"; import { getHashCache } from "../utils/getHashCache"; import { privateDB } from "../databases/databases"; import { Request, Response } from "express"; @@ -20,15 +18,11 @@ interface AddUserAsTempVIPRequest extends Request { } } -function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { - return (config.newLeafURLs) ? YouTubeAPI.listVideos(videoID, ignoreCache) : null; -} - const getChannelInfo = async (videoID: VideoID): Promise<{id: string | null, name: string | null }> => { - const videoInfo = await getYouTubeVideoInfo(videoID); + const videoInfo = await getVideoDetails(videoID); return { - id: videoInfo?.data?.authorId, - name: videoInfo?.data?.author + id: videoInfo?.authorId, + name: videoInfo?.authorName }; }; diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts index 755b5b7..97cd3c7 100644 --- a/src/routes/postSkipSegments.ts +++ b/src/routes/postSkipSegments.ts @@ -1,7 +1,7 @@ import { config } from "../config"; import { Logger } from "../utils/logger"; import { db, privateDB } from "../databases/databases"; -import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi"; +import { getMaxResThumbnail } from "../utils/youtubeApi"; import { getSubmissionUUID } from "../utils/getSubmissionUUID"; import { getHash } from "../utils/getHash"; import { getHashCache } from "../utils/getHashCache"; @@ -13,7 +13,6 @@ import { ActionType, Category, IncomingSegment, IPAddress, SegmentUUID, Service, import { deleteLockCategories } from "./deleteLockCategories"; import { QueryCacher } from "../utils/queryCacher"; import { getReputation } from "../utils/reputation"; -import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model"; import { HashedUserID, UserID } from "../types/user.model"; import { isUserVIP } from "../utils/isUserVIP"; import { isUserTempVIP } from "../utils/isUserTempVIP"; @@ -22,6 +21,7 @@ import { getService } from "../utils/getService"; import axios from "axios"; import { vote } from "./voteOnSponsorTime"; import { canSubmit } from "../utils/permissions"; +import { getVideoDetails, videoDetails } from "../utils/getVideoDetails"; type CheckResult = { pass: boolean, @@ -35,7 +35,7 @@ const CHECK_PASS: CheckResult = { errorCode: 0 }; -async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, { submissionStart, submissionEnd }: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { +async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: videoDetails, { submissionStart, submissionEnd }: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) { const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); const userName = row !== undefined ? row.userName : null; @@ -48,7 +48,7 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st "video": { "id": videoID, "title": youtubeData?.title, - "thumbnail": getMaxResThumbnail(youtubeData) || null, + "thumbnail": getMaxResThumbnail(videoID), "url": `https://www.youtube.com/watch?v=${videoID}`, }, "submission": { @@ -64,16 +64,13 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st }); } -async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) { - if (apiVideoInfo && service == Service.YouTube) { +async function sendWebhooks(apiVideoDetails: videoDetails, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) { + if (apiVideoDetails && service == Service.YouTube) { const userSubmissionCountRow = await db.prepare("get", `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]); - const { data, err } = apiVideoInfo; - if (err) return; - const startTime = parseFloat(segmentInfo.segment[0]); const endTime = parseFloat(segmentInfo.segment[1]); - sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, { + sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, apiVideoDetails, { submissionStart: startTime, submissionEnd: endTime, }, segmentInfo).catch(Logger.error); @@ -84,7 +81,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: axios.post(config.discordFirstTimeSubmissionsWebhookURL, { embeds: [{ - title: data?.title, + title: apiVideoDetails.title, url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`, description: `Submission ID: ${UUID}\ \n\nTimestamp: \ @@ -95,7 +92,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: name: userID, }, thumbnail: { - url: getMaxResThumbnail(data) || "", + url: getMaxResThumbnail(videoID), }, }], }) @@ -120,18 +117,10 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return // false for a pass - it was confusing and lead to this bug - any use of this function in // the future could have the same problem. -async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, +async function autoModerateSubmission(apiVideoDetails: videoDetails, submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) { - - const apiVideoDuration = (apiVideoInfo: APIVideoInfo) => { - if (!apiVideoInfo) return undefined; - const { err, data } = apiVideoInfo; - // return undefined if API error - if (err) return undefined; - return data?.lengthSeconds; - }; // get duration from API - const apiDuration = apiVideoDuration(apiVideoInfo); + const apiDuration = apiVideoDetails.duration; // if API fail or returns 0, get duration from client const duration = apiDuration || submission.videoDuration; // return false on undefined or 0 @@ -165,14 +154,6 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, return false; } -function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { - if (config.newLeafURLs !== null) { - return YouTubeAPI.listVideos(videoID, ignoreCache); - } else { - return null; - } -} - async function checkUserActiveWarning(userID: string): Promise { const MILLISECONDS_IN_HOUR = 3600000; const now = Date.now(); @@ -345,10 +326,10 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user return CHECK_PASS; } -async function checkByAutoModerator(videoID: any, userID: any, segments: Array, service:string, apiVideoInfo: APIVideoInfo, videoDuration: number): Promise { +async function checkByAutoModerator(videoID: any, userID: any, segments: Array, service:string, apiVideoDetails: videoDetails, videoDuration: number): Promise { // Auto moderator check if (service == Service.YouTube) { - const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service, videoDuration }); + const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { userID, videoID, segments, service, videoDuration }); if (autoModerateResult) { return { pass: false, @@ -377,12 +358,13 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic const videoDurationChanged = (videoDuration: number) => videoDuration != 0 && previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); - let apiVideoInfo: APIVideoInfo = null; + let apiVideoDetails: videoDetails = null; if (service == Service.YouTube) { // Don't use cache if we don't know the video duration, or the client claims that it has changed - apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam)); + const ignoreCache = !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam); + apiVideoDetails = await getVideoDetails(videoID, ignoreCache); } - const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration; + const apiVideoDuration = apiVideoDetails?.duration as VideoDuration; if (!videoDurationParam || (apiVideoDuration && Math.abs(videoDurationParam - apiVideoDuration) > 2)) { // If api duration is far off, take that one instead (it is only precise to seconds, not millis) videoDuration = apiVideoDuration || 0 as VideoDuration; @@ -400,7 +382,7 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic return { videoDuration, - apiVideoInfo, + apiVideoDetails, lockedCategoryList }; } @@ -501,10 +483,6 @@ export async function postSkipSegments(req: Request, res: Response): Promise { - return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null; -} - const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2); async function updateSegmentVideoDuration(UUID: SegmentUUID) { const { videoDuration, videoID, service } = await db.prepare("get", `select "videoDuration", "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]); - let apiVideoInfo: APIVideoInfo = null; + let apiVideoDetails: videoDetails = null; if (service == Service.YouTube) { // don't use cache since we have no information about the video length - apiVideoInfo = await getYouTubeVideoInfo(videoID); + apiVideoDetails = await getVideoDetails(videoID); } - const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration; + const apiVideoDuration = apiVideoDetails?.duration as VideoDuration; if (videoDurationChanged(videoDuration, apiVideoDuration)) { Logger.info(`Video duration changed for ${videoID} from ${videoDuration} to ${apiVideoDuration}`); await db.prepare("run", `UPDATE "sponsorTimes" SET "videoDuration" = ? WHERE "UUID" = ?`, [apiVideoDuration, UUID]); @@ -74,12 +70,12 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) { async function checkVideoDuration(UUID: SegmentUUID) { const { videoID, service } = await db.prepare("get", `select "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]); - let apiVideoInfo: APIVideoInfo = null; + let apiVideoDetails: videoDetails = null; if (service == Service.YouTube) { // don't use cache since we have no information about the video length - apiVideoInfo = await getYouTubeVideoInfo(videoID, true); + apiVideoDetails = await getVideoDetails(videoID, true); } - const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration; + const apiVideoDuration = apiVideoDetails?.duration as VideoDuration; // if no videoDuration return early if (isNaN(apiVideoDuration)) return; // fetch latest submission @@ -129,7 +125,8 @@ async function sendWebhooks(voteData: VoteData) { } if (config.newLeafURLs !== null) { - const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID); + const videoID = submissionInfoRow.videoID; + const { err, data } = await YouTubeAPI.listVideos(videoID); if (err) return; const isUpvote = voteData.incrementAmount > 0; @@ -141,8 +138,8 @@ async function sendWebhooks(voteData: VoteData) { "video": { "id": submissionInfoRow.videoID, "title": data?.title, - "url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}`, - "thumbnail": getMaxResThumbnail(data) || null, + "url": `https://www.youtube.com/watch?v=${videoID}`, + "thumbnail": getMaxResThumbnail(videoID), }, "submission": { "UUID": voteData.UUID, @@ -187,7 +184,7 @@ async function sendWebhooks(voteData: VoteData) { `${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`, }, "thumbnail": { - "url": getMaxResThumbnail(data) || "", + "url": getMaxResThumbnail(videoID), }, }], }) diff --git a/src/types/innerTubeApi.model.ts b/src/types/innerTubeApi.model.ts index 3fad700..28e46c8 100644 --- a/src/types/innerTubeApi.model.ts +++ b/src/types/innerTubeApi.model.ts @@ -19,5 +19,6 @@ export interface innerTubeVideoDetails { "author": string, "isPrivate": boolean, "isUnpluggedCorpus": boolean, - "isLiveContent": boolean + "isLiveContent": boolean, + "publishDate": string } \ No newline at end of file diff --git a/src/utils/getVideoDetails.ts b/src/utils/getVideoDetails.ts new file mode 100644 index 0000000..aa5636a --- /dev/null +++ b/src/utils/getVideoDetails.ts @@ -0,0 +1,59 @@ +import { config } from "../config"; +import { innerTubeVideoDetails } from "../types/innerTubeApi.model"; +import { APIVideoData } from "../types/youtubeApi.model"; +import { YouTubeAPI } from "../utils/youtubeApi"; +import { getPlayerData } from "../utils/innerTubeAPI"; + +export interface videoDetails { + videoId: string, + duration: number, + authorId: string, + authorName: string, + title: string, + published: number, + thumbnails: { + url: string, + width: number, + height: number, + }[] +} + +const convertFromInnerTube = (input: innerTubeVideoDetails): videoDetails => ({ + videoId: input.videoId, + duration: Number(input.lengthSeconds), + authorId: input.channelId, + authorName: input.author, + title: input.title, + published: new Date(input.publishDate).getTime()/1000, + thumbnails: input.thumbnail.thumbnails +}); + +const convertFromNewLeaf = (input: APIVideoData): videoDetails => ({ + videoId: input.videoId, + duration: input.lengthSeconds, + authorId: input.authorId, + authorName: input.author, + title: input.title, + published: input.published, + thumbnails: input.videoThumbnails +}); + +async function newLeafWrapper(videoId: string, ignoreCache: boolean) { + const result = await YouTubeAPI.listVideos(videoId, ignoreCache); + return result?.data ?? Promise.reject(); +} + +export function getVideoDetails(videoId: string, ignoreCache = false): Promise { + if (!config.newLeafURLs) { + return getPlayerData(videoId) + .then(data => convertFromInnerTube(data)); + } + return Promise.any([ + newLeafWrapper(videoId, ignoreCache) + .then(videoData => convertFromNewLeaf(videoData)), + getPlayerData(videoId) + .then(data => convertFromInnerTube(data)) + ]).catch(() => { + return null; + }); +} \ No newline at end of file diff --git a/src/utils/innerTubeAPI.ts b/src/utils/innerTubeAPI.ts index 4ff5797..c8f8696 100644 --- a/src/utils/innerTubeAPI.ts +++ b/src/utils/innerTubeAPI.ts @@ -21,9 +21,4 @@ export async function getPlayerData(videoID: string): Promise => - getPlayerData(videoID) - .then(pData => Number(pData.lengthSeconds)) - .catch(err => err); \ No newline at end of file +} \ No newline at end of file diff --git a/src/utils/isUserTempVIP.ts b/src/utils/isUserTempVIP.ts index a2758bc..44bc649 100644 --- a/src/utils/isUserTempVIP.ts +++ b/src/utils/isUserTempVIP.ts @@ -1,19 +1,13 @@ import redis from "../utils/redis"; import { tempVIPKey } from "../utils/redisKeys"; import { HashedUserID } from "../types/user.model"; -import { YouTubeAPI } from "../utils/youtubeApi"; -import { APIVideoInfo } from "../types/youtubeApi.model"; import { VideoID } from "../types/segments.model"; -import { config } from "../config"; import { Logger } from "./logger"; - -function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise { - return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null; -} +import { getVideoDetails } from "./getVideoDetails"; export const isUserTempVIP = async (hashedUserID: HashedUserID, videoID: VideoID): Promise => { - const apiVideoInfo = await getYouTubeVideoInfo(videoID); - const channelID = apiVideoInfo?.data?.authorId; + const apiVideoDetails = await getVideoDetails(videoID); + const channelID = apiVideoDetails?.authorId; try { const reply = await redis.get(tempVIPKey(hashedUserID)); return reply && reply == channelID; diff --git a/src/utils/youtubeApi.ts b/src/utils/youtubeApi.ts index 6580f5e..a227d71 100644 --- a/src/utils/youtubeApi.ts +++ b/src/utils/youtubeApi.ts @@ -52,6 +52,5 @@ export class YouTubeAPI { } } -export function getMaxResThumbnail(apiInfo: APIVideoData): string | void { - return apiInfo?.videoThumbnails?.find((elem) => elem.quality === "maxres")?.second__originalUrl; -} \ No newline at end of file +export const getMaxResThumbnail = (videoID: string): string => + `https://i.ytimg.com/vi/${videoID}/maxresdefault.jpg`; \ No newline at end of file diff --git a/test/cases/getChapterNames.ts b/test/cases/getChapterNames.ts index 0905b47..c73c923 100644 --- a/test/cases/getChapterNames.ts +++ b/test/cases/getChapterNames.ts @@ -19,9 +19,9 @@ if (db instanceof Postgres) { await db.prepare("run", query, [chapterNamesVid1, 70, 75, 2, 0, "chapterNamesVid-2", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "A different one"]); await db.prepare("run", query, [chapterNamesVid1, 71, 76, 2, 0, "chapterNamesVid-3", "testman", 0, 50, "chapter", "chapter", "YouTube", 0, 0, 0, "Something else"]); - await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published", "genreUrl") - SELECT ?, ?, ?, ?, ?`, [ - chapterNamesVid1, chapterChannelID, "", 0, "" + await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published") + SELECT ?, ?, ?, ?`, [ + chapterNamesVid1, chapterChannelID, "", 0 ]); }); diff --git a/test/cases/innerTubeApi.ts b/test/cases/innerTubeApi.ts index 6ef3eb1..cc9cb2d 100644 --- a/test/cases/innerTubeApi.ts +++ b/test/cases/innerTubeApi.ts @@ -4,10 +4,10 @@ import assert from "assert"; import { YouTubeAPI } from "../../src/utils/youtubeApi"; import * as innerTube from "../../src/utils/innerTubeAPI"; import { partialDeepEquals } from "../utils/partialDeepEquals"; - +import { getVideoDetails } from "../../src/utils/getVideoDetails"; const videoID = "dQw4w9WgXcQ"; -const expected = { // partial type of innerTubeVideoDetails +const expectedInnerTube = { // partial type of innerTubeVideoDetails videoId: videoID, title: "Rick Astley - Never Gonna Give You Up (Official Music Video)", lengthSeconds: "212", @@ -25,17 +25,12 @@ const currentViews = 1284257550; describe("innertube API test", function() { it("should be able to get innerTube details", async () => { const result = await innerTube.getPlayerData(videoID); - assert.ok(partialDeepEquals(result, expected)); + assert.ok(partialDeepEquals(result, expectedInnerTube)); }); it("Should have more views than current", async () => { const result = await innerTube.getPlayerData(videoID); assert.ok(Number(result.viewCount) >= currentViews); }); - it("Should have the same video duration from both endpoints", async () => { - const playerData = await innerTube.getPlayerData(videoID); - const length = await innerTube.getLength(videoID); - assert.equal(Number(playerData.lengthSeconds), length); - }); it("Should have equivalent response from NewLeaf", async function () { if (!config.newLeafURLs || config.newLeafURLs.length <= 0 || config.newLeafURLs[0] == "placeholder") this.skip(); const itResponse = await innerTube.getPlayerData(videoID); @@ -48,4 +43,8 @@ describe("innertube API test", function() { // validate authorId assert.strictEqual(itResponse.channelId, newLeafResponse.data?.authorId); }); + it("Should return data from generic endpoint", async function () { + const videoDetail = await getVideoDetails(videoID); + assert.ok(videoDetail); + }); }); \ No newline at end of file diff --git a/test/cases/postSkipSegments.ts b/test/cases/postSkipSegments.ts index 1369282..17729dd 100644 --- a/test/cases/postSkipSegments.ts +++ b/test/cases/postSkipSegments.ts @@ -141,7 +141,6 @@ describe("postSkipSegments", () => { title: "Example Title", channelID: "ExampleChannel", published: 123, - genreUrl: "" }; assert.ok(partialDeepEquals(videoInfo, expectedVideoInfo)); diff --git a/test/youtubeMock.ts b/test/youtubeMock.ts index f0b89d1..bb489af 100644 --- a/test/youtubeMock.ts +++ b/test/youtubeMock.ts @@ -15,7 +15,7 @@ export class YouTubeApiMock { if (obj.id === "noDuration" || obj.id === "full_video_duration_segment") { return { - err: null, + err: false, data: { title: "Example Title", lengthSeconds: 0, @@ -32,7 +32,7 @@ export class YouTubeApiMock { }; } else if (obj.id === "duration-update") { return { - err: null, + err: false, data: { title: "Example Title", lengthSeconds: 500, @@ -49,7 +49,7 @@ export class YouTubeApiMock { }; } else if (obj.id === "channelid-convert") { return { - err: null, + err: false, data: { title: "Video Lookup Title", author: "ChannelAuthor", @@ -58,14 +58,14 @@ export class YouTubeApiMock { }; } else if (obj.id === "duration-changed") { return { - err: null, + err: false, data: { lengthSeconds: 100, } as APIVideoData }; } else { return { - err: null, + err: false, data: { title: "Example Title", authorId: "ExampleChannel", diff --git a/tsconfig.json b/tsconfig.json index c38f013..cb90fb2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,7 @@ /* Basic Options */ // "incremental": true, /* Enable incremental compilation */ - "target": "es2016", + "target": "ES2021", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */