From 1e441c3ebf9b7337c54506338ed43f9a0a25a992 Mon Sep 17 00:00:00 2001 From: Ajay Date: Wed, 20 Jul 2022 00:40:07 -0400 Subject: [PATCH] Only use read replica for shorter queries Help with https://github.com/ajayyy/SponsorBlockServer/issues/487 --- src/databases/IDatabase.ts | 8 ++++++-- src/databases/Postgres.ts | 10 +++++----- src/routes/getSkipSegments.ts | 8 +++++--- src/routes/voteOnSponsorTime.ts | 24 ++++++++++++------------ 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/databases/IDatabase.ts b/src/databases/IDatabase.ts index 86774f4..0211203 100644 --- a/src/databases/IDatabase.ts +++ b/src/databases/IDatabase.ts @@ -1,7 +1,11 @@ +export interface QueryOption { + useReplica?: boolean; +} + export interface IDatabase { init(): Promise; - prepare(type: QueryType, query: string, params?: any[]): Promise; + prepare(type: QueryType, query: string, params?: any[], options?: QueryOption): Promise; } -export type QueryType = "get" | "all" | "run"; +export type QueryType = "get" | "all" | "run"; \ No newline at end of file diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts index 5cf1f7e..cd8efdf 100644 --- a/src/databases/Postgres.ts +++ b/src/databases/Postgres.ts @@ -1,5 +1,5 @@ import { Logger } from "../utils/logger"; -import { IDatabase, QueryType } from "./IDatabase"; +import { IDatabase, QueryOption, QueryType } from "./IDatabase"; import { Client, Pool, PoolClient, types } from "pg"; import fs from "fs"; @@ -82,7 +82,7 @@ export class Postgres implements IDatabase { } } - async prepare(type: QueryType, query: string, params?: any[]): Promise { + async prepare(type: QueryType, query: string, params?: any[], options: QueryOption = {}): Promise { // Convert query to use numbered parameters let count = 1; for (let char = 0; char < query.length; char++) { @@ -95,7 +95,7 @@ export class Postgres implements IDatabase { Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); try { - const queryResult = await this.getPool(type).query({ text: query, values: params }); + const queryResult = await this.getPool(type, options).query({ text: query, values: params }); switch (type) { case "get": { @@ -117,8 +117,8 @@ export class Postgres implements IDatabase { } } - private getPool(type: string): Pool { - const readAvailable = this.poolRead && (type === "get" || type === "all"); + private getPool(type: string, options: QueryOption): Pool { + const readAvailable = this.poolRead && options.useReplica && (type === "get" || type === "all"); const ignroreReadDueToFailure = this.lastPoolReadFail > Date.now() - 1000 * 30; const readDueToFailure = this.lastPoolFail > Date.now() - 1000 * 30; if (readAvailable && !ignroreReadDueToFailure && (readDueToFailure || diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index 8d623e7..40fbf38 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -33,7 +33,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, service: const service = getService(req?.query?.service as string); const fetchData = () => privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ? AND "service" = ?', - [videoID, segment.timeSubmitted, service]) as Promise<{ hashedIP: HashedIP }[]>; + [videoID, segment.timeSubmitted, service], { useReplica: true }) as Promise<{ hashedIP: HashedIP }[]>; cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await QueryCacher.get(fetchData, shadowHiddenIPKey(videoID, segment.timeSubmitted, service)); } @@ -171,7 +171,8 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service "all", `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description" FROM "sponsorTimes" WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, - [`${hashedVideoIDPrefix}%`, service] + [`${hashedVideoIDPrefix}%`, service], + { useReplica: true } ) as Promise; if (hashedVideoIDPrefix.length === 4) { @@ -187,7 +188,8 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P "all", `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "timeSubmitted", "description" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, - [videoID, service] + [videoID, service], + { useReplica: true } ) as Promise; return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service)); diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 7f308ef..c30e976 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -211,7 +211,7 @@ async function sendWebhooks(voteData: VoteData) { async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isTempVIP: boolean, isOwnSubmission: boolean, category: Category , hashedIP: HashedIP, finalResponse: FinalResponse): Promise<{ status: number, message?: string }> { // Check if they've already made a vote - const usersLastVoteInfo = await privateDB.prepare("get", `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]); + const usersLastVoteInfo = await privateDB.prepare("get", `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID], { useReplica: true }); if (usersLastVoteInfo?.category === category) { // Double vote, ignore @@ -219,7 +219,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i } const segmentInfo = (await db.prepare("get", `SELECT "category", "actionType", "videoID", "hashedVideoID", "service", "userID", "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, - [UUID])) as {category: Category, actionType: ActionType, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number}; + [UUID], { useReplica: true })) as {category: Category, actionType: ActionType, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number}; if (segmentInfo.actionType === ActionType.Full) { return { status: 400, message: "Not allowed to change category of a full video segment" }; @@ -232,7 +232,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i } // Ignore vote if the next category is locked - const nextCategoryLocked = await db.prepare("get", `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, [segmentInfo.videoID, segmentInfo.service, category]); + const nextCategoryLocked = await db.prepare("get", `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, [segmentInfo.videoID, segmentInfo.service, category], { useReplica: true }); if (nextCategoryLocked && !isVIP) { return { status: 200 }; } @@ -242,13 +242,13 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i return { status: 200 }; } - const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]); + const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category], { useReplica: true }); const timeSubmitted = Date.now(); const voteAmount = (isVIP || isTempVIP) ? 500 : 1; const ableToVote = finalResponse.finalStatus === 200 - && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID])) === undefined; + && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID], { useReplica: true })) === undefined; if (ableToVote) { // Add the vote @@ -271,9 +271,9 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i } // See if the submissions category is ready to change - const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, segmentInfo.category]); + const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, segmentInfo.category], { useReplica: true }); - const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); + const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID], { useReplica: true }); const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); const startingVotes = isSubmissionVIP ? 10000 : 1; @@ -391,7 +391,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID const isSegmentLocked = segmentInfo.locked; const isVideoLocked = async () => !!(await db.prepare("get", `SELECT "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ? AND "actionType" = ?`, - [segmentInfo.videoID, segmentInfo.service, segmentInfo.category, segmentInfo.actionType])); + [segmentInfo.videoID, segmentInfo.service, segmentInfo.category, segmentInfo.actionType], { useReplica: true })); if (isSegmentLocked || await isVideoLocked()) { finalResponse.blockVote = true; finalResponse.webhookType = VoteWebhookType.Rejected; @@ -420,7 +420,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID try { // check if vote has already happened - const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID]); + const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID], { useReplica: true }); // -1 for downvote, 1 for upvote. Maybe more depending on reputation in the future // oldIncrementAmount will be zero if row is null @@ -478,9 +478,9 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID && !(originalType === VoteType.Malicious && segmentInfo.actionType !== ActionType.Chapter) && !finalResponse.blockVote && finalResponse.finalStatus === 200 - && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined - && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined - && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined); + && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID]), { useReplica: true }) !== undefined + && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID]), { useReplica: true }) === undefined + && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID]), { useReplica: true }) === undefined); const ableToVote = isVIP || isTempVIP || userAbleToVote;