Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-09 21:17:15 +03:00
Code Issues Packages Projects Releases Wiki Activity

Only use read replica for shorter queries

Browse Source 
Help with https://github.com/ajayyy/SponsorBlockServer/issues/487
This commit is contained in:
Ajay
2022-07-20 00:40:07 -04:00
parent f4b66d30ec
commit 1e441c3ebf
4 changed files with 28 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/databases/IDatabase.ts
View File

@@ -1,7 +1,11 @@
export interface QueryOption {
useReplica?: boolean;
}
export interface IDatabase { export interface IDatabase {
init(): Promise<void>; init(): Promise<void>;
prepare(type: QueryType, query: string, params?: any[]): Promise<any | any[] | void>; prepare(type: QueryType, query: string, params?: any[], options?: QueryOption): Promise<any | any[] | void>;
} }
export type QueryType = "get" | "all" | "run"; export type QueryType = "get" | "all" | "run";

10
src/databases/Postgres.ts
View File

@@ -1,5 +1,5 @@
import { Logger } from "../utils/logger"; import { Logger } from "../utils/logger";
import { IDatabase, QueryType } from "./IDatabase"; import { IDatabase, QueryOption, QueryType } from "./IDatabase";
import { Client, Pool, PoolClient, types } from "pg"; import { Client, Pool, PoolClient, types } from "pg";
import fs from "fs"; import fs from "fs";
@@ -82,7 +82,7 @@ export class Postgres implements IDatabase {
} }
} }
async prepare(type: QueryType, query: string, params?: any[]): Promise<any[]> { async prepare(type: QueryType, query: string, params?: any[], options: QueryOption = {}): Promise<any[]> {
// Convert query to use numbered parameters // Convert query to use numbered parameters
let count = 1; let count = 1;
for (let char = 0; char < query.length; char++) { for (let char = 0; char < query.length; char++) {
@@ -95,7 +95,7 @@ export class Postgres implements IDatabase {
Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`); Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
try { try {
const queryResult = await this.getPool(type).query({ text: query, values: params }); const queryResult = await this.getPool(type, options).query({ text: query, values: params });
switch (type) { switch (type) {
case "get": { case "get": {
@@ -117,8 +117,8 @@ export class Postgres implements IDatabase {
} }
} }
private getPool(type: string): Pool { private getPool(type: string, options: QueryOption): Pool {
const readAvailable = this.poolRead && (type === "get" || type === "all"); const readAvailable = this.poolRead && options.useReplica && (type === "get" || type === "all");
const ignroreReadDueToFailure = this.lastPoolReadFail > Date.now() - 1000 * 30; const ignroreReadDueToFailure = this.lastPoolReadFail > Date.now() - 1000 * 30;
const readDueToFailure = this.lastPoolFail > Date.now() - 1000 * 30; const readDueToFailure = this.lastPoolFail > Date.now() - 1000 * 30;
if (readAvailable && !ignroreReadDueToFailure && (readDueToFailure || if (readAvailable && !ignroreReadDueToFailure && (readDueToFailure ||

8
src/routes/getSkipSegments.ts
View File

@@ -33,7 +33,7 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, service:
const service = getService(req?.query?.service as string); const service = getService(req?.query?.service as string);
const fetchData = () => privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ? AND "service" = ?', const fetchData = () => privateDB.prepare("all", 'SELECT "hashedIP" FROM "sponsorTimes" WHERE "videoID" = ? AND "timeSubmitted" = ? AND "service" = ?',
[videoID, segment.timeSubmitted, service]) as Promise<{ hashedIP: HashedIP }[]>; [videoID, segment.timeSubmitted, service], { useReplica: true }) as Promise<{ hashedIP: HashedIP }[]>;
cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await QueryCacher.get(fetchData, shadowHiddenIPKey(videoID, segment.timeSubmitted, service)); cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted] = await QueryCacher.get(fetchData, shadowHiddenIPKey(videoID, segment.timeSubmitted, service));
} }
@@ -171,7 +171,8 @@ async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service
"all", "all",
`SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description" FROM "sponsorTimes" `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description" FROM "sponsorTimes"
WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[`${hashedVideoIDPrefix}%`, service] [`${hashedVideoIDPrefix}%`, service],
{ useReplica: true }
) as Promise<DBSegment[]>; ) as Promise<DBSegment[]>;
if (hashedVideoIDPrefix.length === 4) { if (hashedVideoIDPrefix.length === 4) {
@@ -187,7 +188,8 @@ async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): P
"all", "all",
`SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "timeSubmitted", "description" FROM "sponsorTimes" `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "reputation", "shadowHidden", "timeSubmitted", "description" FROM "sponsorTimes"
WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`, WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 ORDER BY "startTime"`,
[videoID, service] [videoID, service],
{ useReplica: true }
) as Promise<DBSegment[]>; ) as Promise<DBSegment[]>;
return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service)); return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service));

24
src/routes/voteOnSponsorTime.ts
View File

@@ -211,7 +211,7 @@ async function sendWebhooks(voteData: VoteData) {
async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isTempVIP: boolean, isOwnSubmission: boolean, category: Category async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, isTempVIP: boolean, isOwnSubmission: boolean, category: Category
, hashedIP: HashedIP, finalResponse: FinalResponse): Promise<{ status: number, message?: string }> { , hashedIP: HashedIP, finalResponse: FinalResponse): Promise<{ status: number, message?: string }> {
// Check if they've already made a vote // Check if they've already made a vote
const usersLastVoteInfo = await privateDB.prepare("get", `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID]); const usersLastVoteInfo = await privateDB.prepare("get", `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID], { useReplica: true });
if (usersLastVoteInfo?.category === category) { if (usersLastVoteInfo?.category === category) {
// Double vote, ignore // Double vote, ignore
@@ -219,7 +219,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
} }
const segmentInfo = (await db.prepare("get", `SELECT "category", "actionType", "videoID", "hashedVideoID", "service", "userID", "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, const segmentInfo = (await db.prepare("get", `SELECT "category", "actionType", "videoID", "hashedVideoID", "service", "userID", "locked" FROM "sponsorTimes" WHERE "UUID" = ?`,
[UUID])) as {category: Category, actionType: ActionType, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number}; [UUID], { useReplica: true })) as {category: Category, actionType: ActionType, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number};
if (segmentInfo.actionType === ActionType.Full) { if (segmentInfo.actionType === ActionType.Full) {
return { status: 400, message: "Not allowed to change category of a full video segment" }; return { status: 400, message: "Not allowed to change category of a full video segment" };
@@ -232,7 +232,7 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
} }
// Ignore vote if the next category is locked // Ignore vote if the next category is locked
const nextCategoryLocked = await db.prepare("get", `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, [segmentInfo.videoID, segmentInfo.service, category]); const nextCategoryLocked = await db.prepare("get", `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, [segmentInfo.videoID, segmentInfo.service, category], { useReplica: true });
if (nextCategoryLocked && !isVIP) { if (nextCategoryLocked && !isVIP) {
return { status: 200 }; return { status: 200 };
} }
@@ -242,13 +242,13 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
return { status: 200 }; return { status: 200 };
} }
const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category]); const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category], { useReplica: true });
const timeSubmitted = Date.now(); const timeSubmitted = Date.now();
const voteAmount = (isVIP || isTempVIP) ? 500 : 1; const voteAmount = (isVIP || isTempVIP) ? 500 : 1;
const ableToVote = finalResponse.finalStatus === 200 const ableToVote = finalResponse.finalStatus === 200
&& (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID])) === undefined; && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [userID], { useReplica: true })) === undefined;
if (ableToVote) { if (ableToVote) {
// Add the vote // Add the vote
@@ -271,9 +271,9 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
} }
// See if the submissions category is ready to change // See if the submissions category is ready to change
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, segmentInfo.category]); const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, segmentInfo.category], { useReplica: true });
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID], { useReplica: true });
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
const startingVotes = isSubmissionVIP ? 10000 : 1; const startingVotes = isSubmissionVIP ? 10000 : 1;
@@ -391,7 +391,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
const isSegmentLocked = segmentInfo.locked; const isSegmentLocked = segmentInfo.locked;
const isVideoLocked = async () => !!(await db.prepare("get", `SELECT "category" FROM "lockCategories" WHERE const isVideoLocked = async () => !!(await db.prepare("get", `SELECT "category" FROM "lockCategories" WHERE
"videoID" = ? AND "service" = ? AND "category" = ? AND "actionType" = ?`, "videoID" = ? AND "service" = ? AND "category" = ? AND "actionType" = ?`,
[segmentInfo.videoID, segmentInfo.service, segmentInfo.category, segmentInfo.actionType])); [segmentInfo.videoID, segmentInfo.service, segmentInfo.category, segmentInfo.actionType], { useReplica: true }));
if (isSegmentLocked || await isVideoLocked()) { if (isSegmentLocked || await isVideoLocked()) {
finalResponse.blockVote = true; finalResponse.blockVote = true;
finalResponse.webhookType = VoteWebhookType.Rejected; finalResponse.webhookType = VoteWebhookType.Rejected;
@@ -420,7 +420,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
try { try {
// check if vote has already happened // check if vote has already happened
const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID]); const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID], { useReplica: true });
// -1 for downvote, 1 for upvote. Maybe more depending on reputation in the future // -1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
// oldIncrementAmount will be zero if row is null // oldIncrementAmount will be zero if row is null
@@ -478,9 +478,9 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID
&& !(originalType === VoteType.Malicious && segmentInfo.actionType !== ActionType.Chapter) && !(originalType === VoteType.Malicious && segmentInfo.actionType !== ActionType.Chapter)
&& !finalResponse.blockVote && !finalResponse.blockVote
&& finalResponse.finalStatus === 200 && finalResponse.finalStatus === 200
&& (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID]), { useReplica: true }) !== undefined
&& (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined && (await db.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID]), { useReplica: true }) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined); && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID]), { useReplica: true }) === undefined);
const ableToVote = isVIP || isTempVIP || userAbleToVote; const ableToVote = isVIP || isTempVIP || userAbleToVote;