Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-11 05:57:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #389 from mchangrh/replication-mirror

Browse Source 
changes for mirror mode
This commit is contained in:
Ajay Ramachandran
2021-10-26 20:34:22 -04:00
committed by GitHub
parent 3877cd580c 6be034362b
commit eecb238e18
7 changed files with 85 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
databases/_sponsorTimes.db.sql
View File

@@ -38,5 +38,6 @@ CREATE TABLE IF NOT EXISTS "config" (
);
CREATE EXTENSION IF NOT EXISTS pgcrypto; --!sqlite-ignore
CREATE EXTENSION IF NOT EXISTS pg_trgm; --!sqlite-ignore
COMMIT;

10
src/databases/databases.ts
View File

@@ -4,7 +4,6 @@ import { Mysql } from "./Mysql";
import { Postgres } from "./Postgres";
import { IDatabase } from "./IDatabase";
let db: IDatabase;
let privateDB: IDatabase;
if (config.mysql) {
@@ -68,6 +67,15 @@ async function initDb(): Promise<void> {
// Attach private db to main db
(db as Sqlite).attachDatabase(config.privateDB, "privateDB");
}
if (config.mode === "mirror" && db instanceof Postgres) {
const tables = config?.dumpDatabase?.tables ?? [];
const tableNames = tables.map(table => table.name);
for (const table of tableNames) {
const filePath = `${config?.dumpDatabase?.postgresExportPath}/${table}.csv`;
await db.prepare("run", `COPY "${table}" FROM '${filePath}' WITH (FORMAT CSV, HEADER true);`);
}
}
}
export {

3
src/index.ts
View File

@@ -12,6 +12,9 @@ async function init() {
});
await initDb();
// edge case clause for creating compatible .db files, do not enable
if (config.mode === "init-db-and-exit") process.exit(0);
// do not enable init-db-only mode for usage.
(global as any).HEADCOMMIT = config.mode === "development" ? "development"
: config.mode === "test" ? "test"
: getCommit() as string;

2
src/routes/dumpDatabase.ts
View File

@@ -184,7 +184,7 @@ export async function redirectLink(req: Request, res: Response): Promise<void> {
res.sendStatus(404);
}
await queueDump();
if (req.query.generate !== "false") await queueDump();
}
function updateQueueTime(): void {

29
test/cases/getLockCategories.ts
View File

@@ -2,6 +2,7 @@ import { getHash } from "../../src/utils/getHash";
import { db } from "../../src/databases/databases";
import assert from "assert";
import { client } from "../utils/httpClient";
import { mixedDeepEquals } from "../utils/partialDeepEquals";
const endpoint = "/api/lockCategories";
const getLockCategories = (videoID: string) => client.get(endpoint, { params: { videoID } });
const getLockCategoriesWithService = (videoID: string, service: string) => client.get(endpoint, { params: { videoID, service } });
@@ -12,13 +13,13 @@ describe("getLockCategories", () => {
await db.prepare("run", insertVipUserQuery, [getHash("getLockCategoriesVIP")]);
const insertLockCategoryQuery = 'INSERT INTO "lockCategories" ("userID", "videoID", "category", "reason", "service") VALUES (?, ?, ?, ?, ?)';
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "sponsor", "1-short", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock1", "interaction", "1-longer-reason", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "sponsor", "1-short", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory1", "interaction", "1-longer-reason", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock2", "preview", "2-reason", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory2", "preview", "2-reason", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "nonmusic", "3-reason", "PeerTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLock3", "sponsor", "3-reason", "YouTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "nonmusic", "3-reason", "PeerTube"]);
await db.prepare("run", insertLockCategoryQuery, [getHash("getLockCategoriesVIP"), "getLockCategory3", "sponsor", "3-reason", "YouTube"]);
});
it("Should update the database version when starting the application", async () => {
@@ -27,7 +28,7 @@ describe("getLockCategories", () => {
});
it("Should be able to get multiple locks", (done) => {
getLockCategories("getLock1")
getLockCategories("getLockCategory1")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
@@ -37,14 +38,14 @@ describe("getLockCategories", () => {
],
reason: "1-longer-reason"
};
assert.deepStrictEqual(res.data, expected);
assert.ok(mixedDeepEquals(res.data, expected));
done();
})
.catch(err => done(err));
});
it("Should be able to get single locks", (done) => {
getLockCategories("getLock2")
getLockCategories("getLockCategory2")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
@@ -60,7 +61,7 @@ describe("getLockCategories", () => {
});
it("should return 404 if no lock exists", (done) => {
getLockCategories("getLockNull")
getLockCategories("getLockCategoryNull")
.then(res => {
assert.strictEqual(res.status, 404);
done();
@@ -78,7 +79,7 @@ describe("getLockCategories", () => {
});
it("Should be able to get multiple locks with service", (done) => {
getLockCategoriesWithService("getLock1", "YouTube")
getLockCategoriesWithService("getLockCategory1", "YouTube")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
@@ -88,14 +89,14 @@ describe("getLockCategories", () => {
],
reason: "1-longer-reason"
};
assert.deepStrictEqual(res.data, expected);
assert.ok(mixedDeepEquals(res.data, expected));
done();
})
.catch(err => done(err));
});
it("Should be able to get single locks with service", (done) => {
getLockCategoriesWithService("getLock3", "PeerTube")
getLockCategoriesWithService("getLockCategory3", "PeerTube")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
@@ -111,7 +112,7 @@ describe("getLockCategories", () => {
});
it("Should be able to get single locks with service", (done) => {
getLockCategoriesWithService("getLock3", "Youtube")
getLockCategoriesWithService("getLockCategory3", "Youtube")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {
@@ -127,7 +128,7 @@ describe("getLockCategories", () => {
});
it("should return result from Youtube service if service not match", (done) => {
getLockCategoriesWithService("getLock3", "Dailymotion")
getLockCategoriesWithService("getLockCategory3", "Dailymotion")
.then(res => {
assert.strictEqual(res.status, 200);
const expected = {

48
test/cases/postSkipSegments.ts
View File

@@ -1,6 +1,6 @@
import { config } from "../../src/config";
import { getHash } from "../../src/utils/getHash";
import { partialDeepEquals } from "../utils/partialDeepEquals";
import { partialDeepEquals, arrayDeepEquals } from "../utils/partialDeepEquals";
import { db } from "../../src/databases/databases";
import { ImportMock } from "ts-mock-imports";
import * as YouTubeAPIModule from "../../src/utils/youtubeApi";
@@ -29,7 +29,7 @@ describe("postSkipSegments", () => {
const submitUserOneHash = getHash(submitUserOne);
const submitVIPuser = `VIPPostSkipUser${".".repeat(16)}`;
const warnVideoID = "dQw4w9WgXcF";
const warnVideoID = "postSkip2";
const badInputVideoID = "dQw4w9WgXcQ";
const queryDatabase = (videoID: string) => db.prepare("get", `SELECT "startTime", "endTime", "locked", "category" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
@@ -91,7 +91,7 @@ describe("postSkipSegments", () => {
});
it("Should be able to submit a single time (Params method)", (done) => {
const videoID = "dQw4w9WgXcR";
const videoID = "postSkip1";
postSkipSegmentParam({
videoID,
startTime: 2,
@@ -125,7 +125,7 @@ describe("postSkipSegments", () => {
});
it("Should be able to submit a single time (JSON method)", (done) => {
const videoID = "dQw4w9WgXcF";
const videoID = "postSkip2";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -150,7 +150,7 @@ describe("postSkipSegments", () => {
});
it("Should be able to submit a single time with an action type (JSON method)", (done) => {
const videoID = "dQw4w9WgXcV";
const videoID = "postSkip3";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -176,7 +176,7 @@ describe("postSkipSegments", () => {
});
it("Should not be able to submit an intro with mute action type (JSON method)", (done) => {
const videoID = "dQw4w9WgXpQ";
const videoID = "postSkip4";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -196,7 +196,7 @@ describe("postSkipSegments", () => {
});
it("Should be able to submit a single time with a duration from the YouTube API (JSON method)", (done) => {
const videoID = "dQw4w9WgXZX";
const videoID = "postSkip5";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -222,7 +222,7 @@ describe("postSkipSegments", () => {
});
it("Should be able to submit a single time with a precise duration close to the one from the YouTube API (JSON method)", (done) => {
const videoID = "dQw4w9WgXZH";
const videoID = "postSkip6";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -331,7 +331,7 @@ describe("postSkipSegments", () => {
});
it("Should be able to submit a single time under a different service (JSON method)", (done) => {
const videoID = "dQw4w9WgXcG";
const videoID = "postSkip7";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -383,7 +383,7 @@ describe("postSkipSegments", () => {
});
it("Should be able to submit multiple times (JSON method)", (done) => {
const videoID = "dQw4w9WgXcT";
const videoID = "postSkip11";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -407,14 +407,14 @@ describe("postSkipSegments", () => {
endTime: 60,
category: "intro"
}];
assert.deepStrictEqual(rows, expected);
assert.ok(arrayDeepEquals(rows, expected));
done();
})
.catch(err => done(err));
}).timeout(5000);
it("Should allow multiple times if total is under 80% of video(JSON method)", (done) => {
const videoID = "L_jWHffIx5E";
const videoID = "postSkip9";
postSkipSegmentJSON({
userID: submitUserOne,
videoID,
@@ -452,7 +452,7 @@ describe("postSkipSegments", () => {
endTime: 170,
category: "sponsor"
}];
assert.deepStrictEqual(rows, expected);
assert.ok(arrayDeepEquals(rows, expected));
done();
})
.catch(err => done(err));
@@ -505,20 +505,20 @@ describe("postSkipSegments", () => {
.then(async res => {
assert.strictEqual(res.status, 403);
const expected = [{
category: "sponsor",
startTime: 2000,
endTime: 4000
category: "interaction",
startTime: 0,
endTime: 1000
}, {
category: "sponsor",
startTime: 1500,
endTime: 2750
category: "interaction",
startTime: 1001,
endTime: 1005
}, {
category: "sponsor",
startTime: 4050,
endTime: 4750
category: "interaction",
startTime: 0,
endTime: 5000
}];
const rows = await queryDatabase(videoID);
assert.notDeepStrictEqual(rows, expected);
const rows = await db.prepare("all", `SELECT "category", "startTime", "endTime" FROM "sponsorTimes" WHERE "videoID" = ?`, [videoID]);
assert.ok(arrayDeepEquals(rows, expected));
done();
})
.catch(err => done(err));

33
test/utils/partialDeepEquals.ts
View File

@@ -21,4 +21,35 @@ export const partialDeepEquals = (actual: Record<string, any>, expected: Record<
}
}
return true;
};
};
export const arrayDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
if (actual.length !== expected.length) return false;
let flag = true;
const actualString = JSON.stringify(actual);
const expectedString = JSON.stringify(expected);
// check every value in arr1 for match in arr2
actual.every((value: any) => { if (flag && !expectedString.includes(JSON.stringify(value))) flag = false; });
// check arr2 for match in arr1
expected.every((value: any) => { if (flag && !actualString.includes(JSON.stringify(value))) flag = false; });
if (!flag && print) printActualExpected(actual, expected);
return flag;
};
export const mixedDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
for (const [ key, value ] of Object.entries(expected)) {
// if value is object or array, recurse
if (Array.isArray(value)) {
if (!arrayDeepEquals(actual?.[key], value, false)) {
if (print) printActualExpected(actual, expected);
return false;
}
}
else if (actual?.[key] !== value) {
if (print) printActualExpected(actual, expected);
return false;
}
}
return true;
};