From 733bd338e71df12560e9de999b411eedc19338a5 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 2 Dec 2021 01:10:09 -0500 Subject: [PATCH 1/5] Add bulk rating fetching --- src/routes/ratings/getRating.ts | 37 +++++++++++++++++-------- src/utils/queryCacher.ts | 49 +++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/routes/ratings/getRating.ts b/src/routes/ratings/getRating.ts index ec7e9f0..edaae8e 100644 --- a/src/routes/ratings/getRating.ts +++ b/src/routes/ratings/getRating.ts @@ -17,11 +17,24 @@ interface DBRating { } export async function getRating(req: Request, res: Response): Promise { - let hashPrefix = req.params.prefix as VideoIDHash; - if (!hashPrefix || !hashPrefixTester(hashPrefix)) { + let hashPrefixes: VideoIDHash[] = []; + try { + hashPrefixes = req.query.hashPrefixes + ? JSON.parse(req.query.hashPrefixes as string) + : Array.isArray(req.query.prefix) + ? req.query.prefix + : [req.query.prefix ?? req.params.prefix]; + if (!Array.isArray(hashPrefixes)) { + return res.status(400).send("hashPrefixes parameter does not match format requirements."); + } + + hashPrefixes.map((hashPrefix) => hashPrefix?.toLowerCase()); + } catch(error) { + return res.status(400).send("Bad parameter: hashPrefixes (invalid JSON)"); + } + if (hashPrefixes.some((hashPrefix) => !hashPrefix || !hashPrefixTester(hashPrefix))) { return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix } - hashPrefix = hashPrefix.toLowerCase() as VideoIDHash; let types: RatingType[] = []; try { @@ -44,7 +57,7 @@ export async function getRating(req: Request, res: Response): Promise const service: Service = getService(req.query.service, req.body.service); try { - const ratings = (await getRatings(hashPrefix, service)) + const ratings = (await getRatings(hashPrefixes, service)) .filter((rating) => types.includes(rating.type)) .map((rating) => ({ videoID: rating.videoID, @@ -53,23 +66,25 @@ export async function getRating(req: Request, res: Response): Promise type: rating.type, count: rating.count })); + return res.status((ratings.length) ? 200 : 404) .send(ratings ?? []); } catch (err) { Logger.error(err as string); + return res.sendStatus(500); } } -function getRatings(hashPrefix: VideoIDHash, service: Service): Promise { - const fetchFromDB = () => db +function getRatings(hashPrefixes: VideoIDHash[], service: Service): Promise { + const fetchFromDB = (hashPrefixes: VideoIDHash[]) => db .prepare( "all", - `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "hashedVideoID"`, - [`${hashPrefix}%`, service] + `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" ~* ? AND "service" = ? ORDER BY "hashedVideoID"`, + [`^(?:${hashPrefixes.join("|")})`, service] ) as Promise; - return (hashPrefix.length === 4) - ? QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service)) - : fetchFromDB(); + return (hashPrefixes.every((hashPrefix) => hashPrefix.length === 4)) + ? QueryCacher.getAndSplit(fetchFromDB, (prefix) => ratingHashKey(prefix, service), "hashedVideoID", hashPrefixes) + : fetchFromDB(hashPrefixes); } \ No newline at end of file diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index d8ecb96..d5ea110 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -10,6 +10,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { if (!err && reply) { try { Logger.debug(`Got data from redis: ${reply}`); + return JSON.parse(reply); } catch (e) { // If all else, continue on to fetching from the database @@ -22,6 +23,53 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { return data; } +/** + * Gets from redis for all specified values and splits the result before adding it to redis cache + */ +async function getAndSplit(fetchFromDB: (values: U[]) => Promise>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise> { + const cachedValues = await Promise.all(values.map(async (value) => { + const key = keyGenerator(value); + const { err, reply } = await redis.getAsync(key); + + if (!err && reply) { + try { + Logger.debug(`Got data from redis: ${reply}`); + + return { + value, + result: JSON.parse(reply) + }; + } catch (e) { + // eslint-disable-next-line no-console + } + } + + return { + value, + result: null + }; + })); + + const data = await fetchFromDB( + cachedValues.filter((cachedValue) => cachedValue.result === null) + .map((cachedValue) => cachedValue.value)); + + new Promise(() => { + const newResults: Record = {}; + for (const item of data) { + const key = (item as unknown as Record)[splitKey]; + newResults[key] ??= []; + newResults[key].push(item); + } + + for (const key in newResults) { + redis.setAsync(keyGenerator(key as unknown as U), JSON.stringify(newResults[key])); + } + }); + + return data.concat(cachedValues.map((cachedValue) => cachedValue.result).filter((result) => result !== null)); +} + function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void { if (videoInfo) { redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); @@ -38,6 +86,7 @@ function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Serv export const QueryCacher = { get, + getAndSplit, clearSegmentCache, clearRatingCache }; \ No newline at end of file From 9e86f463d855632bbd4e8c970e4261621ed36bb2 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 2 Dec 2021 01:23:59 -0500 Subject: [PATCH 2/5] Support regex via sqlite --- src/databases/Sqlite.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/databases/Sqlite.ts b/src/databases/Sqlite.ts index fc45bbe..f396791 100644 --- a/src/databases/Sqlite.ts +++ b/src/databases/Sqlite.ts @@ -15,7 +15,7 @@ export class Sqlite implements IDatabase { // eslint-disable-next-line require-await async prepare(type: QueryType, query: string, params: any[] = []): Promise { // Logger.debug(`prepare (sqlite): type: ${type}, query: ${query}, params: ${params}`); - const preparedQuery = this.db.prepare(query); + const preparedQuery = this.db.prepare(Sqlite.processQuery(query)); switch (type) { case "get": { @@ -53,6 +53,10 @@ export class Sqlite implements IDatabase { Sqlite.upgradeDB(this.db, this.config.fileNamePrefix, this.config.dbSchemaFolder); } + this.db.function("regexp", { deterministic: true }, (regex: string, str: string) => { + return str.match(regex) ? 1 : 0; + }); + // Enable WAL mode checkpoint number if (this.config.enableWalCheckpointNumber) { this.db.exec("PRAGMA journal_mode=WAL;"); @@ -67,6 +71,10 @@ export class Sqlite implements IDatabase { this.db.prepare(`ATTACH ? as ${attachAs}`).run(database); } + private static processQuery(query: string): string { + return query.replace(/ ~\* /g, " REGEXP "); + } + private static upgradeDB(db: Database, fileNamePrefix: string, schemaFolder: string) { const versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version"); let versionCode = versionCodeInfo ? versionCodeInfo.value : 0; From dd2db4bbbfdc97653a16126c8049a3c0df93268e Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 2 Dec 2021 01:25:26 -0500 Subject: [PATCH 3/5] Fix wrong eslint disable --- src/utils/queryCacher.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index d5ea110..be7c21d 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -39,9 +39,7 @@ async function getAndSplit(fetchFromDB: (values: U[]) => Promise> value, result: JSON.parse(reply) }; - } catch (e) { - // eslint-disable-next-line no-console - } + } catch (e) { } //eslint-disable-line no-empty } return { From 48ac8d1136c24376248cdece3d4bfdda1fd003c8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 2 Dec 2021 23:12:48 -0500 Subject: [PATCH 4/5] Add test for get bulk rating --- src/app.ts | 1 + src/utils/queryCacher.ts | 1 + test/cases/ratings/getRating.ts | 39 +++++++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index ee0ab8c..cb0f2c6 100644 --- a/src/app.ts +++ b/src/app.ts @@ -193,6 +193,7 @@ function setupRoutes(router: Router) { // ratings router.get("/api/ratings/rate/:prefix", getRating); + router.get("/api/ratings/rate", getRating); router.post("/api/ratings/rate", postRateEndpoints); router.post("/api/ratings/clearCache", ratingPostClearCache); diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index be7c21d..34fc1a0 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -20,6 +20,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { const data = await fetchFromDB(); redis.setAsync(key, JSON.stringify(data)); + return data; } diff --git a/test/cases/ratings/getRating.ts b/test/cases/ratings/getRating.ts index ee82f23..3ba3909 100644 --- a/test/cases/ratings/getRating.ts +++ b/test/cases/ratings/getRating.ts @@ -5,18 +5,25 @@ import { client } from "../../utils/httpClient"; import { AxiosResponse } from "axios"; import { partialDeepEquals } from "../../utils/partialDeepEquals"; -const endpoint = "/api/ratings/rate/"; -const getRating = (hash: string, params?: unknown): Promise => client.get(endpoint + hash, { params }); +const endpoint = "/api/ratings/rate"; +const getRating = (hash: string, params?: unknown): Promise => client.get(`${endpoint}/${hash}`, { params }); +const getBulkRating = (hashes: string[], params?: any): Promise => client.get(endpoint, { params: { ...params, prefix: hashes } }); const videoOneID = "some-likes-and-dislikes"; const videoOneIDHash = getHash(videoOneID, 1); const videoOnePartialHash = videoOneIDHash.substr(0, 4); +const videoTwoID = "some-likes-and-dislikes-2"; +const videoTwoIDHash = getHash(videoTwoID, 1); +const videoTwoPartialHash = videoTwoIDHash.substr(0, 4); describe("getRating", () => { before(async () => { const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 0, 5, videoOneIDHash]); await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 1, 10, videoOneIDHash]); + + await db.prepare("run", insertUserNameQuery, [videoTwoID, "YouTube", 0, 20, videoTwoIDHash]); + await db.prepare("run", insertUserNameQuery, [videoTwoID, "YouTube", 1, 30, videoTwoIDHash]); }); it("Should be able to get dislikes and likes by default", (done) => { @@ -51,6 +58,34 @@ describe("getRating", () => { .catch(err => done(err)); }); + it("Should be able to bulk fetch", (done) => { + getBulkRating([videoOnePartialHash, videoTwoPartialHash]) + .then(res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 20, + hash: videoTwoIDHash, + }, + { + type: 1, + count: 30, + hash: videoTwoIDHash, + }, { + type: 0, + count: 5, + hash: videoOneIDHash, + }, { + type: 1, + count: 10, + hash: videoOneIDHash, + }]; + assert.ok(partialDeepEquals(res.data, expected)); + done(); + }) + .catch(err => done(err)); + }); + it("Should return 400 for invalid hash", (done) => { getRating("a") .then(res => { From 0cd25f0498e240434172b4a25502eae39815d81b Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Thu, 2 Dec 2021 23:42:39 -0500 Subject: [PATCH 5/5] Fix issues with query caching for ratings --- src/utils/queryCacher.ts | 44 +++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index 34fc1a0..5ddccef 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -27,7 +27,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { /** * Gets from redis for all specified values and splits the result before adding it to redis cache */ -async function getAndSplit(fetchFromDB: (values: U[]) => Promise>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise> { +async function getAndSplit(fetchFromDB: (values: U[]) => Promise>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise> { const cachedValues = await Promise.all(values.map(async (value) => { const key = keyGenerator(value); const { err, reply } = await redis.getAsync(key); @@ -49,24 +49,36 @@ async function getAndSplit(fetchFromDB: (values: U[]) => Promise> }; })); - const data = await fetchFromDB( - cachedValues.filter((cachedValue) => cachedValue.result === null) - .map((cachedValue) => cachedValue.value)); + const valuesToBeFetched = cachedValues.filter((cachedValue) => cachedValue.result === null) + .map((cachedValue) => cachedValue.value); - new Promise(() => { - const newResults: Record = {}; - for (const item of data) { - const key = (item as unknown as Record)[splitKey]; - newResults[key] ??= []; - newResults[key].push(item); - } + let data: Array = []; + if (valuesToBeFetched.length > 0) { + data = await fetchFromDB(valuesToBeFetched); - for (const key in newResults) { - redis.setAsync(keyGenerator(key as unknown as U), JSON.stringify(newResults[key])); - } - }); + new Promise(() => { + const newResults: Record = {}; + for (const item of data) { + const splitValue = (item as unknown as Record)[splitKey]; + const key = keyGenerator(splitValue as unknown as U); + newResults[key] ??= []; + newResults[key].push(item); + } - return data.concat(cachedValues.map((cachedValue) => cachedValue.result).filter((result) => result !== null)); + for (const value of valuesToBeFetched) { + // If it wasn't in the result, cache it as blank + newResults[keyGenerator(value)] ??= []; + } + + console.log(newResults); + + for (const key in newResults) { + redis.setAsync(key, JSON.stringify(newResults[key])); + } + }); + } + + return data.concat(...(cachedValues.map((cachedValue) => cachedValue.result).filter((result) => result !== null) || [])); } function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void {