diff --git a/src/app.ts b/src/app.ts index ee0ab8c..cb0f2c6 100644 --- a/src/app.ts +++ b/src/app.ts @@ -193,6 +193,7 @@ function setupRoutes(router: Router) { // ratings router.get("/api/ratings/rate/:prefix", getRating); + router.get("/api/ratings/rate", getRating); router.post("/api/ratings/rate", postRateEndpoints); router.post("/api/ratings/clearCache", ratingPostClearCache); diff --git a/src/databases/Sqlite.ts b/src/databases/Sqlite.ts index fc45bbe..f396791 100644 --- a/src/databases/Sqlite.ts +++ b/src/databases/Sqlite.ts @@ -15,7 +15,7 @@ export class Sqlite implements IDatabase { // eslint-disable-next-line require-await async prepare(type: QueryType, query: string, params: any[] = []): Promise { // Logger.debug(`prepare (sqlite): type: ${type}, query: ${query}, params: ${params}`); - const preparedQuery = this.db.prepare(query); + const preparedQuery = this.db.prepare(Sqlite.processQuery(query)); switch (type) { case "get": { @@ -53,6 +53,10 @@ export class Sqlite implements IDatabase { Sqlite.upgradeDB(this.db, this.config.fileNamePrefix, this.config.dbSchemaFolder); } + this.db.function("regexp", { deterministic: true }, (regex: string, str: string) => { + return str.match(regex) ? 1 : 0; + }); + // Enable WAL mode checkpoint number if (this.config.enableWalCheckpointNumber) { this.db.exec("PRAGMA journal_mode=WAL;"); @@ -67,6 +71,10 @@ export class Sqlite implements IDatabase { this.db.prepare(`ATTACH ? as ${attachAs}`).run(database); } + private static processQuery(query: string): string { + return query.replace(/ ~\* /g, " REGEXP "); + } + private static upgradeDB(db: Database, fileNamePrefix: string, schemaFolder: string) { const versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version"); let versionCode = versionCodeInfo ? versionCodeInfo.value : 0; diff --git a/src/routes/ratings/getRating.ts b/src/routes/ratings/getRating.ts index ec7e9f0..edaae8e 100644 --- a/src/routes/ratings/getRating.ts +++ b/src/routes/ratings/getRating.ts @@ -17,11 +17,24 @@ interface DBRating { } export async function getRating(req: Request, res: Response): Promise { - let hashPrefix = req.params.prefix as VideoIDHash; - if (!hashPrefix || !hashPrefixTester(hashPrefix)) { + let hashPrefixes: VideoIDHash[] = []; + try { + hashPrefixes = req.query.hashPrefixes + ? JSON.parse(req.query.hashPrefixes as string) + : Array.isArray(req.query.prefix) + ? req.query.prefix + : [req.query.prefix ?? req.params.prefix]; + if (!Array.isArray(hashPrefixes)) { + return res.status(400).send("hashPrefixes parameter does not match format requirements."); + } + + hashPrefixes.map((hashPrefix) => hashPrefix?.toLowerCase()); + } catch(error) { + return res.status(400).send("Bad parameter: hashPrefixes (invalid JSON)"); + } + if (hashPrefixes.some((hashPrefix) => !hashPrefix || !hashPrefixTester(hashPrefix))) { return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix } - hashPrefix = hashPrefix.toLowerCase() as VideoIDHash; let types: RatingType[] = []; try { @@ -44,7 +57,7 @@ export async function getRating(req: Request, res: Response): Promise const service: Service = getService(req.query.service, req.body.service); try { - const ratings = (await getRatings(hashPrefix, service)) + const ratings = (await getRatings(hashPrefixes, service)) .filter((rating) => types.includes(rating.type)) .map((rating) => ({ videoID: rating.videoID, @@ -53,23 +66,25 @@ export async function getRating(req: Request, res: Response): Promise type: rating.type, count: rating.count })); + return res.status((ratings.length) ? 200 : 404) .send(ratings ?? []); } catch (err) { Logger.error(err as string); + return res.sendStatus(500); } } -function getRatings(hashPrefix: VideoIDHash, service: Service): Promise { - const fetchFromDB = () => db +function getRatings(hashPrefixes: VideoIDHash[], service: Service): Promise { + const fetchFromDB = (hashPrefixes: VideoIDHash[]) => db .prepare( "all", - `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "hashedVideoID"`, - [`${hashPrefix}%`, service] + `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" ~* ? AND "service" = ? ORDER BY "hashedVideoID"`, + [`^(?:${hashPrefixes.join("|")})`, service] ) as Promise; - return (hashPrefix.length === 4) - ? QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service)) - : fetchFromDB(); + return (hashPrefixes.every((hashPrefix) => hashPrefix.length === 4)) + ? QueryCacher.getAndSplit(fetchFromDB, (prefix) => ratingHashKey(prefix, service), "hashedVideoID", hashPrefixes) + : fetchFromDB(hashPrefixes); } \ No newline at end of file diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index d8ecb96..5ddccef 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -10,6 +10,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { if (!err && reply) { try { Logger.debug(`Got data from redis: ${reply}`); + return JSON.parse(reply); } catch (e) { // If all else, continue on to fetching from the database @@ -19,9 +20,67 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { const data = await fetchFromDB(); redis.setAsync(key, JSON.stringify(data)); + return data; } +/** + * Gets from redis for all specified values and splits the result before adding it to redis cache + */ +async function getAndSplit(fetchFromDB: (values: U[]) => Promise>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise> { + const cachedValues = await Promise.all(values.map(async (value) => { + const key = keyGenerator(value); + const { err, reply } = await redis.getAsync(key); + + if (!err && reply) { + try { + Logger.debug(`Got data from redis: ${reply}`); + + return { + value, + result: JSON.parse(reply) + }; + } catch (e) { } //eslint-disable-line no-empty + } + + return { + value, + result: null + }; + })); + + const valuesToBeFetched = cachedValues.filter((cachedValue) => cachedValue.result === null) + .map((cachedValue) => cachedValue.value); + + let data: Array = []; + if (valuesToBeFetched.length > 0) { + data = await fetchFromDB(valuesToBeFetched); + + new Promise(() => { + const newResults: Record = {}; + for (const item of data) { + const splitValue = (item as unknown as Record)[splitKey]; + const key = keyGenerator(splitValue as unknown as U); + newResults[key] ??= []; + newResults[key].push(item); + } + + for (const value of valuesToBeFetched) { + // If it wasn't in the result, cache it as blank + newResults[keyGenerator(value)] ??= []; + } + + console.log(newResults); + + for (const key in newResults) { + redis.setAsync(key, JSON.stringify(newResults[key])); + } + }); + } + + return data.concat(...(cachedValues.map((cachedValue) => cachedValue.result).filter((result) => result !== null) || [])); +} + function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void { if (videoInfo) { redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service)); @@ -38,6 +97,7 @@ function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Serv export const QueryCacher = { get, + getAndSplit, clearSegmentCache, clearRatingCache }; \ No newline at end of file diff --git a/test/cases/ratings/getRating.ts b/test/cases/ratings/getRating.ts index ee82f23..3ba3909 100644 --- a/test/cases/ratings/getRating.ts +++ b/test/cases/ratings/getRating.ts @@ -5,18 +5,25 @@ import { client } from "../../utils/httpClient"; import { AxiosResponse } from "axios"; import { partialDeepEquals } from "../../utils/partialDeepEquals"; -const endpoint = "/api/ratings/rate/"; -const getRating = (hash: string, params?: unknown): Promise => client.get(endpoint + hash, { params }); +const endpoint = "/api/ratings/rate"; +const getRating = (hash: string, params?: unknown): Promise => client.get(`${endpoint}/${hash}`, { params }); +const getBulkRating = (hashes: string[], params?: any): Promise => client.get(endpoint, { params: { ...params, prefix: hashes } }); const videoOneID = "some-likes-and-dislikes"; const videoOneIDHash = getHash(videoOneID, 1); const videoOnePartialHash = videoOneIDHash.substr(0, 4); +const videoTwoID = "some-likes-and-dislikes-2"; +const videoTwoIDHash = getHash(videoTwoID, 1); +const videoTwoPartialHash = videoTwoIDHash.substr(0, 4); describe("getRating", () => { before(async () => { const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)'; await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 0, 5, videoOneIDHash]); await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 1, 10, videoOneIDHash]); + + await db.prepare("run", insertUserNameQuery, [videoTwoID, "YouTube", 0, 20, videoTwoIDHash]); + await db.prepare("run", insertUserNameQuery, [videoTwoID, "YouTube", 1, 30, videoTwoIDHash]); }); it("Should be able to get dislikes and likes by default", (done) => { @@ -51,6 +58,34 @@ describe("getRating", () => { .catch(err => done(err)); }); + it("Should be able to bulk fetch", (done) => { + getBulkRating([videoOnePartialHash, videoTwoPartialHash]) + .then(res => { + assert.strictEqual(res.status, 200); + const expected = [{ + type: 0, + count: 20, + hash: videoTwoIDHash, + }, + { + type: 1, + count: 30, + hash: videoTwoIDHash, + }, { + type: 0, + count: 5, + hash: videoOneIDHash, + }, { + type: 1, + count: 10, + hash: videoOneIDHash, + }]; + assert.ok(partialDeepEquals(res.data, expected)); + done(); + }) + .catch(err => done(err)); + }); + it("Should return 400 for invalid hash", (done) => { getRating("a") .then(res => {