Merge pull request #417 from ajayyy/bulk-ratings

Add bulk rating fetching

src/app.ts

@@ -193,6 +193,7 @@ function setupRoutes(router: Router) {
 
     // ratings
     router.get("/api/ratings/rate/:prefix", getRating);
+    router.get("/api/ratings/rate", getRating);
     router.post("/api/ratings/rate", postRateEndpoints);
     router.post("/api/ratings/clearCache", ratingPostClearCache);
 

src/databases/Sqlite.ts

@@ -15,7 +15,7 @@ export class Sqlite implements IDatabase {
     // eslint-disable-next-line require-await
     async prepare(type: QueryType, query: string, params: any[] = []): Promise<any[]> {
         // Logger.debug(`prepare (sqlite): type: ${type}, query: ${query}, params: ${params}`);
-        const preparedQuery = this.db.prepare(query);
+        const preparedQuery = this.db.prepare(Sqlite.processQuery(query));
 
         switch (type) {
             case "get": {
@@ -53,6 +53,10 @@ export class Sqlite implements IDatabase {
             Sqlite.upgradeDB(this.db, this.config.fileNamePrefix, this.config.dbSchemaFolder);
         }
 
+        this.db.function("regexp", { deterministic: true }, (regex: string, str: string) => {
+            return str.match(regex) ? 1 : 0;
+        });
+
         // Enable WAL mode checkpoint number
         if (this.config.enableWalCheckpointNumber) {
             this.db.exec("PRAGMA journal_mode=WAL;");
@@ -67,6 +71,10 @@ export class Sqlite implements IDatabase {
         this.db.prepare(`ATTACH ? as ${attachAs}`).run(database);
     }
 
+    private static processQuery(query: string): string {
+        return query.replace(/ ~\* /g, " REGEXP ");
+    }
+
     private static upgradeDB(db: Database, fileNamePrefix: string, schemaFolder: string) {
         const versionCodeInfo = db.prepare("SELECT value FROM config WHERE key = ?").get("version");
         let versionCode = versionCodeInfo ? versionCodeInfo.value : 0;

src/routes/ratings/getRating.ts

@@ -17,11 +17,24 @@ interface DBRating {
 }
 
 export async function getRating(req: Request, res: Response): Promise<Response> {
-    let hashPrefix = req.params.prefix as VideoIDHash;
-    if (!hashPrefix || !hashPrefixTester(hashPrefix)) {
+    let hashPrefixes: VideoIDHash[] = [];
+    try {
+        hashPrefixes = req.query.hashPrefixes
+            ? JSON.parse(req.query.hashPrefixes as string)
+            : Array.isArray(req.query.prefix)
+                ? req.query.prefix
+                : [req.query.prefix ?? req.params.prefix];
+        if (!Array.isArray(hashPrefixes)) {
+            return res.status(400).send("hashPrefixes parameter does not match format requirements.");
+        }
+
+        hashPrefixes.map((hashPrefix) => hashPrefix?.toLowerCase());
+    } catch(error) {
+        return res.status(400).send("Bad parameter: hashPrefixes (invalid JSON)");
+    }
+    if (hashPrefixes.some((hashPrefix) => !hashPrefix || !hashPrefixTester(hashPrefix))) {
         return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
     }
-    hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
 
     let types: RatingType[] = [];
     try {
@@ -44,7 +57,7 @@ export async function getRating(req: Request, res: Response): Promise<Response>
     const service: Service = getService(req.query.service, req.body.service);
 
     try {
-        const ratings = (await getRatings(hashPrefix, service))
+        const ratings = (await getRatings(hashPrefixes, service))
             .filter((rating) => types.includes(rating.type))
             .map((rating) => ({
                 videoID: rating.videoID,
@@ -53,23 +66,25 @@ export async function getRating(req: Request, res: Response): Promise<Response>
                 type: rating.type,
                 count: rating.count
             }));
+
         return res.status((ratings.length) ? 200 : 404)
             .send(ratings ?? []);
     } catch (err) {
         Logger.error(err as string);
+
         return res.sendStatus(500);
     }
 }
 
-function getRatings(hashPrefix: VideoIDHash, service: Service): Promise<DBRating[]> {
-    const fetchFromDB = () => db
+function getRatings(hashPrefixes: VideoIDHash[], service: Service): Promise<DBRating[]> {
+    const fetchFromDB = (hashPrefixes: VideoIDHash[]) => db
         .prepare(
             "all",
-            `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "hashedVideoID"`,
-            [`${hashPrefix}%`, service]
+            `SELECT "videoID", "hashedVideoID", "type", "count" FROM "ratings" WHERE "hashedVideoID" ~* ? AND "service" = ? ORDER BY "hashedVideoID"`,
+            [`^(?:${hashPrefixes.join("|")})`, service]
         ) as Promise<DBRating[]>;
 
-    return (hashPrefix.length === 4)
-        ? QueryCacher.get(fetchFromDB, ratingHashKey(hashPrefix, service))
-        : fetchFromDB();
+    return (hashPrefixes.every((hashPrefix) => hashPrefix.length === 4))
+        ? QueryCacher.getAndSplit(fetchFromDB, (prefix) => ratingHashKey(prefix, service), "hashedVideoID", hashPrefixes)
+        : fetchFromDB(hashPrefixes);
 }

src/utils/queryCacher.ts

@@ -10,6 +10,7 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
     if (!err && reply) {
         try {
             Logger.debug(`Got data from redis: ${reply}`);
+
             return JSON.parse(reply);
         } catch (e) {
             // If all else, continue on to fetching from the database
@@ -19,9 +20,67 @@ async function get<T>(fetchFromDB: () => Promise<T>, key: string): Promise<T> {
     const data = await fetchFromDB();
 
     redis.setAsync(key, JSON.stringify(data));
+
     return data;
 }
 
+/**
+ * Gets from redis for all specified values and splits the result before adding it to redis cache
+ */
+async function getAndSplit<T, U extends string>(fetchFromDB: (values: U[]) => Promise<Array<T>>, keyGenerator: (value: U) => string, splitKey: string, values: U[]): Promise<Array<T>> {
+    const cachedValues = await Promise.all(values.map(async (value) => {
+        const key = keyGenerator(value);
+        const { err, reply } = await redis.getAsync(key);
+
+        if (!err && reply) {
+            try {
+                Logger.debug(`Got data from redis: ${reply}`);
+
+                return {
+                    value,
+                    result: JSON.parse(reply)
+                };
+            } catch (e) { } //eslint-disable-line no-empty
+        }
+
+        return {
+            value,
+            result: null
+        };
+    }));
+
+    const valuesToBeFetched = cachedValues.filter((cachedValue) => cachedValue.result === null)
+        .map((cachedValue) => cachedValue.value);
+
+    let data: Array<T> = [];
+    if (valuesToBeFetched.length > 0) {
+        data = await fetchFromDB(valuesToBeFetched);
+
+        new Promise(() => {
+            const newResults: Record<string, T[]> = {};
+            for (const item of data) {
+                const splitValue = (item as unknown as Record<string, string>)[splitKey];
+                const key = keyGenerator(splitValue as unknown as U);
+                newResults[key] ??= [];
+                newResults[key].push(item);
+            }
+
+            for (const value of valuesToBeFetched) {
+                // If it wasn't in the result, cache it as blank
+                newResults[keyGenerator(value)] ??= [];
+            }
+
+            console.log(newResults);
+
+            for (const key in newResults) {
+                redis.setAsync(key, JSON.stringify(newResults[key]));
+            }
+        });
+    }
+
+    return data.concat(...(cachedValues.map((cachedValue) => cachedValue.result).filter((result) => result !== null) || []));
+}
+
 function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; service: Service; userID?: UserID; }): void {
     if (videoInfo) {
         redis.delAsync(skipSegmentsKey(videoInfo.videoID, videoInfo.service));
@@ -38,6 +97,7 @@ function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Serv
 
 export const QueryCacher = {
     get,
+    getAndSplit,
     clearSegmentCache,
     clearRatingCache
 };

test/cases/ratings/getRating.ts

@@ -5,18 +5,25 @@ import { client } from "../../utils/httpClient";
 import { AxiosResponse } from "axios";
 import { partialDeepEquals } from "../../utils/partialDeepEquals";
 
-const endpoint = "/api/ratings/rate/";
-const getRating = (hash: string, params?: unknown): Promise<AxiosResponse> => client.get(endpoint + hash, { params });
+const endpoint = "/api/ratings/rate";
+const getRating = (hash: string, params?: unknown): Promise<AxiosResponse> => client.get(`${endpoint}/${hash}`, { params });
+const getBulkRating = (hashes: string[], params?: any): Promise<AxiosResponse> => client.get(endpoint, { params: { ...params, prefix: hashes } });
 
 const videoOneID = "some-likes-and-dislikes";
 const videoOneIDHash = getHash(videoOneID, 1);
 const videoOnePartialHash = videoOneIDHash.substr(0, 4);
+const videoTwoID = "some-likes-and-dislikes-2";
+const videoTwoIDHash = getHash(videoTwoID, 1);
+const videoTwoPartialHash = videoTwoIDHash.substr(0, 4);
 
 describe("getRating", () => {
     before(async () => {
         const insertUserNameQuery = 'INSERT INTO "ratings" ("videoID", "service", "type", "count", "hashedVideoID") VALUES (?, ?, ?, ?, ?)';
         await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 0, 5, videoOneIDHash]);
         await db.prepare("run", insertUserNameQuery, [videoOneID, "YouTube", 1, 10, videoOneIDHash]);
+
+        await db.prepare("run", insertUserNameQuery, [videoTwoID, "YouTube", 0, 20, videoTwoIDHash]);
+        await db.prepare("run", insertUserNameQuery, [videoTwoID, "YouTube", 1, 30, videoTwoIDHash]);
     });
 
     it("Should be able to get dislikes and likes by default", (done) => {
@@ -51,6 +58,34 @@ describe("getRating", () => {
             .catch(err => done(err));
     });
 
+    it("Should be able to bulk fetch", (done) => {
+        getBulkRating([videoOnePartialHash, videoTwoPartialHash])
+            .then(res => {
+                assert.strictEqual(res.status, 200);
+                const expected = [{
+                    type: 0,
+                    count: 20,
+                    hash: videoTwoIDHash,
+                },
+                {
+                    type: 1,
+                    count: 30,
+                    hash: videoTwoIDHash,
+                }, {
+                    type: 0,
+                    count: 5,
+                    hash: videoOneIDHash,
+                }, {
+                    type: 1,
+                    count: 10,
+                    hash: videoOneIDHash,
+                }];
+                assert.ok(partialDeepEquals(res.data, expected));
+                done();
+            })
+            .catch(err => done(err));
+    });
+
     it("Should return 400 for invalid hash", (done) => {
         getRating("a")
             .then(res => {