Add tests for get branding and fix issues

Also improve deep partial equals
This commit is contained in:
Ajay
2023-01-28 01:53:59 -05:00
parent 4d8ce40ef4
commit 36f1d15605
4 changed files with 274 additions and 48 deletions

View File

@@ -2,13 +2,14 @@ import { Request, Response } from "express";
import { isEmpty } from "lodash";
import { config } from "../config";
import { db, privateDB } from "../databases/databases";
import { BrandingDBSubmission, BrandingHashDBResult, BrandingHashResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
import { BrandingDBSubmission, BrandingHashDBResult, BrandingResult, ThumbnailDBResult, ThumbnailResult, TitleDBResult, TitleResult } from "../types/branding.model";
import { HashedIP, IPAddress, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model";
import { shuffleArray } from "../utils/array";
import { getHashCache } from "../utils/getHashCache";
import { getIP } from "../utils/getIP";
import { getService } from "../utils/getService";
import { hashPrefixTester } from "../utils/hashPrefixTester";
import { Logger } from "../utils/logger";
import { promiseOrTimeout } from "../utils/promise";
import { QueryCacher } from "../utils/queryCacher";
import { brandingHashKey, brandingIPKey, brandingKey } from "../utils/redisKeys";
@@ -21,7 +22,7 @@ enum BrandingSubmissionType {
export async function getVideoBranding(videoID: VideoID, service: Service, ip: IPAddress): Promise<BrandingResult> {
const getTitles = () => db.prepare(
"all",
`SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID
`SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID"
FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
WHERE "titles"."videoID" = ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
[videoID, service],
@@ -52,10 +53,10 @@ export async function getVideoBranding(videoID: VideoID, service: Service, ip: I
return filterAndSortBranding(await branding.titles, await branding.thumbnails, ip, cache);
}
export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise<Record<VideoID, BrandingHashResult>> {
export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, service: Service, ip: IPAddress): Promise<Record<VideoID, BrandingResult>> {
const getTitles = () => db.prepare(
"all",
`SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "title"."UUID", "title"."videoID", "title"."hashedVideoID
`SELECT "titles"."title", "titles"."original", "titleVotes"."votes", "titleVotes"."locked", "titleVotes"."shadowHidden", "titles"."UUID", "titles"."videoID", "titles"."hashedVideoID"
FROM "titles" JOIN "titleVotes" ON "titles"."UUID" = "titleVotes"."UUID"
WHERE "titles"."hashedVideoID" LIKE ? AND "titles"."service" = ? AND "titleVotes"."votes" > -2`,
[`${videoHashPrefix}%`, service],
@@ -81,20 +82,18 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi
const dbResult: Record<VideoID, BrandingHashDBResult> = {};
const initResult = (submission: BrandingDBSubmission) => {
dbResult[submission.videoID] = dbResult[submission.videoID] || {
branding: {
titles: [],
thumbnails: []
}
};
};
(await branding.titles).map((title) => {
initResult(title);
dbResult[title.videoID].branding.titles.push(title);
dbResult[title.videoID].titles.push(title);
});
(await branding.thumbnails).map((thumbnail) => {
initResult(thumbnail);
dbResult[thumbnail.videoID].branding.thumbnails.push(thumbnail);
dbResult[thumbnail.videoID].thumbnails.push(thumbnail);
});
return dbResult;
@@ -105,12 +104,10 @@ export async function getVideoBrandingByHash(videoHashPrefix: VideoIDHash, servi
currentIP: null as Promise<HashedIP> | null
};
const processedResult: Record<VideoID, BrandingHashResult> = {};
const processedResult: Record<VideoID, BrandingResult> = {};
await Promise.all(Object.keys(branding).map(async (key) => {
const castedKey = key as VideoID;
processedResult[castedKey] = {
branding: await filterAndSortBranding(branding[castedKey].branding.titles, branding[castedKey].branding.thumbnails, ip, cache)
};
processedResult[castedKey] = await filterAndSortBranding(branding[castedKey].titles, branding[castedKey].thumbnails, ip, cache);
}));
return processedResult;
@@ -164,7 +161,7 @@ async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: B
return submitterIP.hashedIP !== hashedIP;
} catch (e) {
// give up on shadow hide for now
return true;
return false;
}
}));
@@ -173,35 +170,41 @@ async function shouldKeepSubmission(submissions: BrandingDBSubmission[], type: B
export async function getBranding(req: Request, res: Response) {
const videoID: VideoID = req.query.videoID as VideoID;
const service: Service = getService(req.query.service, req.body.service);
const service: Service = getService(req.query.service as string);
if (!videoID) {
return res.status(400).send("Missing parameter: videoID");
}
const ip = getIP(req);
try {
const result = await getVideoBranding(videoID, service, ip);
const status = result.titles.length > 0 || result.thumbnails.length > 0 ? 200 : 404;
return res.status(status).json(result);
} catch (e) {
Logger.error(e as string);
return res.status(500).send("Internal server error");
}
}
export async function getBrandingByHashEndpoint(req: Request, res: Response) {
let hashPrefix = req.params.prefix as VideoIDHash;
if (!req.params.prefix || !hashPrefixTester(req.params.prefix)) {
if (!hashPrefix || !hashPrefixTester(hashPrefix) || hashPrefix.length !== 4) {
return res.status(400).send("Hash prefix does not match format requirements."); // Exit early on faulty prefix
}
hashPrefix = hashPrefix.toLowerCase() as VideoIDHash;
const service: Service = getService(req.query.service, req.body.service);
if (!hashPrefix || hashPrefix.length !== 4) {
return res.status(400).send("Hash prefix does not match format requirements.");
}
const service: Service = getService(req.query.service as string);
const ip = getIP(req);
try {
const result = await getVideoBrandingByHash(hashPrefix, service, ip);
const status = !isEmpty(result) ? 200 : 404;
return res.status(status).json(result);
} catch (e) {
Logger.error(e as string);
return res.status(500).send([]);
}
}

View File

@@ -46,14 +46,8 @@ export interface BrandingResult {
}
export interface BrandingHashDBResult {
branding: {
titles: TitleDBResult[],
thumbnails: ThumbnailDBResult[]
};
}
export interface BrandingHashResult {
branding: BrandingResult;
}
export interface OriginalThumbnailSubmission {

217
test/cases/getBranding.ts Normal file
View File

@@ -0,0 +1,217 @@
import { client } from "../utils/httpClient";
import assert from "assert";
import { getHash } from "../../src/utils/getHash";
import { db } from "../../src/databases/databases";
import { Service } from "../../src/types/segments.model";
import { BrandingResult, BrandingUUID } from "../../src/types/branding.model";
import { partialDeepEquals } from "../utils/partialDeepEquals";
describe("getBranding", () => {
const videoID1 = "videoID1";
const videoID2Locked = "videoID2";
const videoID2ShadowHide = "videoID3";
const videoIDEmpty = "videoID4";
const videoID1Hash = getHash(videoID1, 1).slice(0, 4);
const videoID2LockedHash = getHash(videoID2Locked, 1).slice(0, 4);
const videoID2ShadowHideHash = getHash(videoID2ShadowHide, 1).slice(0, 4);
const videoIDEmptyHash = "aaaa";
const endpoint = "/api/branding";
const getBranding = (params: Record<string, any>) => client({
method: "GET",
url: endpoint,
params
});
const getBrandingByHash = (hash: string, params: Record<string, any>) => client({
method: "GET",
url: `${endpoint}/${hash}`,
params
});
before(() => {
const titleQuery = `INSERT INTO "titles" ("videoID", "title", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?, ?)`;
const titleVotesQuery = `INSERT INTO "titleVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, ?, ?, ?)`;
const thumbnailQuery = `INSERT INTO "thumbnails" ("videoID", "original", "userID", "service", "hashedVideoID", "timeSubmitted", "UUID") VALUES (?, ?, ?, ?, ?, ?, ?)`;
const thumbnailTimestampsQuery = `INSERT INTO "thumbnailTimestamps" ("UUID", "timestamp") VALUES (?, ?)`;
const thumbnailVotesQuery = `INSERT INTO "thumbnailVotes" ("UUID", "votes", "locked", "shadowHidden") VALUES (?, ?, ?, ?)`;
db.prepare("run", titleQuery, [videoID1, "title1", 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1"]);
db.prepare("run", titleQuery, [videoID1, "title2", 0, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2"]);
db.prepare("run", titleQuery, [videoID1, "title3", 1, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3"]);
db.prepare("run", titleVotesQuery, ["UUID1", 3, 0, 0]);
db.prepare("run", titleVotesQuery, ["UUID2", 2, 0, 0]);
db.prepare("run", titleVotesQuery, ["UUID3", 1, 0, 0]);
db.prepare("run", thumbnailQuery, [videoID1, 0, "userID1", Service.YouTube, videoID1Hash, 1, "UUID1T"]);
db.prepare("run", thumbnailQuery, [videoID1, 1, "userID2", Service.YouTube, videoID1Hash, 1, "UUID2T"]);
db.prepare("run", thumbnailQuery, [videoID1, 0, "userID3", Service.YouTube, videoID1Hash, 1, "UUID3T"]);
db.prepare("run", thumbnailTimestampsQuery, ["UUID1T", 1]);
db.prepare("run", thumbnailTimestampsQuery, ["UUID3T", 3]);
db.prepare("run", thumbnailVotesQuery, ["UUID1T", 3, 0, 0]);
db.prepare("run", thumbnailVotesQuery, ["UUID2T", 2, 0, 0]);
db.prepare("run", thumbnailVotesQuery, ["UUID3T", 1, 0, 0]);
db.prepare("run", titleQuery, [videoID2Locked, "title1", 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11"]);
db.prepare("run", titleQuery, [videoID2Locked, "title2", 0, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21"]);
db.prepare("run", titleQuery, [videoID2Locked, "title3", 1, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31"]);
db.prepare("run", titleVotesQuery, ["UUID11", 3, 0, 0]);
db.prepare("run", titleVotesQuery, ["UUID21", 2, 0, 0]);
db.prepare("run", titleVotesQuery, ["UUID31", 1, 1, 0]);
db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID1", Service.YouTube, videoID2LockedHash, 1, "UUID11T"]);
db.prepare("run", thumbnailQuery, [videoID2Locked, 1, "userID2", Service.YouTube, videoID2LockedHash, 1, "UUID21T"]);
db.prepare("run", thumbnailQuery, [videoID2Locked, 0, "userID3", Service.YouTube, videoID2LockedHash, 1, "UUID31T"]);
db.prepare("run", thumbnailTimestampsQuery, ["UUID11T", 1]);
db.prepare("run", thumbnailTimestampsQuery, ["UUID31T", 3]);
db.prepare("run", thumbnailVotesQuery, ["UUID11T", 3, 0, 0]);
db.prepare("run", thumbnailVotesQuery, ["UUID21T", 2, 0, 0]);
db.prepare("run", thumbnailVotesQuery, ["UUID31T", 1, 1, 0]);
db.prepare("run", titleQuery, [videoID2ShadowHide, "title1", 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12"]);
db.prepare("run", titleQuery, [videoID2ShadowHide, "title2", 0, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22"]);
db.prepare("run", titleQuery, [videoID2ShadowHide, "title3", 1, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32"]);
db.prepare("run", titleVotesQuery, ["UUID12", 3, 0, 0]);
db.prepare("run", titleVotesQuery, ["UUID22", 2, 0, 0]);
db.prepare("run", titleVotesQuery, ["UUID32", 1, 0, 1]);
db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID1", Service.YouTube, videoID2ShadowHideHash, 1, "UUID12T"]);
db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 1, "userID2", Service.YouTube, videoID2ShadowHideHash, 1, "UUID22T"]);
db.prepare("run", thumbnailQuery, [videoID2ShadowHide, 0, "userID3", Service.YouTube, videoID2ShadowHideHash, 1, "UUID32T"]);
db.prepare("run", thumbnailTimestampsQuery, ["UUID12T", 1]);
db.prepare("run", thumbnailTimestampsQuery, ["UUID32T", 3]);
db.prepare("run", thumbnailVotesQuery, ["UUID12T", 3, 0, 0]);
db.prepare("run", thumbnailVotesQuery, ["UUID22T", 2, 0, 0]);
db.prepare("run", thumbnailVotesQuery, ["UUID32T", 1, 0, 1]);
});
it("should get top titles and thumbnails", async () => {
await checkVideo(videoID1, videoID1Hash, {
titles: [{
title: "title1",
original: false,
votes: 3,
locked: false,
UUID: "UUID1" as BrandingUUID
}, {
title: "title2",
original: false,
votes: 2,
locked: false,
UUID: "UUID2" as BrandingUUID
}, {
title: "title3",
original: true,
votes: 1,
locked: false,
UUID: "UUID3" as BrandingUUID
}],
thumbnails: [{
timestamp: 1,
original: false,
votes: 3,
locked: false,
UUID: "UUID1T" as BrandingUUID
}, {
original: true,
votes: 2,
locked: false,
UUID: "UUID2T" as BrandingUUID
}, {
timestamp: 3,
original: false,
votes: 1,
locked: false,
UUID: "UUID3T" as BrandingUUID
}]
});
});
it("should get top titles and thumbnails prioritizing locks", async () => {
await checkVideo(videoID2Locked, videoID2LockedHash, {
titles: [{
title: "title3",
original: true,
votes: 1,
locked: true,
UUID: "UUID31" as BrandingUUID
}, {
title: "title1",
original: false,
votes: 3,
locked: false,
UUID: "UUID11" as BrandingUUID
}, {
title: "title2",
original: false,
votes: 2,
locked: false,
UUID: "UUID21" as BrandingUUID
}],
thumbnails: [{
timestamp: 3,
original: false,
votes: 1,
locked: true,
UUID: "UUID31T" as BrandingUUID
}, {
timestamp: 1,
original: false,
votes: 3,
locked: false,
UUID: "UUID11T" as BrandingUUID
}, {
original: true,
votes: 2,
locked: false,
UUID: "UUID21T" as BrandingUUID
}]
});
});
it("should get top titles and hide shadow hidden", async () => {
await checkVideo(videoID2ShadowHide, videoID2ShadowHideHash, {
titles: [{
title: "title1",
original: false,
votes: 3,
locked: false,
UUID: "UUID12" as BrandingUUID
}, {
title: "title2",
original: false,
votes: 2,
locked: false,
UUID: "UUID22" as BrandingUUID
}],
thumbnails: [{
timestamp: 1,
original: false,
votes: 3,
locked: false,
UUID: "UUID12T" as BrandingUUID
}, {
original: true,
votes: 2,
locked: false,
UUID: "UUID22T" as BrandingUUID
}]
});
});
it("should get 404 when nothing", async () => {
const result1 = await getBranding({ videoID: videoIDEmpty });
const result2 = await getBrandingByHash(videoIDEmptyHash, {});
assert.strictEqual(result1.status, 404);
assert.strictEqual(result2.status, 404);
});
async function checkVideo(videoID: string, videoIDHash: string, expected: BrandingResult) {
const result1 = await getBranding({ videoID });
const result2 = await getBrandingByHash(videoIDHash, {});
assert.strictEqual(result1.status, 200);
assert.strictEqual(result2.status, 200);
assert.deepEqual(result1.data, result2.data[videoID]);
assert.ok(partialDeepEquals(result1.data, expected));
}
});

View File

@@ -1,8 +1,9 @@
import { Logger } from "../../src/utils/logger";
function printActualExpected(actual: Record<string, any>, expected: Record<string, any>): void {
function printActualExpected(actual: Record<string, any>, expected: Record<string, any>, failedKey: string): void {
Logger.error(`Actual: ${JSON.stringify(actual)}`);
Logger.error(`Expected: ${JSON.stringify(expected)}`);
Logger.error(`Failed on key: ${failedKey}`);
}
export const partialDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
@@ -11,11 +12,11 @@ export const partialDeepEquals = (actual: Record<string, any>, expected: Record<
// if value is object or array, recurse
if (Array.isArray(value) || typeof value === "object") {
if (!partialDeepEquals(actual?.[key], value, false)) {
if (print) printActualExpected(actual, expected);
if (print) printActualExpected(actual, expected, key);
return false;
}
} else if (actual?.[key] !== value) {
if (print) printActualExpected(actual, expected);
if (print) printActualExpected(actual, expected, key);
return false;
}
}
@@ -31,14 +32,25 @@ export const arrayPartialDeepEquals = (actual: Array<any>, expected: Array<any>)
export const arrayDeepEquals = (actual: Record<string, any>, expected: Record<string, any>, print = true): boolean => {
if (actual.length !== expected.length) return false;
let flag = true;
let failedKey = "";
const actualString = JSON.stringify(actual);
const expectedString = JSON.stringify(expected);
// check every value in arr1 for match in arr2
actual.every((value: any) => { if (flag && !expectedString.includes(JSON.stringify(value))) flag = false; });
actual.every((value: any) => {
if (flag && !expectedString.includes(JSON.stringify(value))) {
flag = false;
failedKey = value;
}
});
// check arr2 for match in arr1
expected.every((value: any) => { if (flag && !actualString.includes(JSON.stringify(value))) flag = false; });
expected.every((value: any) => {
if (flag && !actualString.includes(JSON.stringify(value))) {
flag = false;
failedKey = value;
}
});
if (!flag && print) printActualExpected(actual, expected);
if (!flag && print) printActualExpected(actual, expected, failedKey);
return flag;
};
@@ -47,12 +59,12 @@ export const mixedDeepEquals = (actual: Record<string, any>, expected: Record<st
// if value is object or array, recurse
if (Array.isArray(value)) {
if (!arrayDeepEquals(actual?.[key], value, false)) {
if (print) printActualExpected(actual, expected);
if (print) printActualExpected(actual, expected, key);
return false;
}
}
else if (actual?.[key] !== value) {
if (print) printActualExpected(actual, expected);
if (print) printActualExpected(actual, expected, key);
return false;
}
}