From 25b91af8bc6b3db9a82ccebff02bcdcd31923b99 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Tue, 6 Oct 2020 23:25:11 +0200 Subject: [PATCH 1/3] Add cache for getTopUsers See #147 --- config.json.example | 3 +- src/routes/getTopUsers.js | 133 ++++++++++++++++++--------------- src/utils/createMemoryCache.js | 42 +++++++++++ 3 files changed, 117 insertions(+), 61 deletions(-) create mode 100644 src/utils/createMemoryCache.js diff --git a/config.json.example b/config.json.example index a2eee23..d6f9204 100644 --- a/config.json.example +++ b/config.json.example @@ -22,5 +22,6 @@ "mode": "development", "readOnly": false, "webhooks": [], - "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected + "categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected + "getTopUsersCacheTimeMinutes": 5 // cacheTime for getTopUsers result in minutes } diff --git a/src/routes/getTopUsers.js b/src/routes/getTopUsers.js index 22b8ca9..b78dad5 100644 --- a/src/routes/getTopUsers.js +++ b/src/routes/getTopUsers.js @@ -1,6 +1,68 @@ var db = require('../databases/databases.js').db; +const logger = require('../utils/logger.js'); +const createMemoryCache = require('../utils/createMemoryCache.js'); +const config = require('../config.js'); -module.exports = function getTopUsers (req, res) { +const MILLISECONDS_IN_MINUTE = 60000; +const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE); + +function generateTopUsersStats(sortBy, categoryStatsEnabled = false) { + return new Promise((resolve, reject) => { + const userNames = []; + const viewCounts = []; + const totalSubmissions = []; + const minutesSaved = []; + const categoryStats = categoryStatsEnabled ? [] : undefined; + + let additionalFields = ''; + if (categoryStatsEnabled) { + additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " + + "SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " + + "SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " + + "SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " + + "SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " + + "SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, "; + } + + const rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + + "SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " + + "SUM(votes) as userVotes, " + + additionalFields + + "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " + + "LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " + + "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " + + "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " + + "ORDER BY " + sortBy + " DESC LIMIT 100", []); + + for (let i = 0; i < rows.length; i++) { + userNames[i] = rows[i].userName; + + viewCounts[i] = rows[i].viewCount; + totalSubmissions[i] = rows[i].totalSubmissions; + minutesSaved[i] = rows[i].minutesSaved; + if (categoryStatsEnabled) { + categoryStats[i] = [ + rows[i].categorySponsor, + rows[i].categorySumIntro, + rows[i].categorySumOutro, + rows[i].categorySumInteraction, + rows[i].categorySelfpromo, + rows[i].categoryMusicOfftopic, + ]; + } + } + + resolve({ + userNames, + viewCounts, + totalSubmissions, + minutesSaved, + categoryStats + }); + }); +} + +module.exports = async function getTopUsers (req, res) { let sortType = req.query.sortType; let categoryStatsEnabled = req.query.categoryStats; @@ -9,71 +71,22 @@ module.exports = function getTopUsers (req, res) { res.sendStatus(400); return; } - + //setup which sort type to use - let sortBy = ""; + let sortBy = ''; if (sortType == 0) { - sortBy = "minutesSaved"; + sortBy = 'minutesSaved'; } else if (sortType == 1) { - sortBy = "viewCount"; + sortBy = 'viewCount'; } else if (sortType == 2) { - sortBy = "totalSubmissions"; + sortBy = 'totalSubmissions'; } else { - //invalid request - res.sendStatus(400); - return; + //invalid request + return res.sendStatus(400); } - - let userNames = []; - let viewCounts = []; - let totalSubmissions = []; - let minutesSaved = []; - let categoryStats = categoryStatsEnabled ? [] : undefined; - let additionalFields = ''; - if (categoryStatsEnabled) { - additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " + - "SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " + - "SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " + - "SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " + - "SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " + - "SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, "; - } - - let rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," + - "SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " + - "SUM(votes) as userVotes, " + - additionalFields + - "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " + - "LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " + - "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " + - "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " + - "ORDER BY " + sortBy + " DESC LIMIT 100", []); + const stats = await getTopUsersWithCache(sortBy, categoryStatsEnabled); - for (let i = 0; i < rows.length; i++) { - userNames[i] = rows[i].userName; - - viewCounts[i] = rows[i].viewCount; - totalSubmissions[i] = rows[i].totalSubmissions; - minutesSaved[i] = rows[i].minutesSaved; - if (categoryStatsEnabled) { - categoryStats[i] = [ - rows[i].categorySponsor, - rows[i].categorySumIntro, - rows[i].categorySumOutro, - rows[i].categorySumInteraction, - rows[i].categorySelfpromo, - rows[i].categoryMusicOfftopic, - ]; - } - } - //send this result - res.send({ - userNames, - viewCounts, - totalSubmissions, - minutesSaved, - categoryStats - }); -} \ No newline at end of file + res.send(stats); +} diff --git a/src/utils/createMemoryCache.js b/src/utils/createMemoryCache.js new file mode 100644 index 0000000..0f089ce --- /dev/null +++ b/src/utils/createMemoryCache.js @@ -0,0 +1,42 @@ +module.exports = function createMemoryCache(memoryFn, cacheTimeMs) { + // holds the promise results + const cache = new Map; + // holds the promises that are not fulfilled + const promiseMemory = new Map; + return function (...args) { + // create cacheKey by joining arguments as string + const cacheKey = args.join('.'); + // check if promising is already running + if (promiseMemory.has(cacheKey)) { + return promiseMemory.get(cacheKey); + } + else { + // check if result is in cache + if (cache.has(cacheKey)) { + const cacheItem = cache.get(cacheKey); + const now = Date.now(); + // check if cache is valid + if (!(cacheItem.cacheTime + cacheTimeMs < now)) { + return Promise.resolve(cacheItem.result); + } + } + // create new promise + const promise = new Promise(async (resolve, reject) => { + resolve((await memoryFn(...args))); + }); + // store promise reference until fulfilled + promiseMemory.set(cacheKey, promise); + return promise.then(result => { + // store promise result in cache + cache.set(cacheKey, { + result, + cacheTime: Date.now(), + }); + // remove fulfilled promise from memory + promiseMemory.delete(cacheKey); + // return promise result + return result; + }); + } + }; +}; From fb7ff50febbb45ddb2cdcfc8d7a7c60d786f9078 Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Fri, 9 Oct 2020 08:58:15 +0200 Subject: [PATCH 2/3] Update src/utils/createMemoryCache.js Co-authored-by: Ajay Ramachandran --- src/utils/createMemoryCache.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/createMemoryCache.js b/src/utils/createMemoryCache.js index 0f089ce..77053ae 100644 --- a/src/utils/createMemoryCache.js +++ b/src/utils/createMemoryCache.js @@ -1,8 +1,8 @@ module.exports = function createMemoryCache(memoryFn, cacheTimeMs) { // holds the promise results - const cache = new Map; + const cache = new Map(); // holds the promises that are not fulfilled - const promiseMemory = new Map; + const promiseMemory = new Map(); return function (...args) { // create cacheKey by joining arguments as string const cacheKey = args.join('.'); From 41dc16453eadfd8240e5565acaa597df174d5eee Mon Sep 17 00:00:00 2001 From: Nanobyte Date: Fri, 9 Oct 2020 08:58:31 +0200 Subject: [PATCH 3/3] Update src/utils/createMemoryCache.js Co-authored-by: Ajay Ramachandran --- src/utils/createMemoryCache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/createMemoryCache.js b/src/utils/createMemoryCache.js index 77053ae..ec4402d 100644 --- a/src/utils/createMemoryCache.js +++ b/src/utils/createMemoryCache.js @@ -3,7 +3,7 @@ module.exports = function createMemoryCache(memoryFn, cacheTimeMs) { const cache = new Map(); // holds the promises that are not fulfilled const promiseMemory = new Map(); - return function (...args) { + return (...args) => { // create cacheKey by joining arguments as string const cacheKey = args.join('.'); // check if promising is already running