Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 03:26:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add cache for getTopUsers

Browse Source 
See #147
This commit is contained in:
Nanobyte
2020-10-06 23:25:11 +02:00
parent 7dcdc883e4
commit 25b91af8bc
3 changed files with 117 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config.json.example
View File

@@ -22,5 +22,6 @@
"mode": "development",
"readOnly": false,
"webhooks": [],
"categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"] // List of supported categories any other category will be rejected
"categoryList": ["sponsor", "intro", "outro", "interaction", "selfpromo", "music_offtopic"], // List of supported categories any other category will be rejected
"getTopUsersCacheTimeMinutes": 5 // cacheTime for getTopUsers result in minutes
}

133
src/routes/getTopUsers.js
View File

@@ -1,6 +1,68 @@
var db = require('../databases/databases.js').db;
const logger = require('../utils/logger.js');
const createMemoryCache = require('../utils/createMemoryCache.js');
const config = require('../config.js');
module.exports = function getTopUsers (req, res) {
const MILLISECONDS_IN_MINUTE = 60000;
const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
function generateTopUsersStats(sortBy, categoryStatsEnabled = false) {
return new Promise((resolve, reject) => {
const userNames = [];
const viewCounts = [];
const totalSubmissions = [];
const minutesSaved = [];
const categoryStats = categoryStatsEnabled ? [] : undefined;
let additionalFields = '';
if (categoryStatsEnabled) {
additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " +
"SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " +
"SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " +
"SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " +
"SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " +
"SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, ";
}
const rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," +
"SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " +
"SUM(votes) as userVotes, " +
additionalFields +
"IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " +
"LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " +
"WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " +
"GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " +
"ORDER BY " + sortBy + " DESC LIMIT 100", []);
for (let i = 0; i < rows.length; i++) {
userNames[i] = rows[i].userName;
viewCounts[i] = rows[i].viewCount;
totalSubmissions[i] = rows[i].totalSubmissions;
minutesSaved[i] = rows[i].minutesSaved;
if (categoryStatsEnabled) {
categoryStats[i] = [
rows[i].categorySponsor,
rows[i].categorySumIntro,
rows[i].categorySumOutro,
rows[i].categorySumInteraction,
rows[i].categorySelfpromo,
rows[i].categoryMusicOfftopic,
];
}
}
resolve({
userNames,
viewCounts,
totalSubmissions,
minutesSaved,
categoryStats
});
});
}
module.exports = async function getTopUsers (req, res) {
let sortType = req.query.sortType;
let categoryStatsEnabled = req.query.categoryStats;
@@ -9,71 +71,22 @@ module.exports = function getTopUsers (req, res) {
res.sendStatus(400);
return;
}
//setup which sort type to use
let sortBy = "";
let sortBy = '';
if (sortType == 0) {
sortBy = "minutesSaved";
sortBy = 'minutesSaved';
} else if (sortType == 1) {
sortBy = "viewCount";
sortBy = 'viewCount';
} else if (sortType == 2) {
sortBy = "totalSubmissions";
sortBy = 'totalSubmissions';
} else {
//invalid request
res.sendStatus(400);
return;
//invalid request
return res.sendStatus(400);
}
let userNames = [];
let viewCounts = [];
let totalSubmissions = [];
let minutesSaved = [];
let categoryStats = categoryStatsEnabled ? [] : undefined;
let additionalFields = '';
if (categoryStatsEnabled) {
additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " +
"SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " +
"SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " +
"SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " +
"SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " +
"SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, ";
}
let rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," +
"SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " +
"SUM(votes) as userVotes, " +
additionalFields +
"IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " +
"LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " +
"WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " +
"GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " +
"ORDER BY " + sortBy + " DESC LIMIT 100", []);
const stats = await getTopUsersWithCache(sortBy, categoryStatsEnabled);
for (let i = 0; i < rows.length; i++) {
userNames[i] = rows[i].userName;
viewCounts[i] = rows[i].viewCount;
totalSubmissions[i] = rows[i].totalSubmissions;
minutesSaved[i] = rows[i].minutesSaved;
if (categoryStatsEnabled) {
categoryStats[i] = [
rows[i].categorySponsor,
rows[i].categorySumIntro,
rows[i].categorySumOutro,
rows[i].categorySumInteraction,
rows[i].categorySelfpromo,
rows[i].categoryMusicOfftopic,
];
}
}
//send this result
res.send({
userNames,
viewCounts,
totalSubmissions,
minutesSaved,
categoryStats
});
}
res.send(stats);
}

42
src/utils/createMemoryCache.js Normal file
View File

@@ -0,0 +1,42 @@
module.exports = function createMemoryCache(memoryFn, cacheTimeMs) {
// holds the promise results
const cache = new Map;
// holds the promises that are not fulfilled
const promiseMemory = new Map;
return function (...args) {
// create cacheKey by joining arguments as string
const cacheKey = args.join('.');
// check if promising is already running
if (promiseMemory.has(cacheKey)) {
return promiseMemory.get(cacheKey);
}
else {
// check if result is in cache
if (cache.has(cacheKey)) {
const cacheItem = cache.get(cacheKey);
const now = Date.now();
// check if cache is valid
if (!(cacheItem.cacheTime + cacheTimeMs < now)) {
return Promise.resolve(cacheItem.result);
}
}
// create new promise
const promise = new Promise(async (resolve, reject) => {
resolve((await memoryFn(...args)));
});
// store promise reference until fulfilled
promiseMemory.set(cacheKey, promise);
return promise.then(result => {
// store promise result in cache
cache.set(cacheKey, {
result,
cacheTime: Date.now(),
});
// remove fulfilled promise from memory
promiseMemory.delete(cacheKey);
// return promise result
return result;
});
}
};
};