Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-08 12:37:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #134 from Joe-Dowd/youtube-cache

Browse Source 
Youtube cache
This commit is contained in:
Ajay Ramachandran
2020-09-11 17:37:21 -04:00
committed by GitHub
parent 1536cc085c bbe31149b4
commit fdc20e28ff
10 changed files with 306 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@@ -4,6 +4,8 @@ COPY package.json .
RUN npm install
COPY index.js .
COPY src src
RUN mkdir databases
COPY databases/*.sql databases/
COPY entrypoint.sh .
EXPOSE 8080
CMD ./entrypoint.sh

240
package-lock.json generated
View File

@@ -68,6 +68,11 @@
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"asap": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz",
"integrity": "sha1-sqRdpf36ILBJb8N2jMJ8EvqRan0="
},
"asn1": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@@ -96,12 +101,39 @@
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz",
"integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug=="
},
"babel-runtime": {
"version": "6.26.0",
"resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
"integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
"requires": {
"core-js": "^2.4.0",
"regenerator-runtime": "^0.11.0"
}
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
},
"barrage": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/barrage/-/barrage-1.1.0.tgz",
"integrity": "sha1-b1OEdlxGP3cYAS8WDczwM/xvw5Q=",
"requires": {
"promise": "^6.0.0"
},
"dependencies": {
"promise": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz",
"integrity": "sha1-LOcp9rlLRcJoka0GAsXJDgTG7vY=",
"requires": {
"asap": "~1.0.0"
}
}
}
},
"base64url": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/base64url/-/base64url-1.0.6.tgz",
@@ -128,6 +160,11 @@
"tar": "^4.4.10"
}
},
"bignumber.js": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
"integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="
},
"binary-extensions": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
@@ -269,6 +306,11 @@
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
},
"buffer-from": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@@ -471,6 +513,11 @@
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
},
"core-js": {
"version": "2.6.11",
"resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz",
"integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg=="
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@@ -559,6 +606,11 @@
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"denque": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
"integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -877,6 +929,11 @@
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
"dev": true
},
"get-port": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz",
"integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw="
},
"get-stdin": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
@@ -1744,6 +1801,56 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"mysql": {
"version": "2.18.1",
"resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
"integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
"requires": {
"bignumber.js": "9.0.0",
"readable-stream": "2.3.7",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
},
"dependencies": {
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"negotiator": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@@ -2005,6 +2112,21 @@
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
"integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
},
"promise": {
"version": "7.3.1",
"resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
"integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
"requires": {
"asap": "~2.0.3"
},
"dependencies": {
"asap": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
}
}
},
"proxy-addr": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
@@ -2089,6 +2211,40 @@
"picomatch": "^2.0.4"
}
},
"redis": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
"integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
"requires": {
"denque": "^1.4.1",
"redis-commands": "^1.5.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0"
}
},
"redis-commands": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz",
"integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"requires": {
"redis-errors": "^1.0.0"
}
},
"regenerator-runtime": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
"integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg=="
},
"registry-auth-token": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz",
@@ -2277,6 +2433,11 @@
"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
"dev": true
},
"sqlstring": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
"integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
},
"sshpk": {
"version": "1.16.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
@@ -2414,6 +2575,75 @@
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
},
"sync-mysql": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/sync-mysql/-/sync-mysql-3.0.1.tgz",
"integrity": "sha1-B6ArGSk04EzZDtU+lvCfgbFM68g=",
"requires": {
"babel-runtime": "^6.18.0",
"concat-stream": "^1.6.0",
"sync-rpc": "^1.1.1",
"then-mysql": "^1.1.1"
},
"dependencies": {
"concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"requires": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
}
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"sync-rpc": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz",
"integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==",
"requires": {
"get-port": "^3.1.0"
}
},
"tar": {
"version": "4.4.13",
"resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz",
@@ -2437,6 +2667,16 @@
"execa": "^0.7.0"
}
},
"then-mysql": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/then-mysql/-/then-mysql-1.1.1.tgz",
"integrity": "sha1-p1Q2zGXyvm37t3+HIg6YKuDjYR0=",
"requires": {
"barrage": "^1.1.0",
"mysql": "^2.10.0",
"promise": "^7.1.1"
}
},
"timed-out": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz",

1
package.json
View File

@@ -16,6 +16,7 @@
"express": "^4.17.1",
"http": "0.0.0",
"iso8601-duration": "^1.2.0",
"redis": "^3.0.2",
"sync-mysql": "^3.0.1",
"uuid": "^3.3.2",
"youtube-api": "^2.0.10"

1
src/app.js
View File

@@ -2,6 +2,7 @@ var express = require('express');
// Create a service (the app object is just a callback).
var app = express();
var config = require('./config.js');
var redis = require('./utils/redis.js');
// Middleware
var corsMiddleware = require('./middleware/cors.js');

11
src/routes/postSkipSegments.js
View File

@@ -48,10 +48,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) {
if (config.youtubeAPIKey !== null) {
let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]);
YouTubeAPI.videos.list({
part: "snippet",
id: videoID
}, function (err, data) {
YouTubeAPI.listVideos(videoID, "snippet", (err, data) => {
if (err || data.items.length === 0) {
err && logger.error(err);
return;
@@ -108,10 +105,7 @@ async function autoModerateSubmission(submission, callback) {
// Get the video information from the youtube API
if (config.youtubeAPIKey !== null) {
let {err, data} = await new Promise((resolve, reject) => {
YouTubeAPI.videos.list({
part: "contentDetails",
id: submission.videoID
}, (err, data) => resolve({err, data}));
YouTubeAPI.listVideos(submission.videoID, "contentDetails", (err, data) => resolve({err, data}));
});
if (err) {
@@ -134,7 +128,6 @@ async function autoModerateSubmission(submission, callback) {
}
}
}
} else {
logger.debug("Skipped YouTube API");

5
src/routes/voteOnSponsorTime.js
View File

@@ -48,10 +48,7 @@ function sendWebhooks(voteData) {
}
if (config.youtubeAPIKey !== null) {
YouTubeAPI.videos.list({
part: "snippet",
id: submissionInfoRow.videoID
}, function (err, data) {
YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => {
if (err || data.items.length === 0) {
err && logger.error(err);
return;

15
src/utils/redis.js Normal file
View File

@@ -0,0 +1,15 @@
const config = require('../config.js');
const logger = require('./logger.js');
if (config.redis) {
const redis = require('redis');
logger.info('Connected to redis');
const client = redis.createClient(config.redis);
module.exports = client;
} else {
module.exports = {
get: (key, callback) => {
callback((true));
}
};
}

37
src/utils/youtubeAPI.js
View File

@@ -2,6 +2,8 @@ var config = require('../config.js');
// YouTube API
const YouTubeAPI = require("youtube-api");
const redis = require('./redis.js');
const logger = require('./logger.js');
var exportObject;
// If in test mode, return a mocked youtube object
@@ -14,6 +16,41 @@ if (config.mode === "test") {
key: config.youtubeAPIKey
});
exportObject = YouTubeAPI;
// YouTubeAPI.videos.list wrapper with cacheing
exportObject.listVideos = (videoID, part, callback) => {
let redisKey = "youtube.video." + videoID + "." + part;
redis.get(redisKey, (getErr, result) => {
if (getErr || !result) {
logger.debug("redis: no cache for video information: " + videoID);
YouTubeAPI.videos.list({
part,
id: videoID
}, (ytErr, data) => {
if (!ytErr) {
// Only set cache if data returned
if (data.items.length > 0) {
redis.set(redisKey, JSON.stringify(data), (setErr) => {
if(setErr) {
logger.warn(setErr);
} else {
logger.debug("redis: video information cache set for: " + videoID);
}
callback(false, data); // don't fail
});
} else {
callback(false, data); // don't fail
}
} else {
callback(ytErr, data)
}
});
} else {
logger.debug("redis: fetched video information from cache: " + videoID);
callback(getErr, JSON.parse(result));
}
});
};
}
module.exports = exportObject;

2
test.js
View File

@@ -32,7 +32,7 @@ var mockServer = createMockServer(() => {
var server = createServer(() => {
logger.info("Started main HTTP server");
// Run the tests.
mocha.run(function(failures) {
mocha.run((failures) => {
mockServer.close();
server.close();
process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures

6
test/youtubeMock.js
View File

@@ -8,6 +8,12 @@ YouTubeAPI.videos.list({
// https://developers.google.com/youtube/v3/docs/videos
const YouTubeAPI = {
listVideos: (id, part, callback) => {
YouTubeAPI.videos.list({
part: part,
id: id
}, callback);
},
videos: {
list: (obj, callback) => {
if (obj.id === "knownWrongID") {