diff --git a/package-lock.json b/package-lock.json index d314594..0878451 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,6 +68,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "asap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz", + "integrity": "sha1-sqRdpf36ILBJb8N2jMJ8EvqRan0=" + }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -96,12 +101,39 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==" }, + "babel-runtime": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", + "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", + "requires": { + "core-js": "^2.4.0", + "regenerator-runtime": "^0.11.0" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "barrage": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/barrage/-/barrage-1.1.0.tgz", + "integrity": "sha1-b1OEdlxGP3cYAS8WDczwM/xvw5Q=", + "requires": { + "promise": "^6.0.0" + }, + "dependencies": { + "promise": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz", + "integrity": "sha1-LOcp9rlLRcJoka0GAsXJDgTG7vY=", + "requires": { + "asap": "~1.0.0" + } + } + } + }, "base64url": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/base64url/-/base64url-1.0.6.tgz", @@ -128,6 +160,11 @@ "tar": "^4.4.10" } }, + "bignumber.js": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", + "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" + }, "binary-extensions": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", @@ -269,6 +306,11 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -471,6 +513,11 @@ "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, + "core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==" + }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", @@ -559,6 +606,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "denque": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", + "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -877,6 +929,11 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, + "get-port": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", + "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=" + }, "get-stdin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", @@ -1744,6 +1801,56 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "mysql": { + "version": "2.18.1", + "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", + "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==", + "requires": { + "bignumber.js": "9.0.0", + "readable-stream": "2.3.7", + "safe-buffer": "5.1.2", + "sqlstring": "2.3.1" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "negotiator": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", @@ -2005,6 +2112,21 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" }, + "promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "requires": { + "asap": "~2.0.3" + }, + "dependencies": { + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + } + } + }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -2089,6 +2211,40 @@ "picomatch": "^2.0.4" } }, + "redis": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", + "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "requires": { + "denque": "^1.4.1", + "redis-commands": "^1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + } + }, + "redis-commands": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz", + "integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, + "regenerator-runtime": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", + "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==" + }, "registry-auth-token": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", @@ -2277,6 +2433,11 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, + "sqlstring": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", + "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -2414,6 +2575,75 @@ "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, + "sync-mysql": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/sync-mysql/-/sync-mysql-3.0.1.tgz", + "integrity": "sha1-B6ArGSk04EzZDtU+lvCfgbFM68g=", + "requires": { + "babel-runtime": "^6.18.0", + "concat-stream": "^1.6.0", + "sync-rpc": "^1.1.1", + "then-mysql": "^1.1.1" + }, + "dependencies": { + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "sync-rpc": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz", + "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==", + "requires": { + "get-port": "^3.1.0" + } + }, "tar": { "version": "4.4.13", "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", @@ -2437,6 +2667,16 @@ "execa": "^0.7.0" } }, + "then-mysql": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/then-mysql/-/then-mysql-1.1.1.tgz", + "integrity": "sha1-p1Q2zGXyvm37t3+HIg6YKuDjYR0=", + "requires": { + "barrage": "^1.1.0", + "mysql": "^2.10.0", + "promise": "^7.1.1" + } + }, "timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", diff --git a/package.json b/package.json index 9c4e7b5..59acb78 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "express": "^4.17.1", "http": "0.0.0", "iso8601-duration": "^1.2.0", + "redis": "^3.0.2", "sync-mysql": "^3.0.1", "uuid": "^3.3.2", "youtube-api": "^2.0.10" diff --git a/src/app.js b/src/app.js index bf4dce8..a102a76 100644 --- a/src/app.js +++ b/src/app.js @@ -2,6 +2,7 @@ var express = require('express'); // Create a service (the app object is just a callback). var app = express(); var config = require('./config.js'); +var redis = require('./utils/redis.js'); // Middleware var corsMiddleware = require('./middleware/cors.js'); diff --git a/src/routes/postSkipSegments.js b/src/routes/postSkipSegments.js index 48fbb76..f135495 100644 --- a/src/routes/postSkipSegments.js +++ b/src/routes/postSkipSegments.js @@ -48,10 +48,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) { if (config.youtubeAPIKey !== null) { let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]); - YouTubeAPI.videos.list({ - part: "snippet", - id: videoID - }, function (err, data) { + YouTubeAPI.listVideos(videoID, "snippet", (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; @@ -108,10 +105,7 @@ async function autoModerateSubmission(submission, callback) { // Get the video information from the youtube API if (config.youtubeAPIKey !== null) { let {err, data} = await new Promise((resolve, reject) => { - YouTubeAPI.videos.list({ - part: "contentDetails", - id: submission.videoID - }, (err, data) => resolve({err, data})); + YouTubeAPI.listVideos(submission.videoID, "contentDetails", (err, data) => resolve({err, data})); }); if (err) { @@ -134,7 +128,6 @@ async function autoModerateSubmission(submission, callback) { } } } - } else { logger.debug("Skipped YouTube API"); @@ -214,6 +207,10 @@ module.exports = async function postSkipSegments(req, res) { if (isNaN(startTime) || isNaN(endTime) || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) { //invalid request + logger.debug(videoID); + logger.debug(userID); + logger.debug(JSON.stringify(segments)); + logger.debug('400'); res.sendStatus(400); return; } diff --git a/src/routes/voteOnSponsorTime.js b/src/routes/voteOnSponsorTime.js index 5ebdc6d..43fb63b 100644 --- a/src/routes/voteOnSponsorTime.js +++ b/src/routes/voteOnSponsorTime.js @@ -48,10 +48,7 @@ function sendWebhooks(voteData) { } if (config.youtubeAPIKey !== null) { - YouTubeAPI.videos.list({ - part: "snippet", - id: submissionInfoRow.videoID - }, function (err, data) { + YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => { if (err || data.items.length === 0) { err && logger.error(err); return; diff --git a/src/utils/redis.js b/src/utils/redis.js new file mode 100644 index 0000000..2d822c6 --- /dev/null +++ b/src/utils/redis.js @@ -0,0 +1,15 @@ +const config = require('../config.js'); +const logger = require('./logger.js'); + +if (config.redis) { + const redis = require('redis'); + logger.info('Connected to redis'); + const client = redis.createClient(config.redis); + module.exports = client; +} else { + module.exports = { + get: (key, callback) => { + callback((true)); + } + }; +} diff --git a/src/utils/youtubeAPI.js b/src/utils/youtubeAPI.js index c28f882..c1a47d6 100644 --- a/src/utils/youtubeAPI.js +++ b/src/utils/youtubeAPI.js @@ -2,6 +2,8 @@ var config = require('../config.js'); // YouTube API const YouTubeAPI = require("youtube-api"); +const redis = require('./redis.js'); +const logger = require('./logger.js'); var exportObject; // If in test mode, return a mocked youtube object @@ -14,6 +16,44 @@ if (config.mode === "test") { key: config.youtubeAPIKey }); exportObject = YouTubeAPI; + + // YouTubeAPI.videos.list wrapper with cacheing + exportObject.listVideos = (videoID, part, callback) => { + let redisKey = "youtube.video." + videoID + "." + part; + redis.get(redisKey, (getErr, result) => { + if (getErr || !result) { + logger.debug("redis: no cache for video information: " + videoID); + YouTubeAPI.videos.list({ + part, + id: videoID + }, (ytErr, data) => { + if (!ytErr) { + // Only set cache if data returned + if (data.items.length > 0) { + redis.set(redisKey, JSON.stringify(data), (setErr) => { + logger.debug("redis: video information cache set for: " + videoID); + setErr && logger.warn(setErr); + callback(false, data); // don't fail + }); + } else { + callback(false, data); // don't fail + } + } else { + callback(ytErr, data) + } + }); + } else { + logger.debug("redis: fetched video information from cache: " + videoID); + callback(getErr, JSON.parse(result)); + } + }); + }; } +/* YouTubeAPI.videos.list({ + part: "snippet", + id: videoID + }, function (err, data) {*/ + + module.exports = exportObject; \ No newline at end of file diff --git a/test.js b/test.js index 9810786..f4e54ff 100644 --- a/test.js +++ b/test.js @@ -32,7 +32,7 @@ var mockServer = createMockServer(() => { var server = createServer(() => { logger.info("Started main HTTP server"); // Run the tests. - mocha.run(function(failures) { + mocha.run((failures) => { mockServer.close(); server.close(); process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures diff --git a/test/youtubeMock.js b/test/youtubeMock.js index dd64898..cc722d9 100644 --- a/test/youtubeMock.js +++ b/test/youtubeMock.js @@ -8,6 +8,12 @@ YouTubeAPI.videos.list({ // https://developers.google.com/youtube/v3/docs/videos const YouTubeAPI = { + listVideos: (id, part, callback) => { + YouTubeAPI.videos.list({ + part: part, + id: id + }, callback); + }, videos: { list: (obj, callback) => { if (obj.id === "knownWrongID") {