Added youtube api cache (optional)

package-lock.json

@@ -68,6 +68,11 @@
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
       "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
     },
+    "asap": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz",
+      "integrity": "sha1-sqRdpf36ILBJb8N2jMJ8EvqRan0="
+    },
     "asn1": {
       "version": "0.2.4",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz",
@@ -96,12 +101,39 @@
       "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz",
       "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug=="
     },
+    "babel-runtime": {
+      "version": "6.26.0",
+      "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
+      "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
+      "requires": {
+        "core-js": "^2.4.0",
+        "regenerator-runtime": "^0.11.0"
+      }
+    },
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
       "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
       "dev": true
     },
+    "barrage": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/barrage/-/barrage-1.1.0.tgz",
+      "integrity": "sha1-b1OEdlxGP3cYAS8WDczwM/xvw5Q=",
+      "requires": {
+        "promise": "^6.0.0"
+      },
+      "dependencies": {
+        "promise": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz",
+          "integrity": "sha1-LOcp9rlLRcJoka0GAsXJDgTG7vY=",
+          "requires": {
+            "asap": "~1.0.0"
+          }
+        }
+      }
+    },
     "base64url": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/base64url/-/base64url-1.0.6.tgz",
@@ -128,6 +160,11 @@
         "tar": "^4.4.10"
       }
     },
+    "bignumber.js": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
+      "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="
+    },
     "binary-extensions": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
@@ -269,6 +306,11 @@
       "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
       "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk="
     },
+    "buffer-from": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+      "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
+    },
     "bytes": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@@ -471,6 +513,11 @@
       "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
       "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
     },
+    "core-js": {
+      "version": "2.6.11",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz",
+      "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg=="
+    },
     "core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@@ -559,6 +606,11 @@
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
     },
+    "denque": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
+      "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
+    },
     "depd": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -877,6 +929,11 @@
       "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
       "dev": true
     },
+    "get-port": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz",
+      "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw="
+    },
     "get-stdin": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
@@ -1744,6 +1801,56 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
     },
+    "mysql": {
+      "version": "2.18.1",
+      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
+      "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
+      "requires": {
+        "bignumber.js": "9.0.0",
+        "readable-stream": "2.3.7",
+        "safe-buffer": "5.1.2",
+        "sqlstring": "2.3.1"
+      },
+      "dependencies": {
+        "isarray": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+        },
+        "process-nextick-args": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+          "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+        },
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
     "negotiator": {
       "version": "0.6.2",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
@@ -2005,6 +2112,21 @@
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
       "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
     },
+    "promise": {
+      "version": "7.3.1",
+      "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
+      "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
+      "requires": {
+        "asap": "~2.0.3"
+      },
+      "dependencies": {
+        "asap": {
+          "version": "2.0.6",
+          "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
+          "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
+        }
+      }
+    },
     "proxy-addr": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
@@ -2089,6 +2211,40 @@
         "picomatch": "^2.0.4"
       }
     },
+    "redis": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
+      "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
+      "requires": {
+        "denque": "^1.4.1",
+        "redis-commands": "^1.5.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0"
+      }
+    },
+    "redis-commands": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.6.0.tgz",
+      "integrity": "sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ=="
+    },
+    "redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
+    },
+    "redis-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+      "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
+      "requires": {
+        "redis-errors": "^1.0.0"
+      }
+    },
+    "regenerator-runtime": {
+      "version": "0.11.1",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
+      "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg=="
+    },
     "registry-auth-token": {
       "version": "3.4.0",
       "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz",
@@ -2277,6 +2433,11 @@
       "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
       "dev": true
     },
+    "sqlstring": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
+      "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
+    },
     "sshpk": {
       "version": "1.16.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
@@ -2414,6 +2575,75 @@
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
       "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
     },
+    "sync-mysql": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/sync-mysql/-/sync-mysql-3.0.1.tgz",
+      "integrity": "sha1-B6ArGSk04EzZDtU+lvCfgbFM68g=",
+      "requires": {
+        "babel-runtime": "^6.18.0",
+        "concat-stream": "^1.6.0",
+        "sync-rpc": "^1.1.1",
+        "then-mysql": "^1.1.1"
+      },
+      "dependencies": {
+        "concat-stream": {
+          "version": "1.6.2",
+          "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+          "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
+          "requires": {
+            "buffer-from": "^1.0.0",
+            "inherits": "^2.0.3",
+            "readable-stream": "^2.2.2",
+            "typedarray": "^0.0.6"
+          }
+        },
+        "isarray": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+        },
+        "process-nextick-args": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+          "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+        },
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
+    "sync-rpc": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz",
+      "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==",
+      "requires": {
+        "get-port": "^3.1.0"
+      }
+    },
     "tar": {
       "version": "4.4.13",
       "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz",
@@ -2437,6 +2667,16 @@
         "execa": "^0.7.0"
       }
     },
+    "then-mysql": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/then-mysql/-/then-mysql-1.1.1.tgz",
+      "integrity": "sha1-p1Q2zGXyvm37t3+HIg6YKuDjYR0=",
+      "requires": {
+        "barrage": "^1.1.0",
+        "mysql": "^2.10.0",
+        "promise": "^7.1.1"
+      }
+    },
     "timed-out": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz",

package.json

@@ -16,6 +16,7 @@
     "express": "^4.17.1",
     "http": "0.0.0",
     "iso8601-duration": "^1.2.0",
+    "redis": "^3.0.2",
     "sync-mysql": "^3.0.1",
     "uuid": "^3.3.2",
     "youtube-api": "^2.0.10"

src/app.js

@@ -2,6 +2,7 @@ var express = require('express');
 // Create a service (the app object is just a callback).
 var app = express();
 var config = require('./config.js');
+var redis = require('./utils/redis.js');
 
 // Middleware 
 var corsMiddleware = require('./middleware/cors.js');

src/routes/postSkipSegments.js

@@ -48,10 +48,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) {
     if (config.youtubeAPIKey !== null) {
         let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]);
 
-        YouTubeAPI.videos.list({
+        YouTubeAPI.listVideos(videoID, "snippet", (err, data) => {
-            part: "snippet",
-            id: videoID
-        }, function (err, data) {
             if (err || data.items.length === 0) {
                 err && logger.error(err);
                 return;
@@ -108,10 +105,7 @@ async function autoModerateSubmission(submission, callback) {
     // Get the video information from the youtube API
     if (config.youtubeAPIKey !== null) {
         let {err, data} = await new Promise((resolve, reject) => {
-            YouTubeAPI.videos.list({
+            YouTubeAPI.listVideos(submission.videoID, "contentDetails", (err, data) => resolve({err, data}));
-                part: "contentDetails",
-                id: submission.videoID
-            }, (err, data) => resolve({err, data}));
         });
 
         if (err) {
@@ -134,7 +128,6 @@ async function autoModerateSubmission(submission, callback) {
                 }
             }
         }
-         
     } else {
         logger.debug("Skipped YouTube API");
 
@@ -214,6 +207,10 @@ module.exports = async function postSkipSegments(req, res) {
         if (isNaN(startTime) || isNaN(endTime)
                 || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime >= endTime) {
             //invalid request
+            logger.debug(videoID);
+            logger.debug(userID);
+            logger.debug(JSON.stringify(segments));
+            logger.debug('400');
             res.sendStatus(400);
             return;
         }

src/routes/voteOnSponsorTime.js

@@ -48,10 +48,7 @@ function sendWebhooks(voteData) {
         }
 
         if (config.youtubeAPIKey !== null) {
-            YouTubeAPI.videos.list({
+            YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => {
-                part: "snippet",
-                id: submissionInfoRow.videoID
-            }, function (err, data) {
                 if (err || data.items.length === 0) {
                     err && logger.error(err);
                     return;

src/utils/redis.js

@@ -0,0 +1,15 @@
+const config = require('../config.js');
+const logger = require('./logger.js');
+
+if (config.redis) {
+    const redis = require('redis');
+    logger.info('Connected to redis');
+    const client = redis.createClient(config.redis);
+    module.exports = client;
+} else {
+    module.exports = {
+        get: (key, callback) => {
+            callback((true));
+        }
+    };
+}

src/utils/youtubeAPI.js

@@ -2,6 +2,8 @@ var config = require('../config.js');
 
 // YouTube API
 const YouTubeAPI = require("youtube-api");
+const redis = require('./redis.js');
+const logger = require('./logger.js');
 
 var exportObject;
 // If in test mode, return a mocked youtube object
@@ -14,6 +16,44 @@ if (config.mode === "test") {
         key: config.youtubeAPIKey
     });
     exportObject = YouTubeAPI;
+
+    // YouTubeAPI.videos.list wrapper with cacheing
+    exportObject.listVideos = (videoID, part, callback) => {
+        let redisKey = "youtube.video." + videoID + "." + part;
+        redis.get(redisKey, (getErr, result) => {
+            if (getErr || !result) {
+                logger.debug("redis: no cache for video information: " + videoID);
+                YouTubeAPI.videos.list({
+                    part,
+                    id: videoID
+                }, (ytErr, data) => {
+                    if (!ytErr) {
+                        // Only set cache if data returned
+                        if (data.items.length > 0) {
+                            redis.set(redisKey, JSON.stringify(data), (setErr) => {
+                                logger.debug("redis: video information cache set for: " + videoID);
+                                setErr && logger.warn(setErr);
+                                callback(false, data); // don't fail
+                            });
+                        } else {
+                            callback(false, data); // don't fail
+                        }
+                    } else {
+                        callback(ytErr, data)
+                    }
+                });
+            } else {
+                logger.debug("redis: fetched video information from cache: " + videoID);
+                callback(getErr, JSON.parse(result));
+            }
+        });
+    };
 }
 
+/* YouTubeAPI.videos.list({
+            part: "snippet",
+            id: videoID
+        }, function (err, data) {*/
+
+
 module.exports = exportObject;

test.js

@@ -32,7 +32,7 @@ var mockServer = createMockServer(() => {
   var server = createServer(() => {
     logger.info("Started main HTTP server");
     // Run the tests.
-    mocha.run(function(failures) {
+    mocha.run((failures) => {
       mockServer.close();
       server.close();
       process.exitCode = failures ? 1 : 0; // exit with non-zero status if there were failures

test/youtubeMock.js

@@ -8,6 +8,12 @@ YouTubeAPI.videos.list({
  // https://developers.google.com/youtube/v3/docs/videos
 
 const YouTubeAPI = {
+  listVideos: (id, part, callback) => {
+    YouTubeAPI.videos.list({
+      part: part,
+      id: id
+    }, callback);
+  },
   videos: {
     list: (obj, callback) => {
       if (obj.id === "knownWrongID") {