Add disk caching for youtube api calls

Fixes https://github.com/ajayyy/SponsorBlockServer/issues/239

.gitignore

@@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
 test/databases/sponsorTimes.db-shm
 test/databases/sponsorTimes.db-wal
 test/databases/private.db
+databases/cache
 docker/database-export
 
 # Config files

package-lock.json

@@ -9,6 +9,7 @@
       "version": "0.1.0",
       "license": "MIT",
       "dependencies": {
+        "@ajayyy/lru-diskcache": "^1.1.9",
         "abort-controller": "^3.0.0",
         "better-sqlite3": "^7.1.5",
         "dotenv": "^8.2.0",
@@ -40,6 +41,20 @@
         "typescript": "^4.0.3"
       }
     },
+    "node_modules/@ajayyy/lru-diskcache": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/@ajayyy/lru-diskcache/-/lru-diskcache-1.1.9.tgz",
+      "integrity": "sha512-RT04axsNvACDXYW2vSdBnrIYFTTk4jpHg39P29BQv/vZmckLyo2LTH7wZhfRui7Wlllrff2Z12T418pdGhohRw==",
+      "dependencies": {
+        "crc": "^3.4.0",
+        "fs-extra": "^0.26.5",
+        "fswrite-stream": "^1.0.0",
+        "lodash": "^4.17.10",
+        "lru-cache": "^4.1.5",
+        "q": "^1.4.1",
+        "tmp": "^0.0.28"
+      }
+    },
     "node_modules/@sinonjs/commons": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.1.tgz",
@@ -452,8 +467,7 @@
     "node_modules/balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
-      "dev": true
     },
     "node_modules/barrage": {
       "version": "1.1.0",
@@ -616,7 +630,6 @@
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dev": true,
       "dependencies": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -858,8 +871,7 @@
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
-      "dev": true
     },
     "node_modules/configstore": {
       "version": "3.1.2",
@@ -930,6 +942,14 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
+    "node_modules/crc": {
+      "version": "3.8.0",
+      "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz",
+      "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==",
+      "dependencies": {
+        "buffer": "^5.1.0"
+      }
+    },
     "node_modules/create-error-class": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -1318,6 +1338,18 @@
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
+    "node_modules/fs-extra": {
+      "version": "0.26.7",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.26.7.tgz",
+      "integrity": "sha1-muH92UiXeY7at20JGM9C0MMYT6k=",
+      "dependencies": {
+        "graceful-fs": "^4.1.2",
+        "jsonfile": "^2.1.0",
+        "klaw": "^1.0.0",
+        "path-is-absolute": "^1.0.0",
+        "rimraf": "^2.2.8"
+      }
+    },
     "node_modules/fs-minipass": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -1332,8 +1364,7 @@
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
-      "dev": true
     },
     "node_modules/fsevents": {
       "version": "2.3.2",
@@ -1349,6 +1380,14 @@
         "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
       }
     },
+    "node_modules/fswrite-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fswrite-stream/-/fswrite-stream-1.0.0.tgz",
+      "integrity": "sha1-esPP5gkt3/mYQ1Z1UCOJ5lKjb64=",
+      "dependencies": {
+        "length-stream": "^0.1.1"
+      }
+    },
     "node_modules/gauge": {
       "version": "2.7.4",
       "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
@@ -1423,7 +1462,6 @@
       "version": "7.1.6",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
       "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
-      "dev": true,
       "dependencies": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -1488,8 +1526,7 @@
     "node_modules/graceful-fs": {
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
-      "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==",
+      "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ=="
-      "dev": true
     },
     "node_modules/growl": {
       "version": "1.10.5",
@@ -1601,7 +1638,6 @@
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
       "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
-      "dev": true,
       "dependencies": {
         "once": "^1.3.0",
         "wrappy": "1"
@@ -1795,12 +1831,28 @@
         "js-yaml": "bin/js-yaml.js"
       }
     },
+    "node_modules/jsonfile": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
+      "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=",
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "node_modules/just-extend": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz",
       "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==",
       "dev": true
     },
+    "node_modules/klaw": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz",
+      "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=",
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.9"
+      }
+    },
     "node_modules/latest-version": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz",
@@ -1813,6 +1865,17 @@
         "node": ">=4"
       }
     },
+    "node_modules/length-stream": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/length-stream/-/length-stream-0.1.1.tgz",
+      "integrity": "sha1-5ySxvi46lh1MQxNDfkllaRD65tY=",
+      "dependencies": {
+        "pass-stream": "~0.1.0"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
     "node_modules/locate-path": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@@ -1828,6 +1891,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
     "node_modules/lodash.get": {
       "version": "4.4.2",
       "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
@@ -1859,7 +1927,6 @@
       "version": "4.1.5",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
       "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
-      "dev": true,
       "dependencies": {
         "pseudomap": "^1.0.2",
         "yallist": "^2.1.2"
@@ -1868,8 +1935,7 @@
     "node_modules/lru-cache/node_modules/yallist": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
-      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
+      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
-      "dev": true
     },
     "node_modules/make-dir": {
       "version": "1.3.0",
@@ -1955,7 +2021,6 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
       "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
-      "dev": true,
       "dependencies": {
         "brace-expansion": "^1.1.7"
       },
@@ -2393,6 +2458,14 @@
         "wrappy": "1"
       }
     },
+    "node_modules/os-tmpdir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
+      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/p-finally": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
@@ -2460,6 +2533,23 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/pass-stream": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/pass-stream/-/pass-stream-0.1.5.tgz",
+      "integrity": "sha1-njr6TVglzdE3YHW9tW3jbfszZJ8=",
+      "dependencies": {
+        "readable-stream": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz"
+      },
+      "engines": {
+        "node": ">=0.8"
+      }
+    },
+    "node_modules/pass-stream/node_modules/readable-stream": {
+      "version": "1.0.2",
+      "resolved": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz",
+      "integrity": "sha512-4DfqsAX+1OToUZ/uHLsC3NUHN7wCn2BNfkG/Eyr2ii4c4suSipjLFbVymS7pj49+aRcSutnK9dIuHS3X7fwApQ==",
+      "license": "BSD"
+    },
     "node_modules/path-exists": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -2473,7 +2563,6 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
       "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
-      "dev": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -2683,8 +2772,7 @@
     "node_modules/pseudomap": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
-      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=",
+      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
-      "dev": true
     },
     "node_modules/pstree.remy": {
       "version": "1.1.7",
@@ -2701,6 +2789,15 @@
         "once": "^1.3.1"
       }
     },
+    "node_modules/q": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
+      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=",
+      "engines": {
+        "node": ">=0.6.0",
+        "teleport": ">=0.2.0"
+      }
+    },
     "node_modules/qs": {
       "version": "6.7.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@@ -2857,6 +2954,17 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/rimraf": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
+      "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
+      "dependencies": {
+        "glob": "^7.1.3"
+      },
+      "bin": {
+        "rimraf": "bin.js"
+      }
+    },
     "node_modules/safe-buffer": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
@@ -3301,6 +3409,17 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/tmp": {
+      "version": "0.0.28",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz",
+      "integrity": "sha1-Fyc1t/YU6nrzlmT6hM8N5OUV0SA=",
+      "dependencies": {
+        "os-tmpdir": "~1.0.1"
+      },
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/to-regex-range": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@@ -3813,6 +3932,20 @@
     }
   },
   "dependencies": {
+    "@ajayyy/lru-diskcache": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/@ajayyy/lru-diskcache/-/lru-diskcache-1.1.9.tgz",
+      "integrity": "sha512-RT04axsNvACDXYW2vSdBnrIYFTTk4jpHg39P29BQv/vZmckLyo2LTH7wZhfRui7Wlllrff2Z12T418pdGhohRw==",
+      "requires": {
+        "crc": "^3.4.0",
+        "fs-extra": "^0.26.5",
+        "fswrite-stream": "^1.0.0",
+        "lodash": "^4.17.10",
+        "lru-cache": "^4.1.5",
+        "q": "^1.4.1",
+        "tmp": "^0.0.28"
+      }
+    },
     "@sinonjs/commons": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.1.tgz",
@@ -4204,8 +4337,7 @@
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
-      "dev": true
     },
     "barrage": {
       "version": "1.1.0",
@@ -4349,7 +4481,6 @@
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "dev": true,
       "requires": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -4526,8 +4657,7 @@
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
-      "dev": true
     },
     "configstore": {
       "version": "3.1.2",
@@ -4588,6 +4718,14 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
+    "crc": {
+      "version": "3.8.0",
+      "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz",
+      "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==",
+      "requires": {
+        "buffer": "^5.1.0"
+      }
+    },
     "create-error-class": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -4888,6 +5026,18 @@
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
+    "fs-extra": {
+      "version": "0.26.7",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.26.7.tgz",
+      "integrity": "sha1-muH92UiXeY7at20JGM9C0MMYT6k=",
+      "requires": {
+        "graceful-fs": "^4.1.2",
+        "jsonfile": "^2.1.0",
+        "klaw": "^1.0.0",
+        "path-is-absolute": "^1.0.0",
+        "rimraf": "^2.2.8"
+      }
+    },
     "fs-minipass": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -4899,8 +5049,7 @@
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
-      "dev": true
     },
     "fsevents": {
       "version": "2.3.2",
@@ -4909,6 +5058,14 @@
       "dev": true,
       "optional": true
     },
+    "fswrite-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fswrite-stream/-/fswrite-stream-1.0.0.tgz",
+      "integrity": "sha1-esPP5gkt3/mYQ1Z1UCOJ5lKjb64=",
+      "requires": {
+        "length-stream": "^0.1.1"
+      }
+    },
     "gauge": {
       "version": "2.7.4",
       "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
@@ -4970,7 +5127,6 @@
       "version": "7.1.6",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
       "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
-      "dev": true,
       "requires": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -5020,8 +5176,7 @@
     "graceful-fs": {
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
-      "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==",
+      "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ=="
-      "dev": true
     },
     "growl": {
       "version": "1.10.5",
@@ -5098,7 +5253,6 @@
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
       "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
-      "dev": true,
       "requires": {
         "once": "^1.3.0",
         "wrappy": "1"
@@ -5244,12 +5398,28 @@
         "argparse": "^2.0.1"
       }
     },
+    "jsonfile": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
+      "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=",
+      "requires": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "just-extend": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz",
       "integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==",
       "dev": true
     },
+    "klaw": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz",
+      "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=",
+      "requires": {
+        "graceful-fs": "^4.1.9"
+      }
+    },
     "latest-version": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz",
@@ -5259,6 +5429,14 @@
         "package-json": "^4.0.0"
       }
     },
+    "length-stream": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/length-stream/-/length-stream-0.1.1.tgz",
+      "integrity": "sha1-5ySxvi46lh1MQxNDfkllaRD65tY=",
+      "requires": {
+        "pass-stream": "~0.1.0"
+      }
+    },
     "locate-path": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@@ -5268,6 +5446,11 @@
         "p-locate": "^5.0.0"
       }
     },
+    "lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
     "lodash.get": {
       "version": "4.4.2",
       "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
@@ -5293,7 +5476,6 @@
       "version": "4.1.5",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
       "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
-      "dev": true,
       "requires": {
         "pseudomap": "^1.0.2",
         "yallist": "^2.1.2"
@@ -5302,8 +5484,7 @@
         "yallist": {
           "version": "2.1.2",
           "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
-          "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
+          "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
-          "dev": true
         }
       }
     },
@@ -5364,7 +5545,6 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
       "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
-      "dev": true,
       "requires": {
         "brace-expansion": "^1.1.7"
       }
@@ -5708,6 +5888,11 @@
         "wrappy": "1"
       }
     },
+    "os-tmpdir": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
+      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
+    },
     "p-finally": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
@@ -5754,6 +5939,20 @@
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
       "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
     },
+    "pass-stream": {
+      "version": "0.1.5",
+      "resolved": "https://registry.npmjs.org/pass-stream/-/pass-stream-0.1.5.tgz",
+      "integrity": "sha1-njr6TVglzdE3YHW9tW3jbfszZJ8=",
+      "requires": {
+        "readable-stream": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz",
+          "integrity": "sha512-4DfqsAX+1OToUZ/uHLsC3NUHN7wCn2BNfkG/Eyr2ii4c4suSipjLFbVymS7pj49+aRcSutnK9dIuHS3X7fwApQ=="
+        }
+      }
+    },
     "path-exists": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -5763,8 +5962,7 @@
     "path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
-      "dev": true
     },
     "path-is-inside": {
       "version": "1.0.2",
@@ -5931,8 +6129,7 @@
     "pseudomap": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
-      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=",
+      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
-      "dev": true
     },
     "pstree.remy": {
       "version": "1.1.7",
@@ -5949,6 +6146,11 @@
         "once": "^1.3.1"
       }
     },
+    "q": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
+      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
+    },
     "qs": {
       "version": "6.7.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@@ -6068,6 +6270,14 @@
       "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
       "dev": true
     },
+    "rimraf": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
+      "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
+      "requires": {
+        "glob": "^7.1.3"
+      }
+    },
     "safe-buffer": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
@@ -6441,6 +6651,14 @@
       "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=",
       "dev": true
     },
+    "tmp": {
+      "version": "0.0.28",
+      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz",
+      "integrity": "sha1-Fyc1t/YU6nrzlmT6hM8N5OUV0SA=",
+      "requires": {
+        "os-tmpdir": "~1.0.1"
+      }
+    },
     "to-regex-range": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",

package.json

@@ -13,6 +13,7 @@
   "author": "Ajay Ramachandran",
   "license": "MIT",
   "dependencies": {
+    "@ajayyy/lru-diskcache": "^1.1.9",
     "abort-controller": "^3.0.0",
     "better-sqlite3": "^7.1.5",
     "dotenv": "^8.2.0",

src/config.ts

@@ -75,7 +75,8 @@ addDefaults(config, {
         {
             name: "vipUsers"
         }]
-    }
+    },
+    diskCache: null
 });
 
 // Add defaults

src/types/config.model.ts

@@ -1,5 +1,6 @@
 import { PoolConfig } from 'pg';
 import * as redis from 'redis';
+import { CacheOptions } from "@ajayyy/lru-diskcache";
 
 export interface SBSConfig {
     port: number;
@@ -41,6 +42,7 @@ export interface SBSConfig {
     maxRewardTimePerSegmentInSeconds?: number;
     postgres?: PoolConfig;
     dumpDatabase?: DumpDatabase;
+    diskCache: CacheOptions;
 }
 
 export interface WebhookConfig {

src/types/lib.model.ts

@@ -1,5 +1,5 @@
 /**
- * Better ecord that will work with branded types
+ * Better record that will work with branded types
  * Keys still don't work properly though and are always string
  */
 export type SBRecord<K extends string, T> = {

src/utils/diskCache.ts

@@ -0,0 +1,32 @@
+import LRU from "@ajayyy/lru-diskcache";
+import { config } from "../config";
+
+let DiskCache: LRU<string, string>;
+
+if (config.diskCache) {
+    DiskCache = new LRU('./databases/cache', config.diskCache);
+    DiskCache.init();
+} else {
+    DiskCache = {
+        // constructor(rootPath, options): {};
+
+        init() {},
+
+        reset() {},
+
+        has(key) { return false; },
+
+        get(key, opts) { return null; },
+
+        // Returns size
+        set(key, dataOrSteam) { return new Promise((resolve) => 0); },
+
+        del(key) {},
+
+        size() { return 0; },
+
+        prune() {},
+    };
+}
+
+export default DiskCache;

src/utils/youtubeApi.ts

@@ -1,8 +1,8 @@
 import fetch from 'node-fetch';
 import {config} from '../config';
 import {Logger} from './logger';
-import redis from './redis';
 import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
+import DiskCache from './diskCache';
 
 export class YouTubeAPI {
     static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
@@ -10,14 +10,17 @@ export class YouTubeAPI {
             return { err: "Invalid video ID" };
         }
 
-        const redisKey = "yt.newleaf.video." + videoID;
+        const cacheKey = "yt.newleaf.video." + videoID;
         if (!ignoreCache) {
-            const {err, reply} =  await redis.getAsync(redisKey);
+            try {
+                const data = await DiskCache.get(cacheKey);
 
-            if (!err && reply) {
+                if (data) {
-                Logger.debug("redis: no cache for video information: " + videoID);
+                    Logger.debug("YouTube API: cache used for video information: " + videoID);
-
+                    return { err: null, data: JSON.parse(data) }
-                return { err: err?.message, data: JSON.parse(reply) }
+                }
+            } catch (err) {
+                return { err }
             }
         }
 
@@ -33,13 +36,9 @@ export class YouTubeAPI {
                     return { err: data.error, data: null };
                 }
 
-                redis.setAsync(redisKey, JSON.stringify(data)).then((result) => {
+                DiskCache.set(cacheKey, JSON.stringify(data))
-                    if (result?.err) {
+                .catch((err) => Logger.warn(err))
-                        Logger.warn(result?.err.message);
+                .then(() => Logger.debug("YouTube API: video information cache set for: " + videoID));
-                    } else {
-                        Logger.debug("redis: video information cache set for: " + videoID);
-                    }
-                });
                 
                 return { err: false, data };
             } else {