Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 11:36:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add disk caching for youtube api calls

Browse Source 
Fixes https://github.com/ajayyy/SponsorBlockServer/issues/239
This commit is contained in:
Ajay Ramachandran
2021-06-26 23:21:51 -04:00
parent e1cf360825
commit 07ab48da1f
8 changed files with 307 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
test/databases/sponsorTimes.db-shm
test/databases/sponsorTimes.db-wal
test/databases/private.db
databases/cache
docker/database-export
# Config files

292
package-lock.json generated
View File

@@ -9,6 +9,7 @@
"version": "0.1.0",
"license": "MIT",
"dependencies": {
"@ajayyy/lru-diskcache": "^1.1.9",
"abort-controller": "^3.0.0",
"better-sqlite3": "^7.1.5",
"dotenv": "^8.2.0",
@@ -40,6 +41,20 @@
"typescript": "^4.0.3"
}
},
"node_modules/@ajayyy/lru-diskcache": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/@ajayyy/lru-diskcache/-/lru-diskcache-1.1.9.tgz",
"integrity": "sha512-RT04axsNvACDXYW2vSdBnrIYFTTk4jpHg39P29BQv/vZmckLyo2LTH7wZhfRui7Wlllrff2Z12T418pdGhohRw==",
"dependencies": {
"crc": "^3.4.0",
"fs-extra": "^0.26.5",
"fswrite-stream": "^1.0.0",
"lodash": "^4.17.10",
"lru-cache": "^4.1.5",
"q": "^1.4.1",
"tmp": "^0.0.28"
}
},
"node_modules/@sinonjs/commons": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.1.tgz",
@@ -452,8 +467,7 @@
"node_modules/balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"node_modules/barrage": {
"version": "1.1.0",
@@ -616,7 +630,6 @@
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -858,8 +871,7 @@
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"node_modules/configstore": {
"version": "3.1.2",
@@ -930,6 +942,14 @@
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"node_modules/crc": {
"version": "3.8.0",
"resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz",
"integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==",
"dependencies": {
"buffer": "^5.1.0"
}
},
"node_modules/create-error-class": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -1318,6 +1338,18 @@
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"node_modules/fs-extra": {
"version": "0.26.7",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.26.7.tgz",
"integrity": "sha1-muH92UiXeY7at20JGM9C0MMYT6k=",
"dependencies": {
"graceful-fs": "^4.1.2",
"jsonfile": "^2.1.0",
"klaw": "^1.0.0",
"path-is-absolute": "^1.0.0",
"rimraf": "^2.2.8"
}
},
"node_modules/fs-minipass": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -1332,8 +1364,7 @@
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
"dev": true
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"node_modules/fsevents": {
"version": "2.3.2",
@@ -1349,6 +1380,14 @@
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/fswrite-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fswrite-stream/-/fswrite-stream-1.0.0.tgz",
"integrity": "sha1-esPP5gkt3/mYQ1Z1UCOJ5lKjb64=",
"dependencies": {
"length-stream": "^0.1.1"
}
},
"node_modules/gauge": {
"version": "2.7.4",
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
@@ -1423,7 +1462,6 @@
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"dev": true,
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
@@ -1488,8 +1526,7 @@
"node_modules/graceful-fs": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
"integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==",
"dev": true
"integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ=="
},
"node_modules/growl": {
"version": "1.10.5",
@@ -1601,7 +1638,6 @@
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dev": true,
"dependencies": {
"once": "^1.3.0",
"wrappy": "1"
@@ -1795,12 +1831,28 @@
"js-yaml": "bin/js-yaml.js"
}
},
"node_modules/jsonfile": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
"integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=",
"optionalDependencies": {
"graceful-fs": "^4.1.6"
}
},
"node_modules/just-extend": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz",
"integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==",
"dev": true
},
"node_modules/klaw": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz",
"integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=",
"optionalDependencies": {
"graceful-fs": "^4.1.9"
}
},
"node_modules/latest-version": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz",
@@ -1813,6 +1865,17 @@
"node": ">=4"
}
},
"node_modules/length-stream": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/length-stream/-/length-stream-0.1.1.tgz",
"integrity": "sha1-5ySxvi46lh1MQxNDfkllaRD65tY=",
"dependencies": {
"pass-stream": "~0.1.0"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/locate-path": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@@ -1828,6 +1891,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
"node_modules/lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
@@ -1859,7 +1927,6 @@
"version": "4.1.5",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
"integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
"dev": true,
"dependencies": {
"pseudomap": "^1.0.2",
"yallist": "^2.1.2"
@@ -1868,8 +1935,7 @@
"node_modules/lru-cache/node_modules/yallist": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
"integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
"dev": true
"integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
},
"node_modules/make-dir": {
"version": "1.3.0",
@@ -1955,7 +2021,6 @@
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"dependencies": {
"brace-expansion": "^1.1.7"
},
@@ -2393,6 +2458,14 @@
"wrappy": "1"
}
},
"node_modules/os-tmpdir": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
"integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
@@ -2460,6 +2533,23 @@
"node": ">= 0.8"
}
},
"node_modules/pass-stream": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/pass-stream/-/pass-stream-0.1.5.tgz",
"integrity": "sha1-njr6TVglzdE3YHW9tW3jbfszZJ8=",
"dependencies": {
"readable-stream": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz"
},
"engines": {
"node": ">=0.8"
}
},
"node_modules/pass-stream/node_modules/readable-stream": {
"version": "1.0.2",
"resolved": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz",
"integrity": "sha512-4DfqsAX+1OToUZ/uHLsC3NUHN7wCn2BNfkG/Eyr2ii4c4suSipjLFbVymS7pj49+aRcSutnK9dIuHS3X7fwApQ==",
"license": "BSD"
},
"node_modules/path-exists": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -2473,7 +2563,6 @@
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
@@ -2683,8 +2772,7 @@
"node_modules/pseudomap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=",
"dev": true
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
},
"node_modules/pstree.remy": {
"version": "1.1.7",
@@ -2701,6 +2789,15 @@
"once": "^1.3.1"
}
},
"node_modules/q": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
"integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=",
"engines": {
"node": ">=0.6.0",
"teleport": ">=0.2.0"
}
},
"node_modules/qs": {
"version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@@ -2857,6 +2954,17 @@
"node": ">=0.10.0"
}
},
"node_modules/rimraf": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"dependencies": {
"glob": "^7.1.3"
},
"bin": {
"rimraf": "bin.js"
}
},
"node_modules/safe-buffer": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
@@ -3301,6 +3409,17 @@
"node": ">=0.10.0"
}
},
"node_modules/tmp": {
"version": "0.0.28",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz",
"integrity": "sha1-Fyc1t/YU6nrzlmT6hM8N5OUV0SA=",
"dependencies": {
"os-tmpdir": "~1.0.1"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@@ -3813,6 +3932,20 @@
}
},
"dependencies": {
"@ajayyy/lru-diskcache": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/@ajayyy/lru-diskcache/-/lru-diskcache-1.1.9.tgz",
"integrity": "sha512-RT04axsNvACDXYW2vSdBnrIYFTTk4jpHg39P29BQv/vZmckLyo2LTH7wZhfRui7Wlllrff2Z12T418pdGhohRw==",
"requires": {
"crc": "^3.4.0",
"fs-extra": "^0.26.5",
"fswrite-stream": "^1.0.0",
"lodash": "^4.17.10",
"lru-cache": "^4.1.5",
"q": "^1.4.1",
"tmp": "^0.0.28"
}
},
"@sinonjs/commons": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.1.tgz",
@@ -4204,8 +4337,7 @@
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"barrage": {
"version": "1.1.0",
@@ -4349,7 +4481,6 @@
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -4526,8 +4657,7 @@
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"configstore": {
"version": "3.1.2",
@@ -4588,6 +4718,14 @@
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"crc": {
"version": "3.8.0",
"resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz",
"integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==",
"requires": {
"buffer": "^5.1.0"
}
},
"create-error-class": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz",
@@ -4888,6 +5026,18 @@
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"fs-extra": {
"version": "0.26.7",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.26.7.tgz",
"integrity": "sha1-muH92UiXeY7at20JGM9C0MMYT6k=",
"requires": {
"graceful-fs": "^4.1.2",
"jsonfile": "^2.1.0",
"klaw": "^1.0.0",
"path-is-absolute": "^1.0.0",
"rimraf": "^2.2.8"
}
},
"fs-minipass": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -4899,8 +5049,7 @@
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
"dev": true
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"fsevents": {
"version": "2.3.2",
@@ -4909,6 +5058,14 @@
"dev": true,
"optional": true
},
"fswrite-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fswrite-stream/-/fswrite-stream-1.0.0.tgz",
"integrity": "sha1-esPP5gkt3/mYQ1Z1UCOJ5lKjb64=",
"requires": {
"length-stream": "^0.1.1"
}
},
"gauge": {
"version": "2.7.4",
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
@@ -4970,7 +5127,6 @@
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
@@ -5020,8 +5176,7 @@
"graceful-fs": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
"integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==",
"dev": true
"integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ=="
},
"growl": {
"version": "1.10.5",
@@ -5098,7 +5253,6 @@
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dev": true,
"requires": {
"once": "^1.3.0",
"wrappy": "1"
@@ -5244,12 +5398,28 @@
"argparse": "^2.0.1"
}
},
"jsonfile": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
"integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=",
"requires": {
"graceful-fs": "^4.1.6"
}
},
"just-extend": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.1.tgz",
"integrity": "sha512-aWgeGFW67BP3e5181Ep1Fv2v8z//iBJfrvyTnq8wG86vEESwmonn1zPBJ0VfmT9CJq2FIT0VsETtrNFm2a+SHA==",
"dev": true
},
"klaw": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz",
"integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=",
"requires": {
"graceful-fs": "^4.1.9"
}
},
"latest-version": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/latest-version/-/latest-version-3.1.0.tgz",
@@ -5259,6 +5429,14 @@
"package-json": "^4.0.0"
}
},
"length-stream": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/length-stream/-/length-stream-0.1.1.tgz",
"integrity": "sha1-5ySxvi46lh1MQxNDfkllaRD65tY=",
"requires": {
"pass-stream": "~0.1.0"
}
},
"locate-path": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@@ -5268,6 +5446,11 @@
"p-locate": "^5.0.0"
}
},
"lodash": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
"lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
@@ -5293,7 +5476,6 @@
"version": "4.1.5",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
"integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
"dev": true,
"requires": {
"pseudomap": "^1.0.2",
"yallist": "^2.1.2"
@@ -5302,8 +5484,7 @@
"yallist": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
"integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
"dev": true
"integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
}
}
},
@@ -5364,7 +5545,6 @@
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@@ -5708,6 +5888,11 @@
"wrappy": "1"
}
},
"os-tmpdir": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
"integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
},
"p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
@@ -5754,6 +5939,20 @@
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
},
"pass-stream": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/pass-stream/-/pass-stream-0.1.5.tgz",
"integrity": "sha1-njr6TVglzdE3YHW9tW3jbfszZJ8=",
"requires": {
"readable-stream": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz"
},
"dependencies": {
"readable-stream": {
"version": "https://github.com/jeffbski/readable-stream/archive/v1.0.2-object-transform2-ret-self.tar.gz",
"integrity": "sha512-4DfqsAX+1OToUZ/uHLsC3NUHN7wCn2BNfkG/Eyr2ii4c4suSipjLFbVymS7pj49+aRcSutnK9dIuHS3X7fwApQ=="
}
}
},
"path-exists": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -5763,8 +5962,7 @@
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"dev": true
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
},
"path-is-inside": {
"version": "1.0.2",
@@ -5931,8 +6129,7 @@
"pseudomap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=",
"dev": true
"integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
},
"pstree.remy": {
"version": "1.1.7",
@@ -5949,6 +6146,11 @@
"once": "^1.3.1"
}
},
"q": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
"integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
},
"qs": {
"version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
@@ -6068,6 +6270,14 @@
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
"dev": true
},
"rimraf": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"requires": {
"glob": "^7.1.3"
}
},
"safe-buffer": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
@@ -6441,6 +6651,14 @@
"integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=",
"dev": true
},
"tmp": {
"version": "0.0.28",
"resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz",
"integrity": "sha1-Fyc1t/YU6nrzlmT6hM8N5OUV0SA=",
"requires": {
"os-tmpdir": "~1.0.1"
}
},
"to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",

1
package.json
View File

@@ -13,6 +13,7 @@
"author": "Ajay Ramachandran",
"license": "MIT",
"dependencies": {
"@ajayyy/lru-diskcache": "^1.1.9",
"abort-controller": "^3.0.0",
"better-sqlite3": "^7.1.5",
"dotenv": "^8.2.0",

3
src/config.ts
View File

@@ -75,7 +75,8 @@ addDefaults(config, {
{
name: "vipUsers"
}]
}
},
diskCache: null
});
// Add defaults

2
src/types/config.model.ts
View File

@@ -1,5 +1,6 @@
import { PoolConfig } from 'pg';
import * as redis from 'redis';
import { CacheOptions } from "@ajayyy/lru-diskcache";
export interface SBSConfig {
port: number;
@@ -41,6 +42,7 @@ export interface SBSConfig {
maxRewardTimePerSegmentInSeconds?: number;
postgres?: PoolConfig;
dumpDatabase?: DumpDatabase;
diskCache: CacheOptions;
}
export interface WebhookConfig {

2
src/types/lib.model.ts
View File

@@ -1,5 +1,5 @@
/**
* Better ecord that will work with branded types
* Better record that will work with branded types
* Keys still don't work properly though and are always string
*/
export type SBRecord<K extends string, T> = {

32
src/utils/diskCache.ts Normal file
View File

@@ -0,0 +1,32 @@
import LRU from "@ajayyy/lru-diskcache";
import { config } from "../config";
let DiskCache: LRU<string, string>;
if (config.diskCache) {
DiskCache = new LRU('./databases/cache', config.diskCache);
DiskCache.init();
} else {
DiskCache = {
// constructor(rootPath, options): {};
init() {},
reset() {},
has(key) { return false; },
get(key, opts) { return null; },
// Returns size
set(key, dataOrSteam) { return new Promise((resolve) => 0); },
del(key) {},
size() { return 0; },
prune() {},
};
}
export default DiskCache;

27
src/utils/youtubeApi.ts
View File

@@ -1,8 +1,8 @@
import fetch from 'node-fetch';
import {config} from '../config';
import {Logger} from './logger';
import redis from './redis';
import { APIVideoData, APIVideoInfo } from '../types/youtubeApi.model';
import DiskCache from './diskCache';
export class YouTubeAPI {
static async listVideos(videoID: string, ignoreCache = false): Promise<APIVideoInfo> {
@@ -10,14 +10,17 @@ export class YouTubeAPI {
return { err: "Invalid video ID" };
}
const redisKey = "yt.newleaf.video." + videoID;
const cacheKey = "yt.newleaf.video." + videoID;
if (!ignoreCache) {
const {err, reply} = await redis.getAsync(redisKey);
try {
const data = await DiskCache.get(cacheKey);
if (!err && reply) {
Logger.debug("redis: no cache for video information: " + videoID);
return { err: err?.message, data: JSON.parse(reply) }
if (data) {
Logger.debug("YouTube API: cache used for video information: " + videoID);
return { err: null, data: JSON.parse(data) }
}
} catch (err) {
return { err }
}
}
@@ -33,13 +36,9 @@ export class YouTubeAPI {
return { err: data.error, data: null };
}
redis.setAsync(redisKey, JSON.stringify(data)).then((result) => {
if (result?.err) {
Logger.warn(result?.err.message);
} else {
Logger.debug("redis: video information cache set for: " + videoID);
}
});
DiskCache.set(cacheKey, JSON.stringify(data))
.catch((err) => Logger.warn(err))
.then(() => Logger.debug("YouTube API: video information cache set for: " + videoID));
return { err: false, data };
} else {