From 25ec9b0291c043085db1171222120daaf6db9616 Mon Sep 17 00:00:00 2001 From: Ajay Date: Sun, 21 Jan 2024 19:49:36 -0500 Subject: [PATCH] Revert adding redis compression This reverts commit fce311377fe1e2f5f65c04cd002566a30ff03130 and 2ad51842ccd1fac5cf4404f912b0c205da84c26f --- package-lock.json | 347 --------------------------------------- package.json | 1 - src/utils/queryCacher.ts | 8 +- src/utils/redis.ts | 41 +---- src/utils/redisKeys.ts | 24 +-- 5 files changed, 25 insertions(+), 396 deletions(-) diff --git a/package-lock.json b/package-lock.json index 30318be..06ddadc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,6 @@ "express-rate-limit": "^6.7.0", "form-data": "^4.0.0", "lodash": "^4.17.21", - "lz4-napi": "^2.2.0", "pg": "^8.8.0", "rate-limit-redis": "^3.0.1", "redis": "^4.5.0", @@ -63,201 +62,6 @@ "node": ">=6.0.0" } }, - "node_modules/@antoniomuso/lz4-napi-android-arm-eabi": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-android-arm-eabi/-/lz4-napi-android-arm-eabi-2.2.0.tgz", - "integrity": "sha512-9PBt0UhmahcQamcD+eHVB2u38Ae1z7Sqb2zBLExj9DRGT9tRj3RavdgAjRewsShp+k+J2hxfinXp8gLil3rZwg==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-android-arm64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-android-arm64/-/lz4-napi-android-arm64-2.2.0.tgz", - "integrity": "sha512-zW0EPEuSBxjjP9MZUyXmDYAXLpS/TeVEq/la+nJIvM5XtGg1rwvuNdEErO977QI0zRc1jIQBMzE25B/wwP/Bwg==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-darwin-arm64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-darwin-arm64/-/lz4-napi-darwin-arm64-2.2.0.tgz", - "integrity": "sha512-KVE5F+fOmVIXADkgSTgA+J3Hv7l2QEaj4iBbLQfuDSZjm5qMDrXUfBJBJv6xKYpu9/ikMZRKvjfU/krhVMqWXw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-darwin-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-darwin-x64/-/lz4-napi-darwin-x64-2.2.0.tgz", - "integrity": "sha512-K2jRtXU+9qldL6vK7K4VT3UClZZb87nLqMlwkMVS2eaoofqFdrkSAgJP5oSHN9bkE7u3OCLgOUZfFFSi/UNz7w==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-freebsd-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-freebsd-x64/-/lz4-napi-freebsd-x64-2.2.0.tgz", - "integrity": "sha512-KKXiaQ8g0UxwDsfCBH/ARo5FCnoFtoPLDAxz72QLWuuTxfK0epg3ZChb2l1n95bUvSZKRSVQaa7sHJsl+d+ffg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-linux-arm-gnueabihf": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-arm-gnueabihf/-/lz4-napi-linux-arm-gnueabihf-2.2.0.tgz", - "integrity": "sha512-p0WkyFmWSaveGPNolSxH23/NksbhORj7D1dxyHdtZKdY1xMUVxQ/aIrD/Vfn35hO01KFI1LVh6nTpybez7Empg==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-linux-arm64-gnu": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-arm64-gnu/-/lz4-napi-linux-arm64-gnu-2.2.0.tgz", - "integrity": "sha512-a1C4u+dnS8wP+Lu6fUKGOeVm0tFw8gt0VK+3aCCB9AYu8u2NdfYqqpceu9snNk04Rzp3MNJSryWvHtoVOdUtMQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-linux-arm64-musl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-arm64-musl/-/lz4-napi-linux-arm64-musl-2.2.0.tgz", - "integrity": "sha512-+YlKffM2n5/zLJ3JYpzbnwTLj/aiOAM9rEJlDg7LR0YK5o0dbcsCq8fK/fapDkzoLl8BMO6ObJtuZ/6Vi3Targ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-linux-x64-gnu": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-x64-gnu/-/lz4-napi-linux-x64-gnu-2.2.0.tgz", - "integrity": "sha512-pPJVppGpV+La0tb34GMTbiUTLsSqIJGzxIdPM2XjJ23u6ytnog2DQiyylMENhY20X7oHG2oQlCW9J+W8sCNh3Q==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-linux-x64-musl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-x64-musl/-/lz4-napi-linux-x64-musl-2.2.0.tgz", - "integrity": "sha512-KSWJ5TZVLlOA+hzy7gnYytb7xhD6jeoYqno72Zp/jY1fV5xHk71YJppvzOzVpX4mbRFdu5DMQU8GcvTxsmS2zg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-win32-arm64-msvc": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-win32-arm64-msvc/-/lz4-napi-win32-arm64-msvc-2.2.0.tgz", - "integrity": "sha512-bfk4rNPE5Al1aozsRm4oWfMjxS2qDaoYilFzSDWxToFg7DFlAkjBsVLnB9cV+W6oZ/+jowlK8IGFlNhUG+uatw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-win32-ia32-msvc": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-win32-ia32-msvc/-/lz4-napi-win32-ia32-msvc-2.2.0.tgz", - "integrity": "sha512-xu7XMe1KyLDP2wzOxTE9eibFGP+Ut1Sl3GbW1zW3wjV4RIXyJqZRvwIOsZ0P0lfXJmykmzN6yIzs1QksOzgMog==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@antoniomuso/lz4-napi-win32-x64-msvc": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-win32-x64-msvc/-/lz4-napi-win32-x64-msvc-2.2.0.tgz", - "integrity": "sha512-sJZjXEIjtK7upjh1oHpDELAH45jL/dNdg/mzYg61ljprtfjsza40h8e/Sq0IQOCmJc6IR746bGcttPM297nhzQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@babel/code-frame": { "version": "7.22.13", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", @@ -988,19 +792,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@napi-rs/triples": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@napi-rs/triples/-/triples-1.2.0.tgz", - "integrity": "sha512-HAPjR3bnCsdXBsATpDIP5WCrw0JcACwhhrwIAQhiR46n+jm+a2F8kBsfseAuWtSyQ+H3Yebt2k43B5dy+04yMA==" - }, - "node_modules/@node-rs/helper": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@node-rs/helper/-/helper-1.5.0.tgz", - "integrity": "sha512-4E1laLlg532nkrvUohZzl/wFos3bsHExSDw/guaoLekMcTKtD3ZsH6Vc4/qIRzOzAhQv5LLRnTE+vOeJ6C9Z8w==", - "dependencies": { - "@napi-rs/triples": "^1.2.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3723,32 +3514,6 @@ "node": "*" } }, - "node_modules/lz4-napi": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/lz4-napi/-/lz4-napi-2.2.0.tgz", - "integrity": "sha512-n2xx3j7rILyd7OXxQoFXmyEpEJywfAXSnK6br2woep7j/7RV86gc7xLTITaM5/KZnQJb5exSvpcz+lTZ8npXgg==", - "dependencies": { - "@node-rs/helper": "^1.3.3" - }, - "engines": { - "node": ">= 10" - }, - "optionalDependencies": { - "@antoniomuso/lz4-napi-android-arm-eabi": "2.2.0", - "@antoniomuso/lz4-napi-android-arm64": "2.2.0", - "@antoniomuso/lz4-napi-darwin-arm64": "2.2.0", - "@antoniomuso/lz4-napi-darwin-x64": "2.2.0", - "@antoniomuso/lz4-napi-freebsd-x64": "2.2.0", - "@antoniomuso/lz4-napi-linux-arm-gnueabihf": "2.2.0", - "@antoniomuso/lz4-napi-linux-arm64-gnu": "2.2.0", - "@antoniomuso/lz4-napi-linux-arm64-musl": "2.2.0", - "@antoniomuso/lz4-napi-linux-x64-gnu": "2.2.0", - "@antoniomuso/lz4-napi-linux-x64-musl": "2.2.0", - "@antoniomuso/lz4-napi-win32-arm64-msvc": "2.2.0", - "@antoniomuso/lz4-napi-win32-ia32-msvc": "2.2.0", - "@antoniomuso/lz4-napi-win32-x64-msvc": "2.2.0" - } - }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -5964,84 +5729,6 @@ "@jridgewell/trace-mapping": "^0.3.9" } }, - "@antoniomuso/lz4-napi-android-arm-eabi": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-android-arm-eabi/-/lz4-napi-android-arm-eabi-2.2.0.tgz", - "integrity": "sha512-9PBt0UhmahcQamcD+eHVB2u38Ae1z7Sqb2zBLExj9DRGT9tRj3RavdgAjRewsShp+k+J2hxfinXp8gLil3rZwg==", - "optional": true - }, - "@antoniomuso/lz4-napi-android-arm64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-android-arm64/-/lz4-napi-android-arm64-2.2.0.tgz", - "integrity": "sha512-zW0EPEuSBxjjP9MZUyXmDYAXLpS/TeVEq/la+nJIvM5XtGg1rwvuNdEErO977QI0zRc1jIQBMzE25B/wwP/Bwg==", - "optional": true - }, - "@antoniomuso/lz4-napi-darwin-arm64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-darwin-arm64/-/lz4-napi-darwin-arm64-2.2.0.tgz", - "integrity": "sha512-KVE5F+fOmVIXADkgSTgA+J3Hv7l2QEaj4iBbLQfuDSZjm5qMDrXUfBJBJv6xKYpu9/ikMZRKvjfU/krhVMqWXw==", - "optional": true - }, - "@antoniomuso/lz4-napi-darwin-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-darwin-x64/-/lz4-napi-darwin-x64-2.2.0.tgz", - "integrity": "sha512-K2jRtXU+9qldL6vK7K4VT3UClZZb87nLqMlwkMVS2eaoofqFdrkSAgJP5oSHN9bkE7u3OCLgOUZfFFSi/UNz7w==", - "optional": true - }, - "@antoniomuso/lz4-napi-freebsd-x64": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-freebsd-x64/-/lz4-napi-freebsd-x64-2.2.0.tgz", - "integrity": "sha512-KKXiaQ8g0UxwDsfCBH/ARo5FCnoFtoPLDAxz72QLWuuTxfK0epg3ZChb2l1n95bUvSZKRSVQaa7sHJsl+d+ffg==", - "optional": true - }, - "@antoniomuso/lz4-napi-linux-arm-gnueabihf": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-arm-gnueabihf/-/lz4-napi-linux-arm-gnueabihf-2.2.0.tgz", - "integrity": "sha512-p0WkyFmWSaveGPNolSxH23/NksbhORj7D1dxyHdtZKdY1xMUVxQ/aIrD/Vfn35hO01KFI1LVh6nTpybez7Empg==", - "optional": true - }, - "@antoniomuso/lz4-napi-linux-arm64-gnu": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-arm64-gnu/-/lz4-napi-linux-arm64-gnu-2.2.0.tgz", - "integrity": "sha512-a1C4u+dnS8wP+Lu6fUKGOeVm0tFw8gt0VK+3aCCB9AYu8u2NdfYqqpceu9snNk04Rzp3MNJSryWvHtoVOdUtMQ==", - "optional": true - }, - "@antoniomuso/lz4-napi-linux-arm64-musl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-arm64-musl/-/lz4-napi-linux-arm64-musl-2.2.0.tgz", - "integrity": "sha512-+YlKffM2n5/zLJ3JYpzbnwTLj/aiOAM9rEJlDg7LR0YK5o0dbcsCq8fK/fapDkzoLl8BMO6ObJtuZ/6Vi3Targ==", - "optional": true - }, - "@antoniomuso/lz4-napi-linux-x64-gnu": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-x64-gnu/-/lz4-napi-linux-x64-gnu-2.2.0.tgz", - "integrity": "sha512-pPJVppGpV+La0tb34GMTbiUTLsSqIJGzxIdPM2XjJ23u6ytnog2DQiyylMENhY20X7oHG2oQlCW9J+W8sCNh3Q==", - "optional": true - }, - "@antoniomuso/lz4-napi-linux-x64-musl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-linux-x64-musl/-/lz4-napi-linux-x64-musl-2.2.0.tgz", - "integrity": "sha512-KSWJ5TZVLlOA+hzy7gnYytb7xhD6jeoYqno72Zp/jY1fV5xHk71YJppvzOzVpX4mbRFdu5DMQU8GcvTxsmS2zg==", - "optional": true - }, - "@antoniomuso/lz4-napi-win32-arm64-msvc": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-win32-arm64-msvc/-/lz4-napi-win32-arm64-msvc-2.2.0.tgz", - "integrity": "sha512-bfk4rNPE5Al1aozsRm4oWfMjxS2qDaoYilFzSDWxToFg7DFlAkjBsVLnB9cV+W6oZ/+jowlK8IGFlNhUG+uatw==", - "optional": true - }, - "@antoniomuso/lz4-napi-win32-ia32-msvc": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-win32-ia32-msvc/-/lz4-napi-win32-ia32-msvc-2.2.0.tgz", - "integrity": "sha512-xu7XMe1KyLDP2wzOxTE9eibFGP+Ut1Sl3GbW1zW3wjV4RIXyJqZRvwIOsZ0P0lfXJmykmzN6yIzs1QksOzgMog==", - "optional": true - }, - "@antoniomuso/lz4-napi-win32-x64-msvc": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@antoniomuso/lz4-napi-win32-x64-msvc/-/lz4-napi-win32-x64-msvc-2.2.0.tgz", - "integrity": "sha512-sJZjXEIjtK7upjh1oHpDELAH45jL/dNdg/mzYg61ljprtfjsza40h8e/Sq0IQOCmJc6IR746bGcttPM297nhzQ==", - "optional": true - }, "@babel/code-frame": { "version": "7.22.13", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", @@ -6609,19 +6296,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "@napi-rs/triples": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@napi-rs/triples/-/triples-1.2.0.tgz", - "integrity": "sha512-HAPjR3bnCsdXBsATpDIP5WCrw0JcACwhhrwIAQhiR46n+jm+a2F8kBsfseAuWtSyQ+H3Yebt2k43B5dy+04yMA==" - }, - "@node-rs/helper": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@node-rs/helper/-/helper-1.5.0.tgz", - "integrity": "sha512-4E1laLlg532nkrvUohZzl/wFos3bsHExSDw/guaoLekMcTKtD3ZsH6Vc4/qIRzOzAhQv5LLRnTE+vOeJ6C9Z8w==", - "requires": { - "@napi-rs/triples": "^1.2.0" - } - }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -8613,27 +8287,6 @@ "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.1.tgz", "integrity": "sha512-gYHAa180mKrNIUJCbwpmD0aTu9kV0dREDrwNnuyFAsO1Wt0EVYSZelPnJlbj9HplzXX/YWXHFTL45kvZ53M0pw==" }, - "lz4-napi": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/lz4-napi/-/lz4-napi-2.2.0.tgz", - "integrity": "sha512-n2xx3j7rILyd7OXxQoFXmyEpEJywfAXSnK6br2woep7j/7RV86gc7xLTITaM5/KZnQJb5exSvpcz+lTZ8npXgg==", - "requires": { - "@antoniomuso/lz4-napi-android-arm-eabi": "2.2.0", - "@antoniomuso/lz4-napi-android-arm64": "2.2.0", - "@antoniomuso/lz4-napi-darwin-arm64": "2.2.0", - "@antoniomuso/lz4-napi-darwin-x64": "2.2.0", - "@antoniomuso/lz4-napi-freebsd-x64": "2.2.0", - "@antoniomuso/lz4-napi-linux-arm-gnueabihf": "2.2.0", - "@antoniomuso/lz4-napi-linux-arm64-gnu": "2.2.0", - "@antoniomuso/lz4-napi-linux-arm64-musl": "2.2.0", - "@antoniomuso/lz4-napi-linux-x64-gnu": "2.2.0", - "@antoniomuso/lz4-napi-linux-x64-musl": "2.2.0", - "@antoniomuso/lz4-napi-win32-arm64-msvc": "2.2.0", - "@antoniomuso/lz4-napi-win32-ia32-msvc": "2.2.0", - "@antoniomuso/lz4-napi-win32-x64-msvc": "2.2.0", - "@node-rs/helper": "^1.3.3" - } - }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", diff --git a/package.json b/package.json index 77ecee7..ac5864d 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,6 @@ "express-rate-limit": "^6.7.0", "form-data": "^4.0.0", "lodash": "^4.17.21", - "lz4-napi": "^2.2.0", "pg": "^8.8.0", "rate-limit-redis": "^3.0.1", "redis": "^4.5.0", diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index 85ec76f..fe89c65 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -7,7 +7,7 @@ import { config } from "../config"; async function get(fetchFromDB: () => Promise, key: string): Promise { try { - const reply = await redis.getCompressed(key); + const reply = await redis.get(key); if (reply) { Logger.debug(`Got data from redis: ${reply}`); @@ -21,7 +21,7 @@ async function get(fetchFromDB: () => Promise, key: string): Promise { const data = await fetchFromDB(); - redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); + redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); return data; } @@ -36,7 +36,7 @@ async function getTraced(fetchFromDB: () => Promise, key: string): Promise const startTime = Date.now(); try { - const reply = await redis.getCompressed(key); + const reply = await redis.get(key); if (reply) { Logger.debug(`Got data from redis: ${reply}`); @@ -55,7 +55,7 @@ async function getTraced(fetchFromDB: () => Promise, key: string): Promise const dbStartTime = Date.now(); const data = await fetchFromDB(); - redis.setExCompressed(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); + redis.setEx(key, config.redis?.expiryTime, JSON.stringify(data)).catch((err) => Logger.error(err)); return { data, diff --git a/src/utils/redis.ts b/src/utils/redis.ts index ba09f10..3a495e7 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -6,7 +6,6 @@ import { RedisClientOptions } from "@redis/client/dist/lib/client"; import { RedisReply } from "rate-limit-redis"; import { db } from "../databases/databases"; import { Postgres } from "../databases/Postgres"; -import { compress, uncompress } from "lz4-napi"; export interface RedisStats { activeRequests: number; @@ -17,11 +16,8 @@ export interface RedisStats { interface RedisSB { get(key: RedisCommandArgument): Promise; - getCompressed(key: RedisCommandArgument): Promise; set(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; - setCompressed(key: RedisCommandArgument, value: RedisCommandArgument, options?: SetOptions): Promise; setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; - setExCompressed(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; del(...keys: [RedisCommandArgument]): Promise; increment?(key: RedisCommandArgument): Promise; sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise; @@ -30,17 +26,14 @@ interface RedisSB { } let exportClient: RedisSB = { - get: () => Promise.resolve(null), - getCompressed: () => Promise.resolve(null), - set: () => Promise.resolve(null), - setCompressed: () => Promise.resolve(null), - setEx: () => Promise.resolve(null), - setExCompressed: () => Promise.resolve(null), - del: () => Promise.resolve(null), - increment: () => Promise.resolve(null), - sendCommand: () => Promise.resolve(null), - quit: () => Promise.resolve(null), - ttl: () => Promise.resolve(null), + get: () => new Promise((resolve) => resolve(null)), + set: () => new Promise((resolve) => resolve(null)), + setEx: () => new Promise((resolve) => resolve(null)), + del: () => new Promise((resolve) => resolve(null)), + increment: () => new Promise((resolve) => resolve(null)), + sendCommand: () => new Promise((resolve) => resolve(null)), + quit: () => new Promise((resolve) => resolve(null)), + ttl: () => new Promise((resolve) => resolve(null)), }; let lastClientFail = 0; @@ -63,24 +56,8 @@ if (config.redis?.enabled) { const readClient = config.redisRead?.enabled ? createClient(config.redisRead) : null; connectionPromise = client.connect(); void readClient?.connect(); // void as we don't care about the promise - exportClient = client as unknown as RedisSB; + exportClient = client as RedisSB; - exportClient.getCompressed = (key) => { - return exportClient.get(key).then((reply) => { - if (reply === null) return null; - return uncompress(Buffer.from(reply, "base64")).then((decompressed) => decompressed.toString("utf-8")); - }); - }; - exportClient.setCompressed = (key, value, options) => { - return compress(Buffer.from(value as string, "utf-8")).then((compressed) => - exportClient.set(key, compressed.toString("base64"), options) - ); - }; - exportClient.setExCompressed = (key, seconds, value) => { - return compress(Buffer.from(value as string, "utf-8")).then((compressed) => - exportClient.setEx(key, seconds, compressed.toString("base64")) - ); - }; const get = client.get.bind(client); const getRead = readClient?.get?.bind(readClient); diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index 9d5ca3c..ae6927e 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -5,43 +5,43 @@ import { Logger } from "./logger"; import { BrandingUUID } from "../types/branding.model"; export const skipSegmentsKey = (videoID: VideoID, service: Service): string => - `segments.v6.${service}.videoID.${videoID}`; + `segments.v4.${service}.videoID.${videoID}`; export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string => - `segments.groups.v5.${service}.videoID.${videoID}`; + `segments.groups.v3.${service}.videoID.${videoID}`; export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); - return `segments.v6.${service}.${hashedVideoIDPrefix}`; + return `segments.v4.${service}.${hashedVideoIDPrefix}`; } export const brandingKey = (videoID: VideoID, service: Service): string => - `branding.v4.${service}.videoID.${videoID}`; + `branding.v2.${service}.videoID.${videoID}`; export function brandingHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); - return `branding.v4.${service}.${hashedVideoIDPrefix}`; + return `branding.v2.${service}.${hashedVideoIDPrefix}`; } export const brandingIPKey = (uuid: BrandingUUID): string => - `branding.v2.shadow.${uuid}`; + `branding.shadow.${uuid}`; export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string => - `segments.v2.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; + `segments.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; export const reputationKey = (userID: UserID): string => - `reputation.v2.user.${userID}`; + `reputation.user.${userID}`; export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string { hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash; if (hashPrefix.length !== 4) Logger.warn(`Redis rating hash-prefix key is not length 4! ${hashPrefix}`); - return `rating.v2.${service}.${hashPrefix}`; + return `rating.${service}.${hashPrefix}`; } export function shaHashKey(singleIter: HashedValue): string { @@ -54,15 +54,15 @@ export const tempVIPKey = (userID: HashedUserID): string => `vip.temp.${userID}`; export const videoLabelsKey = (videoID: VideoID, service: Service): string => - `labels.v3.${service}.videoID.${videoID}`; + `labels.v1.${service}.videoID.${videoID}`; export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash; if (hashedVideoIDPrefix.length !== 3) Logger.warn(`Redis video labels hash-prefix key is not length 3! ${hashedVideoIDPrefix}`); - return `labels.v3.${service}.${hashedVideoIDPrefix}`; + return `labels.v1.${service}.${hashedVideoIDPrefix}`; } export function userFeatureKey (userID: HashedUserID, feature: Feature): string { - return `user.v2.${userID}.feature.${feature}`; + return `user.${userID}.feature.${feature}`; } \ No newline at end of file