Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into searchSegments

.gitignore

@@ -2,47 +2,9 @@
 logs
 *.log
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-lerna-debug.log*
-
-# Diagnostic reports (https://nodejs.org/api/report.html)
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-*.lcov
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
 
 # Dependency directories
 node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
 
 # TypeScript cache
 *.tsbuildinfo
@@ -53,40 +15,10 @@ typings/
 # Optional eslint cache
 .eslintcache
 
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
 # dotenv environment variables file
 .env
 .env.test
 
-# parcel-bundler cache (https://parceljs.org/)
-.cache
-
-# next.js build output
-.next
-
-# nuxt.js build output
-.nuxt
-
-# vuepress build output
-.vuepress/dist
-
-# Serverless directories
-.serverless/
-
-# FuseBox cache
-.fusebox/
-
-# DynamoDB Local files
-.dynamodb/
-
 # Databases
 databases/sponsorTimes.db
 databases/sponsorTimes.db-shm

README.MD

@@ -34,4 +34,4 @@ If you want to make changes, run `npm run dev` to automatically reload the serve
 
 # API Docs
 
-Available [here](https://github.com/ajayyy/SponsorBlock/wiki/API-Docs)
+Available [here](https://wiki.sponsor.ajay.app/index.php/API_Docs)

nginx/nginx.conf

@@ -2,7 +2,7 @@ worker_processes 8;
 worker_rlimit_nofile 65536;
 
 events {
-  worker_connections  432768;  ## Default: 1024
+	worker_connections 432768; # Default: 1024
 }
 
 http {
@@ -75,7 +75,6 @@ http {
 		#    return 400 "Gateway Timeout";
 		#}
 
-
 		location /news {
 			return	301		https://blog.ajay.app/sponsorblock;
 		}
@@ -85,7 +84,8 @@ http {
 		}
 
 		location /test/ {
-	    proxy_pass http://localhost:4440/;
+			return			404 "";
+			#proxy_pass		http://localhost:4440/;
 			#proxy_pass		https://sbtest.etcinit.com/;
 		}
 
@@ -111,7 +111,6 @@ http {
 			#return 200 "";
 		}
 
-        
 		location /api/getVideoSponsorTimes {
 			proxy_pass http://backend_GET;
 		}
@@ -120,7 +119,6 @@ http {
 			access_log /etc/nginx/logs/download.log no_ip;
 			gzip on;
 			gzip_types	text/plain application/json;
-	     #alias /home/sbadmin/sponsor/docker/database-export/;
 			alias		/home/sbadmin/sponsor/docker/database-export/;
 			#return 307 https://cdnsponsor.ajay.app$request_uri;
 		}
@@ -130,16 +128,15 @@ http {
 		}
 
 		location = /database.db {
-	    #return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
+			return	404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
 			#alias	/home/sbadmin/sponsor/databases/sponsorTimes.db;
-            alias /home/sbadmin/test-db/database.db;
+			#alias	/home/sbadmin/test-db/database.db;
 		}
 
 		#location = /database/sponsorTimes.csv {
 		#	alias /home/sbadmin/sponsorTimes.csv;
 		#}
 
-
 		#location /api/voteOnSponsorTime {
 		#	return 200 "Success";
 		#}
@@ -156,7 +153,6 @@ http {
 			root	/home/sbadmin/SponsorBlockSite/public-prod;
 		}
 		
-    
 		listen 443 default_server ssl http2; # managed by Certbot
 		#listen 443 http3 reuseport;
 		#ssl_protocols TLSv1.2 TLSv1.3;
@@ -165,33 +161,19 @@ http {
 		ssl_certificate_key	/etc/letsencrypt/live/sponsor.ajay.app-0003/privkey.pem; # managed by Certbot
 		include				/etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
 		ssl_dhparam			/etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
-
-
-
-
-
-
-
-
-
-
-
-
-
 	}
 
 	server {
 		server_name cdnsponsor.ajay.app;
-
 		error_page 404 /404.html;
 
 		#location /database/ {
 		#	alias /home/sbadmin/sponsor/docker/database-export/;
 		#}
 
-	location /download/ {
+		#location /download/ {
-	     alias /home/sbadmin/sponsor/docker/database-export/;
+		#	alias /home/sbadmin/sponsor/docker/database-export/;
-	}
+		#}
 
 		location / {
 			root /home/sbadmin/SponsorBlockSite/public-prod;
@@ -203,36 +185,23 @@ http {
 		ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
 		include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
 		ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
-
-
 	}
 
-
-
-
 	server {
-
 		access_log off;
 		error_log /etc/nginx/logs/log.txt;
 
-
 		if ($host = api.sponsor.ajay.app) {
 			return 301 https://$host$request_uri;
 		} # managed by Certbot
 
-
 		if ($host = sponsor.ajay.app) {
 			return 301 https://$host$request_uri;
 		} # managed by Certbot
 
-
 		listen 80;
 		server_name sponsor.ajay.app api.sponsor.ajay.app;
 		return 404; # managed by Certbot
-
-
-
-
 	}
 
 	server {
@@ -240,22 +209,17 @@ http {
 			return 301 https://$host$request_uri;
 		} # managed by Certbot
 
-
 		server_name cdnsponsor.ajay.app;
 		listen 80;
 		return 404; # managed by Certbot
-
-
 	}
 
 	server {
 		server_name wiki.sponsor.ajay.app; # managed by Certbot
-
 		location / {
 			proxy_pass http://10.0.0.3:8080;
 		}
 
-    
 		listen 443  ssl http2; # managed by Certbot
 		#listen 443 http3 reuseport;
 		#ssl_protocols TLSv1.2 TLSv1.3;
@@ -264,19 +228,5 @@ http {
 		ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0003/privkey.pem; # managed by Certbot
 		include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
 		ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
-
-
-
-
-
-
-
-
-
-
-
-
-
 	}
-
 }

package.json

@@ -17,35 +17,35 @@
   "license": "MIT",
   "dependencies": {
     "@ajayyy/lru-diskcache": "^1.1.9",
-    "@types/request": "^2.48.6",
+    "@types/request": "^2.48.7",
-    "better-sqlite3": "^7.4.1",
+    "better-sqlite3": "^7.4.3",
     "cron": "^1.8.2",
     "express": "^4.17.1",
     "express-promise-router": "^4.1.0",
     "express-rate-limit": "^5.3.0",
     "node-fetch": "^2.6.1",
-    "pg": "^8.6.0",
+    "pg": "^8.7.1",
     "redis": "^3.1.2",
     "sync-mysql": "^3.0.1"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^5.4.3",
+    "@types/better-sqlite3": "^7.4.0",
     "@types/cron": "^1.7.3",
     "@types/express": "^4.17.13",
     "@types/express-rate-limit": "^5.1.3",
-    "@types/mocha": "^8.2.3",
+    "@types/mocha": "^9.0.0",
-    "@types/node": "^16.3.0",
+    "@types/node": "^16.7.4",
-    "@types/node-fetch": "^2.5.11",
+    "@types/node-fetch": "^2.5.12",
     "@types/pg": "^8.6.1",
     "@types/redis": "^2.8.31",
-    "@typescript-eslint/eslint-plugin": "^4.28.2",
+    "@typescript-eslint/eslint-plugin": "^4.29.3",
-    "@typescript-eslint/parser": "^4.28.2",
+    "@typescript-eslint/parser": "^4.29.3",
-    "eslint": "^7.30.0",
+    "eslint": "^7.32.0",
-    "mocha": "^9.0.2",
+    "mocha": "^9.1.0",
-    "nodemon": "^2.0.2",
+    "nodemon": "^2.0.12",
-    "sinon": "^11.1.1",
+    "sinon": "^11.1.2",
     "ts-mock-imports": "^1.3.7",
-    "ts-node": "^10.0.0",
+    "ts-node": "^10.2.1",
-    "typescript": "^4.3.5"
+    "typescript": "^4.4.2"
   }
 }

src/config.ts

@@ -50,6 +50,7 @@ addDefaults(config, {
     userCounterURL: null,
     newLeafURLs: null,
     maxRewardTimePerSegmentInSeconds: 600,
+    poiMinimumStartTime: 2,
     postgres: null,
     dumpDatabase: {
         enabled: false,

src/cronjob/downvoteSegmentArchiveJob.ts

@@ -29,7 +29,7 @@ export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number
 
     } catch (err) {
         Logger.error("Execption when insert segment in archivedSponsorTimes");
-        Logger.error(err);
+        Logger.error(err as string);
         return 1;
     }
 
@@ -47,7 +47,7 @@ export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number
 
     } catch (err) {
         Logger.error("Execption when deleting segment in sponsorTimes");
-        Logger.error(err);
+        Logger.error(err as string);
         return 1;
     }
 

src/databases/Postgres.ts

@@ -38,7 +38,7 @@ export class Postgres implements IDatabase {
                 await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
             } catch (e) {
                 Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
-                Logger.warn(e);
+                Logger.warn(e as string);
             }
         }
     }

src/routes/addUnlistedVideo.ts

@@ -23,7 +23,7 @@ export function addUnlistedVideo(req: Request, res: Response): Response {
         const timeSubmitted = Date.now();
         db.prepare("run", `INSERT INTO "unlistedVideos" ("videoID", "year", "views", "channelID", "timeSubmitted") values (?, ?, ?, ?, ?)`, [videoID, year, views, channelID, timeSubmitted]);
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 

src/routes/dumpDatabase.ts

@@ -215,7 +215,7 @@ async function queueDump(): Promise<void> {
 
             lastUpdate = startTime;
         } catch(e) {
-            Logger.error(e);
+            Logger.error(e as string);
         } finally {
             updateQueued = false;
             updateRunning = false;

src/routes/getIsUserVIP.ts

@@ -22,7 +22,7 @@ export async function getIsUserVIP(req: Request, res: Response): Promise<Respons
             vip: vipState,
         });
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 }

src/routes/getLockCategories.ts

@@ -25,7 +25,7 @@ export async function getLockCategories(req: Request, res: Response): Promise<Re
             categories
         });
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 }

src/routes/getLockCategoriesByHash.ts

@@ -54,7 +54,7 @@ export async function getLockCategoriesByHash(req: Request, res: Response): Prom
         // merge all locks
         return res.send(mergeLocks(lockedRows));
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 }

src/routes/getSkipSegments.ts

@@ -31,14 +31,14 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
         }
 
         //if this isn't their ip, don't send it to them
-        return cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted].some((shadowHiddenSegment) => {
+        return cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted]?.some((shadowHiddenSegment) => {
             if (cache.userHashedIP === undefined) {
                 //hash the IP only if it's strictly necessary
                 cache.userHashedIP = getHash((getIP(req) + config.globalSalt) as IPAddress);
             }
 
             return shadowHiddenSegment.hashedIP === cache.userHashedIP;
-        });
+        }) ?? false;
     }));
 
     const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
@@ -80,7 +80,7 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
         return segments;
     } catch (err) {
         if (err) {
-            Logger.error(err);
+            Logger.error(err as string);
             return null;
         }
     }
@@ -125,12 +125,10 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
 
         return segments;
     } catch (err) {
-        if (err) {
+        Logger.error(err as string);
-            Logger.error(err);
         return null;
     }
 }
-}
 
 async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {
     const fetchFromDB = () => db

src/routes/getUsername.ts

@@ -28,7 +28,7 @@ export async function getUsername(req: Request, res: Response): Promise<Response
             });
         }
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 }

src/routes/getViewsForUser.ts

@@ -26,7 +26,7 @@ export async function getViewsForUser(req: Request, res: Response): Promise<Resp
             return res.sendStatus(404);
         }
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 }

src/routes/postLockCategories.ts

@@ -68,7 +68,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
             await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason") VALUES(?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason]);
         } catch (err) {
             Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`);
-            Logger.error(err);
+            Logger.error(err as string);
             res.status(500).json({
                 message: "Internal Server Error: Could not write marker to the database.",
             });
@@ -89,7 +89,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
                     [reason, userID, videoID, category]);
             } catch (err) {
                 Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`);
-                Logger.error(err);
+                Logger.error(err as string);
                 res.status(500).json({
                     message: "Internal Server Error: Could not write marker to the database.",
                 });

src/routes/postPurgeAllSegments.ts

@@ -29,7 +29,7 @@ export async function postPurgeAllSegments(req: Request, res: Response): Promise
         await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "videoID" = ?`, [videoID]);
 
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
     return res.sendStatus(200);

src/routes/postSegmentShift.ts

@@ -92,7 +92,7 @@ export async function postSegmentShift(req: Request, res: Response): Promise<Res
             }
         }
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 

src/routes/postSkipSegments.ts

@@ -390,9 +390,9 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID
             return { pass: false, errorMessage: "One of your segments times are invalid (too short, startTime before endTime, etc.)", errorCode: 400};
         }
 
-        // Check for POI segments before 1 second
+        // Check for POI segments before some seconds
-        if (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime < 1) {
+        if (!isVIP && getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime < config.poiMinimumStartTime) {
-            return { pass: false, errorMessage: "POI must be after 1 second", errorCode: 400};
+            return { pass: false, errorMessage: `POI cannot be that early`, errorCode: 400};
         }
 
         if (!isVIP && segments[i].category === "sponsor" && Math.abs(startTime - endTime) < 1) {
@@ -684,7 +684,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
             });
         }
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 

src/routes/setUsername.ts

@@ -57,7 +57,7 @@ export async function setUsername(req: Request, res: Response): Promise<Response
         }
     }
     catch (error) {
-        Logger.error(error);
+        Logger.error(error as string);
         return res.sendStatus(500);
     }
 
@@ -80,7 +80,7 @@ export async function setUsername(req: Request, res: Response): Promise<Response
 
         return res.sendStatus(200);
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.sendStatus(500);
     }
 }

src/routes/voteOnSponsorTime.ts

@@ -458,7 +458,7 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
         }
         return res.status(finalResponse.finalStatus).send(finalResponse.finalMessage ?? undefined);
     } catch (err) {
-        Logger.error(err);
+        Logger.error(err as string);
         return res.status(500).json({error: "Internal error creating segment vote"});
     }
 }

src/types/youtubeApi.model.ts

@@ -106,6 +106,6 @@ export interface APIVideoData {
 }
 
 export interface APIVideoInfo {
-    err: string | boolean,
+    err: boolean | string,
     data?: APIVideoData
 }

src/utils/youtubeApi.ts

@@ -20,7 +20,7 @@ export class YouTubeAPI {
                     return { err: null, data: JSON.parse(data) };
                 }
             } catch (err) {
-                return { err };
+                return { err: err as string | boolean, data: null };
             }
         }
 
@@ -45,7 +45,7 @@ export class YouTubeAPI {
                 return { err: result.statusText, data: null };
             }
         } catch (err) {
-            return {err, data: null};
+            return {err: err as string | boolean, data: null};
         }
     }
 }

test/cases/setUsername.ts

@@ -1,5 +1,5 @@
 import fetch from "node-fetch";
-import { Done, getbaseURL, partialDeepEquals } from "../utils";
+import { Done, getbaseURL } from "../utils";
 import { db, privateDB } from "../../src/databases/databases";
 import { getHash } from "../../src/utils/getHash";
 import assert from "assert";