Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-31 21:59:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into searchSegments

Browse Source
This commit is contained in:
Michael C
2021-08-31 23:10:04 -04:00
parent bb02033567 ba7fb6525f
commit 9bef529486
26 changed files with 1586 additions and 1370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
.gitignore vendored
View File

@@ -2,47 +2,9 @@
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
@@ -53,40 +15,10 @@ typings/
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
# next.js build output
.next
# nuxt.js build output
.nuxt
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# Databases
databases/sponsorTimes.db
databases/sponsorTimes.db-shm

2
README.MD
View File

@@ -34,4 +34,4 @@ If you want to make changes, run `npm run dev` to automatically reload the serve
# API Docs
Available [here](https://github.com/ajayyy/SponsorBlock/wiki/API-Docs)
Available [here](https://wiki.sponsor.ajay.app/index.php/API_Docs)

76
nginx/nginx.conf
View File

@@ -2,7 +2,7 @@ worker_processes 8;
worker_rlimit_nofile 65536;
events {
worker_connections 432768; ## Default: 1024
worker_connections 432768; # Default: 1024
}
http {
@@ -75,7 +75,6 @@ http {
# return 400 "Gateway Timeout";
#}
location /news {
return 301 https://blog.ajay.app/sponsorblock;
}
@@ -85,7 +84,8 @@ http {
}
location /test/ {
proxy_pass http://localhost:4440/;
return 404 "";
#proxy_pass http://localhost:4440/;
#proxy_pass https://sbtest.etcinit.com/;
}
@@ -111,7 +111,6 @@ http {
#return 200 "";
}
location /api/getVideoSponsorTimes {
proxy_pass http://backend_GET;
}
@@ -120,7 +119,6 @@ http {
access_log /etc/nginx/logs/download.log no_ip;
gzip on;
gzip_types text/plain application/json;
#alias /home/sbadmin/sponsor/docker/database-export/;
alias /home/sbadmin/sponsor/docker/database-export/;
#return 307 https://cdnsponsor.ajay.app$request_uri;
}
@@ -130,16 +128,15 @@ http {
}
location = /database.db {
#return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
#alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
alias /home/sbadmin/test-db/database.db;
#alias /home/sbadmin/test-db/database.db;
}
#location = /database/sponsorTimes.csv {
# alias /home/sbadmin/sponsorTimes.csv;
#}
#location /api/voteOnSponsorTime {
# return 200 "Success";
#}
@@ -156,7 +153,6 @@ http {
root /home/sbadmin/SponsorBlockSite/public-prod;
}
listen 443 default_server ssl http2; # managed by Certbot
#listen 443 http3 reuseport;
#ssl_protocols TLSv1.2 TLSv1.3;
@@ -165,33 +161,19 @@ http {
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0003/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
}
server {
server_name cdnsponsor.ajay.app;
error_page 404 /404.html;
#location /database/ {
# alias /home/sbadmin/sponsor/docker/database-export/;
#}
location /download/ {
alias /home/sbadmin/sponsor/docker/database-export/;
}
#location /download/ {
# alias /home/sbadmin/sponsor/docker/database-export/;
#}
location / {
root /home/sbadmin/SponsorBlockSite/public-prod;
@@ -203,59 +185,41 @@ http {
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
}
server {
access_log off;
error_log /etc/nginx/logs/log.txt;
if ($host = api.sponsor.ajay.app) {
return 301 https://$host$request_uri;
} # managed by Certbot
if ($host = sponsor.ajay.app) {
return 301 https://$host$request_uri;
} # managed by Certbot
listen 80;
server_name sponsor.ajay.app api.sponsor.ajay.app;
return 404; # managed by Certbot
}
}
server {
if ($host = cdnsponsor.ajay.app) {
return 301 https://$host$request_uri;
} # managed by Certbot
server_name cdnsponsor.ajay.app;
listen 80;
return 404; # managed by Certbot
}
}
server {
server_name wiki.sponsor.ajay.app; # managed by Certbot
location / {
proxy_pass http://10.0.0.3:8080;
}
listen 443 ssl http2; # managed by Certbot
#listen 443 http3 reuseport;
#ssl_protocols TLSv1.2 TLSv1.3;
@@ -264,19 +228,5 @@ http {
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0003/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
}
}

2317
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

30
package.json
View File

@@ -17,35 +17,35 @@
"license": "MIT",
"dependencies": {
"@ajayyy/lru-diskcache": "^1.1.9",
"@types/request": "^2.48.6",
"better-sqlite3": "^7.4.1",
"@types/request": "^2.48.7",
"better-sqlite3": "^7.4.3",
"cron": "^1.8.2",
"express": "^4.17.1",
"express-promise-router": "^4.1.0",
"express-rate-limit": "^5.3.0",
"node-fetch": "^2.6.1",
"pg": "^8.6.0",
"pg": "^8.7.1",
"redis": "^3.1.2",
"sync-mysql": "^3.0.1"
},
"devDependencies": {
"@types/better-sqlite3": "^5.4.3",
"@types/better-sqlite3": "^7.4.0",
"@types/cron": "^1.7.3",
"@types/express": "^4.17.13",
"@types/express-rate-limit": "^5.1.3",
"@types/mocha": "^8.2.3",
"@types/node": "^16.3.0",
"@types/node-fetch": "^2.5.11",
"@types/mocha": "^9.0.0",
"@types/node": "^16.7.4",
"@types/node-fetch": "^2.5.12",
"@types/pg": "^8.6.1",
"@types/redis": "^2.8.31",
"@typescript-eslint/eslint-plugin": "^4.28.2",
"@typescript-eslint/parser": "^4.28.2",
"eslint": "^7.30.0",
"mocha": "^9.0.2",
"nodemon": "^2.0.2",
"sinon": "^11.1.1",
"@typescript-eslint/eslint-plugin": "^4.29.3",
"@typescript-eslint/parser": "^4.29.3",
"eslint": "^7.32.0",
"mocha": "^9.1.0",
"nodemon": "^2.0.12",
"sinon": "^11.1.2",
"ts-mock-imports": "^1.3.7",
"ts-node": "^10.0.0",
"typescript": "^4.3.5"
"ts-node": "^10.2.1",
"typescript": "^4.4.2"
}
}

1
src/config.ts
View File

@@ -50,6 +50,7 @@ addDefaults(config, {
userCounterURL: null,
newLeafURLs: null,
maxRewardTimePerSegmentInSeconds: 600,
poiMinimumStartTime: 2,
postgres: null,
dumpDatabase: {
enabled: false,

4
src/cronjob/downvoteSegmentArchiveJob.ts
View File

@@ -29,7 +29,7 @@ export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number
} catch (err) {
Logger.error("Execption when insert segment in archivedSponsorTimes");
Logger.error(err);
Logger.error(err as string);
return 1;
}
@@ -47,7 +47,7 @@ export const archiveDownvoteSegment = async (dayLimit: number, voteLimit: number
} catch (err) {
Logger.error("Execption when deleting segment in sponsorTimes");
Logger.error(err);
Logger.error(err as string);
return 1;
}

2
src/databases/Postgres.ts
View File

@@ -38,7 +38,7 @@ export class Postgres implements IDatabase {
await this.applyIndexes(this.config.fileNamePrefix, this.config.dbSchemaFolder);
} catch (e) {
Logger.warn("Applying indexes failed. See https://github.com/ajayyy/SponsorBlockServer/wiki/Postgres-Extensions for more information.");
Logger.warn(e);
Logger.warn(e as string);
}
}
}

2
src/routes/addUnlistedVideo.ts
View File

@@ -23,7 +23,7 @@ export function addUnlistedVideo(req: Request, res: Response): Response {
const timeSubmitted = Date.now();
db.prepare("run", `INSERT INTO "unlistedVideos" ("videoID", "year", "views", "channelID", "timeSubmitted") values (?, ?, ?, ?, ?)`, [videoID, year, views, channelID, timeSubmitted]);
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}

2
src/routes/dumpDatabase.ts
View File

@@ -215,7 +215,7 @@ async function queueDump(): Promise<void> {
lastUpdate = startTime;
} catch(e) {
Logger.error(e);
Logger.error(e as string);
} finally {
updateQueued = false;
updateRunning = false;

2
src/routes/getIsUserVIP.ts
View File

@@ -22,7 +22,7 @@ export async function getIsUserVIP(req: Request, res: Response): Promise<Respons
vip: vipState,
});
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}
}

2
src/routes/getLockCategories.ts
View File

@@ -25,7 +25,7 @@ export async function getLockCategories(req: Request, res: Response): Promise<Re
categories
});
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}
}

2
src/routes/getLockCategoriesByHash.ts
View File

@@ -54,7 +54,7 @@ export async function getLockCategoriesByHash(req: Request, res: Response): Prom
// merge all locks
return res.send(mergeLocks(lockedRows));
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}
}

12
src/routes/getSkipSegments.ts
View File

@@ -12,7 +12,7 @@ import { QueryCacher } from "../utils/queryCacher";
import { getReputation } from "../utils/reputation";
async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[],cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
async function prepareCategorySegments(req: Request, videoID: VideoID, category: Category, segments: DBSegment[], cache: SegmentCache = {shadowHiddenSegmentIPs: {}}): Promise<Segment[]> {
const shouldFilter: boolean[] = await Promise.all(segments.map(async (segment) => {
if (segment.votes < -1 && !segment.required) {
return false; //too untrustworthy, just ignore it
@@ -31,14 +31,14 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
}
//if this isn't their ip, don't send it to them
return cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted].some((shadowHiddenSegment) => {
return cache.shadowHiddenSegmentIPs[videoID][segment.timeSubmitted]?.some((shadowHiddenSegment) => {
if (cache.userHashedIP === undefined) {
//hash the IP only if it's strictly necessary
cache.userHashedIP = getHash((getIP(req) + config.globalSalt) as IPAddress);
}
return shadowHiddenSegment.hashedIP === cache.userHashedIP;
});
}) ?? false;
}));
const filteredSegments = segments.filter((_, index) => shouldFilter[index]);
@@ -80,7 +80,7 @@ async function getSegmentsByVideoID(req: Request, videoID: VideoID, categories:
return segments;
} catch (err) {
if (err) {
Logger.error(err);
Logger.error(err as string);
return null;
}
}
@@ -125,11 +125,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
return segments;
} catch (err) {
if (err) {
Logger.error(err);
Logger.error(err as string);
return null;
}
}
}
async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise<DBSegment[]> {

2
src/routes/getUsername.ts
View File

@@ -28,7 +28,7 @@ export async function getUsername(req: Request, res: Response): Promise<Response
});
}
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}
}

2
src/routes/getViewsForUser.ts
View File

@@ -26,7 +26,7 @@ export async function getViewsForUser(req: Request, res: Response): Promise<Resp
return res.sendStatus(404);
}
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}
}

4
src/routes/postLockCategories.ts
View File

@@ -68,7 +68,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category", "hashedVideoID", "reason") VALUES(?, ?, ?, ?, ?)`, [videoID, userID, category, hashedVideoID, reason]);
} catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`);
Logger.error(err);
Logger.error(err as string);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",
});
@@ -89,7 +89,7 @@ export async function postLockCategories(req: Request, res: Response): Promise<s
[reason, userID, videoID, category]);
} catch (err) {
Logger.error(`Error submitting 'lockCategories' marker for category '${category}' for video '${videoID}'`);
Logger.error(err);
Logger.error(err as string);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",
});

2
src/routes/postPurgeAllSegments.ts
View File

@@ -29,7 +29,7 @@ export async function postPurgeAllSegments(req: Request, res: Response): Promise
await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "videoID" = ?`, [videoID]);
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}
return res.sendStatus(200);

2
src/routes/postSegmentShift.ts
View File

@@ -92,7 +92,7 @@ export async function postSegmentShift(req: Request, res: Response): Promise<Res
}
}
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}

8
src/routes/postSkipSegments.ts
View File

@@ -390,9 +390,9 @@ async function checkEachSegmentValid(userID: string, videoID: VideoID
return { pass: false, errorMessage: "One of your segments times are invalid (too short, startTime before endTime, etc.)", errorCode: 400};
}
// Check for POI segments before 1 second
if (getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime < 1) {
return { pass: false, errorMessage: "POI must be after 1 second", errorCode: 400};
// Check for POI segments before some seconds
if (!isVIP && getCategoryActionType(segments[i].category) === CategoryActionType.POI && startTime < config.poiMinimumStartTime) {
return { pass: false, errorMessage: `POI cannot be that early`, errorCode: 400};
}
if (!isVIP && segments[i].category === "sponsor" && Math.abs(startTime - endTime) < 1) {
@@ -684,7 +684,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
});
}
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}

4
src/routes/setUsername.ts
View File

@@ -57,7 +57,7 @@ export async function setUsername(req: Request, res: Response): Promise<Response
}
}
catch (error) {
Logger.error(error);
Logger.error(error as string);
return res.sendStatus(500);
}
@@ -80,7 +80,7 @@ export async function setUsername(req: Request, res: Response): Promise<Response
return res.sendStatus(200);
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.sendStatus(500);
}
}

2
src/routes/voteOnSponsorTime.ts
View File

@@ -458,7 +458,7 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise<Re
}
return res.status(finalResponse.finalStatus).send(finalResponse.finalMessage ?? undefined);
} catch (err) {
Logger.error(err);
Logger.error(err as string);
return res.status(500).json({error: "Internal error creating segment vote"});
}
}

2
src/types/youtubeApi.model.ts
View File

@@ -106,6 +106,6 @@ export interface APIVideoData {
}
export interface APIVideoInfo {
err: string | boolean,
err: boolean | string,
data?: APIVideoData
}

4
src/utils/youtubeApi.ts
View File

@@ -20,7 +20,7 @@ export class YouTubeAPI {
return { err: null, data: JSON.parse(data) };
}
} catch (err) {
return { err };
return { err: err as string | boolean, data: null };
}
}
@@ -45,7 +45,7 @@ export class YouTubeAPI {
return { err: result.statusText, data: null };
}
} catch (err) {
return {err, data: null};
return {err: err as string | boolean, data: null};
}
}
}

2
test/cases/setUsername.ts
View File

@@ -1,5 +1,5 @@
import fetch from "node-fetch";
import { Done, getbaseURL, partialDeepEquals } from "../utils";
import { Done, getbaseURL } from "../utils";
import { db, privateDB } from "../../src/databases/databases";
import { getHash } from "../../src/utils/getHash";
import assert from "assert";