Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2026-01-28 21:30:53 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into export

Browse Source
This commit is contained in:
Ajay Ramachandran
2021-05-23 11:17:10 -04:00
parent f6f5570d0c e71399f5af
commit 590ed037dd
29 changed files with 5190 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.MD
View File

@@ -8,7 +8,7 @@ This is the server backend for it
This uses a Postgres or Sqlite database to hold all the timing data. This uses a Postgres or Sqlite database to hold all the timing data.
To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea. Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.

27
config.json.example
View File

@@ -38,5 +38,32 @@
"max": 20, // 20 requests in 15min time window "max": 20, // 20 requests in 15min time window
"statusCode": 200 "statusCode": 200
} }
},
"maxRewardTimePerSegmentInSeconds": 86400, // maximum time a user get rewarded in the leaderboard for a single segment
"dumpDatabase": {
"enabled": true,
"minTimeBetweenMs": 60000, // 1 minute between dumps
"appExportPath": "./docker/database-export",
"postgresExportPath": "/opt/exports",
"tables": [{
"name": "sponsorTimes",
"order": "timeSubmitted"
},
{
"name": "userNames"
},
{
"name": "categoryVotes"
},
{
"name": "lockCategories"
},
{
"name": "warnings",
"order": "issueTime"
},
{
"name": "vipUsers"
}]
} }
} }

4
databases/_sponsorTimes_indexes.sql
View File

@@ -51,10 +51,10 @@ CREATE INDEX IF NOT EXISTS "warnings_issueTime"
("issueTime" ASC NULLS LAST) ("issueTime" ASC NULLS LAST)
TABLESPACE pg_default; TABLESPACE pg_default;
-- noSegments -- lockCategories
CREATE INDEX IF NOT EXISTS "noSegments_videoID" CREATE INDEX IF NOT EXISTS "noSegments_videoID"
ON public."noSegments" USING btree ON public."lockCategories" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default; TABLESPACE pg_default;

8
databases/_upgrade_sponsorTimes_11.sql Normal file
View File

@@ -0,0 +1,8 @@
BEGIN TRANSACTION;
/* Rename table: noSegments to lockCategories */
ALTER TABLE "noSegments" RENAME TO "lockCategories";
UPDATE "config" SET value = 11 WHERE key = 'version';
COMMIT;

8
docker/docker-compose.yml
View File

@@ -7,9 +7,10 @@ services:
- database.env - database.env
volumes: volumes:
- database-data:/var/lib/postgresql/data - database-data:/var/lib/postgresql/data
- ./database-export/:/opt/exports - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
ports: ports:
- 127.0.0.1:5432:5432 - 5432:5432
restart: always
redis: redis:
container_name: redis container_name: redis
image: redis image: redis
@@ -17,7 +18,8 @@ services:
volumes: volumes:
- ./redis/redis.conf:/usr/local/etc/redis/redis.conf - ./redis/redis.conf:/usr/local/etc/redis/redis.conf
ports: ports:
- 127.0.0.1:32773:6379 - 32773:6379
restart: always
volumes: volumes:
database-data: database-data:

167
nginx/nginx.conf
View File

@@ -2,7 +2,7 @@ worker_processes 8;
worker_rlimit_nofile 8192; worker_rlimit_nofile 8192;
events { events {
worker_connections 32768; ## Default: 1024 worker_connections 132768; ## Default: 1024
} }
http { http {
@@ -12,28 +12,41 @@ http {
upstream backend_GET { upstream backend_GET {
least_conn; least_conn;
server localhost:4442; server localhost:4441;
server localhost:4443; server localhost:4442;
server localhost:4444; #server localhost:4443;
server localhost:4445; #server localhost:4444;
server localhost:4446; #server localhost:4445;
#server localhost:4446;
#server localhost:4447; #server localhost:4447;
#server localhost:4448; #server localhost:4448;
server 10.0.0.3:4441;
server 10.0.0.3:4442;
#server 134.209.69.251:80 backup;
server 116.203.32.253:80 backup;
#server 116.203.32.253:80;
} }
upstream backend_POST { upstream backend_POST {
#server localhost:4441;
#server localhost:4442;
server 10.0.0.3:4441;
#server 10.0.0.3:4442;
}
upstream backend_db {
server localhost:4441; server localhost:4441;
#server 10.0.0.3:4441;
} }
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m; proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
proxy_cache_key "$scheme$request_method$host$request_uri"; proxy_cache_key "$scheme$request_method$host$request_uri";
add_header X-Cache $upstream_cache_status; add_header X-Cache $upstream_cache_status;
server { server {
server_name sponsor.ajay.app api.sponsor.ajay.app; server_name sponsor.ajay.app api.sponsor.ajay.app;
access_log off;
error_log /dev/null;
error_page 404 /404.html; error_page 404 /404.html;
error_page 500 @myerrordirective_500; error_page 500 @myerrordirective_500;
error_page 502 @myerrordirective_502; error_page 502 @myerrordirective_502;
@@ -43,14 +56,16 @@ http {
# internal; # internal;
#} #}
#proxy_send_timeout 120s;
location @myerrordirective_500 { location @myerrordirective_500 {
return 502 "Internal Server Error"; return 400 "Internal Server Error";
} }
location @myerrordirective_502 { location @myerrordirective_502 {
return 502 "Bad Gateway"; return 400 "Bad Gateway";
} }
location @myerrordirective_504 { location @myerrordirective_504 {
return 502 "Gateway Timeout"; return 400 "Gateway Timeout";
} }
@@ -62,17 +77,16 @@ http {
return 301 https://sb.ltn.fi; return 301 https://sb.ltn.fi;
} }
location /invidious/ {
proxy_pass https://invidious.fdn.fr/;
}
location /test/ { location /test/ {
proxy_pass http://localhost:4440/; proxy_pass http://localhost:4440/;
#proxy_pass https://sbtest.etcinit.com/; #proxy_pass https://sbtest.etcinit.com/;
} }
location /api/skipSegments { location /api/skipSegments {
proxy_pass http://backend_$request_method; #return 200 "[]";
proxy_pass http://backend_$request_method;
#proxy_cache CACHEZONE;
#proxy_cache_valid 2m;
} }
location /api/getTopUsers { location /api/getTopUsers {
@@ -83,23 +97,112 @@ http {
location /api/getTotalStats { location /api/getTotalStats {
proxy_pass http://backend_GET; proxy_pass http://backend_GET;
} #return 200 "";
}
location /api/getVideoSponsorTimes { location /api/getVideoSponsorTimes {
proxy_pass http://backend_GET; proxy_pass http://backend_GET;
} }
location = /database.db { location /download/ {
alias /home/sbadmin/sponsor/databases/sponsorTimes.db; #alias /home/sbadmin/sponsor/docker/database-export/;
return 307 https://cdnsponsor.ajay.app$request_uri;
}
location /database {
proxy_pass http://backend_db;
#return 200 "Disabled for load reasons";
} }
location = /database.db {
#return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
#alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
alias /home/sbadmin/test-db/database.db;
}
#location = /database/sponsorTimes.csv {
# alias /home/sbadmin/sponsorTimes.csv;
#}
#location /api/voteOnSponsorTime {
# return 200 "Success";
#}
#location /api/viewedVideoSponsorTime {
# return 200 "Success";
#}
location /api { location /api {
proxy_pass http://backend_POST; proxy_pass http://backend_POST;
} }
location / { location / {
root /home/sbadmin/caddy/SponsorBlockSite/public-prod; root /home/sbadmin/SponsorBlockSite/public-prod;
### CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
#
# Custom headers and headers various browsers *should* be OK with but aren't
#
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
#
# Tell client that this pre-flight info is valid for 20 days
#
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain; charset=utf-8';
add_header 'Content-Length' 0;
return 204;
}
if ($request_method = 'POST') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
if ($request_method = 'GET') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
}
listen 443 default_server ssl http2; # managed by Certbot
#listen 443 http3 reuseport;
#ssl_protocols TLSv1.2 TLSv1.3;
#listen 80;
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
server {
server_name cdnsponsor.ajay.app;
error_page 404 /404.html;
#location /database/ {
# alias /home/sbadmin/sponsor/docker/database-export/;
#}
location /download/ {
alias /home/sbadmin/sponsor/docker/database-export/;
}
location / {
root /home/sbadmin/SponsorBlockSite/public-prod;
### CORS ### CORS
if ($request_method = 'OPTIONS') { if ($request_method = 'OPTIONS') {
@@ -133,13 +236,13 @@ http {
listen 443 ssl; # managed by Certbot listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
}
@@ -149,6 +252,7 @@ http {
access_log off; access_log off;
error_log /dev/null; error_log /dev/null;
if ($host = api.sponsor.ajay.app) { if ($host = api.sponsor.ajay.app) {
return 301 https://$host$request_uri; return 301 https://$host$request_uri;
} # managed by Certbot } # managed by Certbot
@@ -166,4 +270,17 @@ http {
}
server {
if ($host = cdnsponsor.ajay.app) {
return 301 https://$host$request_uri;
} # managed by Certbot
server_name cdnsponsor.ajay.app;
listen 80;
return 404; # managed by Certbot
}} }}

4652
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
package.json
View File

@@ -13,7 +13,7 @@
"author": "Ajay Ramachandran", "author": "Ajay Ramachandran",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"better-sqlite3": "^5.4.3", "better-sqlite3": "^7.1.5",
"dotenv": "^8.2.0", "dotenv": "^8.2.0",
"express": "^4.17.1", "express": "^4.17.1",
"express-rate-limit": "^5.1.3", "express-rate-limit": "^5.1.3",

17
src/app.ts
View File

@@ -5,8 +5,8 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
import {postSegmentShift} from './routes/postSegmentShift'; import {postSegmentShift} from './routes/postSegmentShift';
import {postWarning} from './routes/postWarning'; import {postWarning} from './routes/postWarning';
import {getIsUserVIP} from './routes/getIsUserVIP'; import {getIsUserVIP} from './routes/getIsUserVIP';
import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments'; import {deleteLockCategoriesEndpoint} from './routes/deleteLockCategories';
import {postNoSegments} from './routes/postNoSegments'; import {postLockCategories} from './routes/postLockCategories';
import {getUserInfo} from './routes/getUserInfo'; import {getUserInfo} from './routes/getUserInfo';
import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted'; import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
import {getTotalStats} from './routes/getTotalStats'; import {getTotalStats} from './routes/getTotalStats';
@@ -25,8 +25,9 @@ import {endpoint as getSkipSegments} from './routes/getSkipSegments';
import {userCounter} from './middleware/userCounter'; import {userCounter} from './middleware/userCounter';
import {loggerMiddleware} from './middleware/logger'; import {loggerMiddleware} from './middleware/logger';
import {corsMiddleware} from './middleware/cors'; import {corsMiddleware} from './middleware/cors';
import {apiCspMiddleware} from './middleware/apiCsp';
import {rateLimitMiddleware} from './middleware/requestRateLimit'; import {rateLimitMiddleware} from './middleware/requestRateLimit';
import dumpDatabase from './routes/dumpDatabase'; import dumpDatabase, {redirectLink} from './routes/dumpDatabase';
export function createServer(callback: () => void) { export function createServer(callback: () => void) {
@@ -36,6 +37,7 @@ export function createServer(callback: () => void) {
//setup CORS correctly //setup CORS correctly
app.use(corsMiddleware); app.use(corsMiddleware);
app.use(loggerMiddleware); app.use(loggerMiddleware);
app.use("/api/", apiCspMiddleware);
app.use(express.json()); app.use(express.json());
if (config.userCounterURL) app.use(userCounter); if (config.userCounterURL) app.use(userCounter);
@@ -114,10 +116,12 @@ function setupRoutes(app: Express) {
//send out a formatted time saved total //send out a formatted time saved total
app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
//submit video containing no segments //submit video to lock categories
app.post('/api/noSegments', postNoSegments); app.post('/api/noSegments', postLockCategories);
app.post('/api/lockCategories', postLockCategories);
app.delete('/api/noSegments', deleteNoSegmentsEndpoint); app.delete('/api/noSegments', deleteLockCategoriesEndpoint);
app.delete('/api/lockCategories', deleteLockCategoriesEndpoint);
//get if user is a vip //get if user is a vip
app.get('/api/isUserVIP', getIsUserVIP); app.get('/api/isUserVIP', getIsUserVIP);
@@ -131,6 +135,7 @@ function setupRoutes(app: Express) {
if (config.postgres) { if (config.postgres) {
app.get('/database', (req, res) => dumpDatabase(req, res, true)); app.get('/database', (req, res) => dumpDatabase(req, res, true));
app.get('/database.json', (req, res) => dumpDatabase(req, res, false)); app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
app.get('/database/*', redirectLink)
} else { } else {
app.get('/database.db', function (req: Request, res: Response) { app.get('/database.db', function (req: Request, res: Response) {
res.sendFile("./databases/sponsorTimes.db", {root: "./"}); res.sendFile("./databases/sponsorTimes.db", {root: "./"});

29
src/config.ts
View File

@@ -45,7 +45,34 @@ addDefaults(config, {
}, },
userCounterURL: null, userCounterURL: null,
youtubeAPIKey: null, youtubeAPIKey: null,
postgres: null maxRewardTimePerSegmentInSeconds: 86400,
postgres: null,
dumpDatabase: {
enabled: false,
minTimeBetweenMs: 60000,
appExportPath: './docker/database-export',
postgresExportPath: '/opt/exports',
tables: [{
name: "sponsorTimes",
order: "timeSubmitted"
},
{
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "lockCategories",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}]
}
}); });
// Add defaults // Add defaults

6
src/middleware/apiCsp.ts Normal file
View File

@@ -0,0 +1,6 @@
import {NextFunction, Request, Response} from 'express';
export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) {
res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
next();
}

12
src/routes/deleteNoSegments.ts → src/routes/deleteLockCategories.ts
View File

@@ -5,7 +5,7 @@ import {db} from '../databases/databases';
import { Category, VideoID } from '../types/segments.model'; import { Category, VideoID } from '../types/segments.model';
import { UserID } from '../types/user.model'; import { UserID } from '../types/user.model';
export async function deleteNoSegmentsEndpoint(req: Request, res: Response) { export async function deleteLockCategoriesEndpoint(req: Request, res: Response) {
// Collect user input data // Collect user input data
const videoID = req.body.videoID as VideoID; const videoID = req.body.videoID as VideoID;
const userID = req.body.userID as UserID; const userID = req.body.userID as UserID;
@@ -35,9 +35,9 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
return; return;
} }
deleteNoSegments(videoID, categories); deleteLockCategories(videoID, categories);
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID}); res.status(200).json({message: 'Removed lock categories entrys for video ' + videoID});
} }
/** /**
@@ -45,12 +45,12 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
* @param videoID * @param videoID
* @param categories If null, will remove all * @param categories If null, will remove all
*/ */
export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise<void> { export async function deleteLockCategories(videoID: VideoID, categories: Category[]): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => { const entries = (await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
return categories === null || categories.indexOf(entry.category) !== -1; return categories === null || categories.indexOf(entry.category) !== -1;
}); });
for (const entry of entries) { for (const entry of entries) {
await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]); await db.prepare('run', 'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
} }
} }

203
src/routes/dumpDatabase.ts
View File

@@ -2,51 +2,111 @@ import {db} from '../databases/databases';
import {Logger} from '../utils/logger'; import {Logger} from '../utils/logger';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
import { config } from '../config'; import { config } from '../config';
import util from 'util';
import fs from 'fs';
import path from 'path';
const unlink = util.promisify(fs.unlink);
const ONE_MINUTE = 1000 * 60; const ONE_MINUTE = 1000 * 60;
const styleHeader = `<style>body{font-family: sans-serif}</style>` const styleHeader = `<style>
body {
font-family: sans-serif
}
table th,
table td {
padding: 7px;
}
table th {
text-align: left;
}
table tbody tr:nth-child(odd) {
background: #efefef;
}
</style>`
const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p> const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p> <p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`; <p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
const tables = [{ const tables = config?.dumpDatabase?.tables ?? [];
name: "sponsorTimes", const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
order: "timeSubmitted" const appExportPath = config?.dumpDatabase?.appExportPath ?? './docker/database-export';
}, const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? '/opt/exports';
{ const tableNames = tables.map(table => table.name);
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "noSegments",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}];
const links: string[] = tables.map((table) => `/database/${table.name}.csv`); interface TableDumpList {
fileName: string;
tableName: string;
};
let latestDumpFiles: TableDumpList[] = [];
const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`) interface TableFile {
.reduce((acc, url) => acc + url, ""); file: string,
timestamp: number
};
if (tables.length === 0) {
Logger.warn('[dumpDatabase] No tables configured');
}
let lastUpdate = 0; let lastUpdate = 0;
let updateQueued = false;
let updateRunning = false;
export default function dumpDatabase(req: Request, res: Response, showPage: boolean) { function removeOutdatedDumps(exportPath: string): Promise<void> {
return new Promise((resolve, reject) => {
// Get list of table names
// Create array for each table
const tableFiles: Record<string, TableFile[]> = tableNames.reduce((obj: any, tableName) => {
obj[tableName] = [];
return obj;
}, {});
// read files in export directory
fs.readdir(exportPath, async (err: any, files: string[]) => {
if (err) Logger.error(err);
if (err) return resolve();
files.forEach(file => {
// we only care about files that start with "<tablename>_" and ends with .csv
tableNames.forEach(tableName => {
if (file.startsWith(`${tableName}`) && file.endsWith('.csv')) {
const filePath = path.join(exportPath, file);
tableFiles[tableName].push({
file: filePath,
timestamp: fs.statSync(filePath).mtime.getTime()
});
}
});
});
for (let tableName in tableFiles) {
const files = tableFiles[tableName].sort((a, b) => b.timestamp - a.timestamp);
for (let i = 2; i < files.length; i++) {
// remove old file
await unlink(files[i].file).catch((error: any) => {
Logger.error(`[dumpDatabase] Garbage collection failed ${error}`);
});
}
}
resolve();
});
});
}
export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) { if (!config.postgres) {
res.status(404).send("Not supported on this instance"); res.status(404).send("Not supported on this instance");
return; return;
} }
const now = Date.now(); updateQueueTime();
const updateQueued = now - lastUpdate > ONE_MINUTE;
res.status(200) res.status(200)
@@ -54,25 +114,100 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
res.send(`${styleHeader} res.send(`${styleHeader}
<h1>SponsorBlock database dumps</h1>${licenseHeader} <h1>SponsorBlock database dumps</h1>${licenseHeader}
<h3>How this works</h3> <h3>How this works</h3>
Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to get a list of urls and the update status database dump to run.
Then, you can download the csv files below, or use the links returned from the JSON request. Then, you can download the csv files below, or use the links returned from the JSON request.
A dump will also be triggered by making a request to one of these urls.
<h3>Links</h3> <h3>Links</h3>
${linksHTML}<br/> <table>
<thead>
<tr>
<th>Table</th>
<th>CSV</th>
</tr>
</thead>
<tbody>
${latestDumpFiles.map((item:any) => {
return `
<tr>
<td>${item.tableName}</td>
<td><a href="/database/${item.tableName}.csv">${item.tableName}.csv</a></td>
</tr>
`;
}).join('')}
${latestDumpFiles.length === 0 ? '<tr><td colspan="2">Please wait: Generating files</td></tr>' : ''}
</tbody>
</table>
<hr/>
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
} else { } else {
res.send({ res.send({
lastUpdated: lastUpdate, lastUpdated: lastUpdate,
updateQueued, updateQueued,
links links: latestDumpFiles.map((item:any) => {
return {
table: item.tableName,
url: `/database/${item.tableName}.csv`,
size: item.fileSize,
};
}),
}) })
} }
if (updateQueued) { await queueDump();
lastUpdate = Date.now(); }
export async function redirectLink(req: Request, res: Response): Promise<void> {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) {
res.status(404).send("Not supported on this instance");
return;
}
const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path);
updateQueueTime();
if (file) {
res.redirect("/download/" + file.fileName);
} else {
res.status(404).send();
}
await queueDump();
}
function updateQueueTime(): void {
updateQueued ||= Date.now() - lastUpdate > MILLISECONDS_BETWEEN_DUMPS;
}
async function queueDump(): Promise<void> {
if (updateQueued && !updateRunning) {
const startTime = Date.now();
updateRunning = true;
await removeOutdatedDumps(appExportPath);
const dumpFiles = [];
for (const table of tables) { for (const table of tables) {
db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) const fileName = `${table.name}_${startTime}.csv`;
TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); const file = `${postgresExportPath}/${fileName}`;
await db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '${file}' WITH (FORMAT CSV, HEADER true);`);
dumpFiles.push({
fileName,
tableName: table.name,
});
} }
latestDumpFiles = [...dumpFiles];
updateQueued = false;
updateRunning = false;
lastUpdate = startTime;
} }
} }

5
src/routes/getSavedTimeForUser.ts
View File

@@ -1,8 +1,11 @@
import {db} from '../databases/databases'; import {db} from '../databases/databases';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
import {getHash} from '../utils/getHash'; import {getHash} from '../utils/getHash';
import {config} from '../config';
import { Logger } from '../utils/logger'; import { Logger } from '../utils/logger';
const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
export async function getSavedTimeForUser(req: Request, res: Response) { export async function getSavedTimeForUser(req: Request, res: Response) {
let userID = req.query.userID as string; let userID = req.query.userID as string;
@@ -16,7 +19,7 @@ export async function getSavedTimeForUser(req: Request, res: Response) {
userID = getHash(userID); userID = getHash(userID);
try { try {
let row = await db.prepare("get", 'SELECT SUM(("endTime" - "startTime") / 60 * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]); let row = await db.prepare("get", 'SELECT SUM(((CASE WHEN "endTime" - "startTime" > ' + maxRewardTimePerSegmentInSeconds + ' THEN ' + maxRewardTimePerSegmentInSeconds + ' ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]);
if (row.minutesSaved != null) { if (row.minutesSaved != null) {
res.send({ res.send({

4
src/routes/getSkipSegments.ts
View File

@@ -268,6 +268,10 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
: req.query.category : req.query.category
? [req.query.category] ? [req.query.category]
: ['sponsor']; : ['sponsor'];
if (!Array.isArray(categories)) {
res.status(400).send("Categories parameter does not match format requirements.");
return false;
}
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube; let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) { if (!Object.values(Service).some((val) => val == service)) {

9
src/routes/getTopUsers.ts
View File

@@ -5,6 +5,7 @@ import {Request, Response} from 'express';
const MILLISECONDS_IN_MINUTE = 60000; const MILLISECONDS_IN_MINUTE = 60000;
const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE); const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = false) { async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = false) {
const userNames = []; const userNames = [];
@@ -24,14 +25,14 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
} }
const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
SUM(("sponsorTimes"."endTime" - "sponsorTimes"."startTime") / 60 * "sponsorTimes"."views") as "minutesSaved", SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ${maxRewardTimePerSegmentInSeconds} THEN ${maxRewardTimePerSegmentInSeconds} ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
SUM("votes") as "userVotes", ` + SUM("votes") as "userVotes", ` +
additionalFields + additionalFields +
`IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" `IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID" LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID"
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL
GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20 GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
ORDER BY "` + sortBy + `" DESC LIMIT 100`, []); ORDER BY "${sortBy}" DESC LIMIT 100`, []);
for (let i = 0; i < rows.length; i++) { for (let i = 0; i < rows.length; i++) {
userNames[i] = rows[i].userName; userNames[i] = rows[i].userName;
@@ -70,6 +71,10 @@ export async function getTopUsers(req: Request, res: Response) {
return; return;
} }
//TODO: remove. This is broken for now
res.status(200).send();
return;
//setup which sort type to use //setup which sort type to use
let sortBy = ''; let sortBy = '';
if (sortType == 0) { if (sortType == 0) {

42
src/routes/getUserInfo.ts
View File

@@ -1,11 +1,15 @@
import {db} from '../databases/databases'; import {db} from '../databases/databases';
import {getHash} from '../utils/getHash'; import {getHash} from '../utils/getHash';
import {isUserVIP} from '../utils/isUserVIP';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
import {Logger} from '../utils/logger' import {Logger} from '../utils/logger';
import { HashedUserID, UserID } from '../types/user.model';
async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSaved: number, segmentCount: number }> { async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> {
try { try {
let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved", count(*) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved",
count(*) as "segmentCount" FROM "sponsorTimes"
WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
if (row.minutesSaved != null) { if (row.minutesSaved != null) {
return { return {
minutesSaved: row.minutesSaved, minutesSaved: row.minutesSaved,
@@ -22,7 +26,7 @@ async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSa
} }
} }
async function dbGetUsername(userID: string) { async function dbGetUsername(userID: HashedUserID) {
try { try {
let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
if (row !== undefined) { if (row !== undefined) {
@@ -36,24 +40,19 @@ async function dbGetUsername(userID: string) {
} }
} }
async function dbGetViewsForUser(userID: string) { async function dbGetViewsForUser(userID: HashedUserID) {
try { try {
let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
//increase the view count by one return row?.viewCount ?? 0;
if (row.viewCount != null) {
return row.viewCount;
} else {
return 0;
}
} catch (err) { } catch (err) {
return false; return false;
} }
} }
async function dbGetWarningsForUser(userID: string): Promise<number> { async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
try { try {
let rows = await db.prepare('all', `SELECT * FROM "warnings" WHERE "userID" = ?`, [userID]); let row = await db.prepare('get', `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]);
return rows.length; return row?.total ?? 0;
} catch (err) { } catch (err) {
Logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0'); Logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0');
return 0; return 0;
@@ -61,7 +60,7 @@ async function dbGetWarningsForUser(userID: string): Promise<number> {
} }
export async function getUserInfo(req: Request, res: Response) { export async function getUserInfo(req: Request, res: Response) {
let userID = req.query.userID as string; let userID = req.query.userID as UserID;
if (userID == undefined) { if (userID == undefined) {
//invalid request //invalid request
@@ -70,17 +69,18 @@ export async function getUserInfo(req: Request, res: Response) {
} }
//hash the userID //hash the userID
userID = getHash(userID); const hashedUserID: HashedUserID = getHash(userID);
const segmentsSummary = await dbGetSubmittedSegmentSummary(userID); const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID);
if (segmentsSummary) { if (segmentsSummary) {
res.send({ res.send({
userID, userID: hashedUserID,
userName: await dbGetUsername(userID), userName: await dbGetUsername(hashedUserID),
minutesSaved: segmentsSummary.minutesSaved, minutesSaved: segmentsSummary.minutesSaved,
segmentCount: segmentsSummary.segmentCount, segmentCount: segmentsSummary.segmentCount,
viewCount: await dbGetViewsForUser(userID), viewCount: await dbGetViewsForUser(hashedUserID),
warnings: await dbGetWarningsForUser(userID), warnings: await dbGetWarningsForUser(hashedUserID),
vip: await isUserVIP(hashedUserID),
}); });
} else { } else {
res.status(400).send(); res.status(400).send();

18
src/routes/postNoSegments.ts → src/routes/postLockCategories.ts
View File

@@ -4,7 +4,7 @@ import {isUserVIP} from '../utils/isUserVIP';
import {db} from '../databases/databases'; import {db} from '../databases/databases';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
export async function postNoSegments(req: Request, res: Response) { export async function postLockCategories(req: Request, res: Response) {
// Collect user input data // Collect user input data
let videoID = req.body.videoID; let videoID = req.body.videoID;
let userID = req.body.userID; let userID = req.body.userID;
@@ -34,12 +34,12 @@ export async function postNoSegments(req: Request, res: Response) {
return; return;
} }
// Get existing no segment markers // Get existing lock categories markers
let noSegmentList = await db.prepare('all', 'SELECT "category" from "noSegments" where "videoID" = ?', [videoID]); let noCategoryList = await db.prepare('all', 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]);
if (!noSegmentList || noSegmentList.length === 0) { if (!noCategoryList || noCategoryList.length === 0) {
noSegmentList = []; noCategoryList = [];
} else { } else {
noSegmentList = noSegmentList.map((obj: any) => { noCategoryList = noCategoryList.map((obj: any) => {
return obj.category; return obj.category;
}); });
} }
@@ -48,7 +48,7 @@ export async function postNoSegments(req: Request, res: Response) {
let categoriesToMark = categories.filter((category) => { let categoriesToMark = categories.filter((category) => {
return !!category.match(/^[_a-zA-Z]+$/); return !!category.match(/^[_a-zA-Z]+$/);
}).filter((category) => { }).filter((category) => {
return noSegmentList.indexOf(category) === -1; return noCategoryList.indexOf(category) === -1;
}); });
// remove any duplicates // remove any duplicates
@@ -59,9 +59,9 @@ export async function postNoSegments(req: Request, res: Response) {
// create database entry // create database entry
for (const category of categoriesToMark) { for (const category of categoriesToMark) {
try { try {
await db.prepare('run', `INSERT INTO "noSegments" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]); await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]);
} catch (err) { } catch (err) {
Logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'"); Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
Logger.error(err); Logger.error(err);
res.status(500).json({ res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.", message: "Internal Server Error: Could not write marker to the database.",

14
src/routes/postSkipSegments.ts
View File

@@ -14,7 +14,7 @@ import {Request, Response} from 'express';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import redis from '../utils/redis'; import redis from '../utils/redis';
import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { deleteNoSegments } from './deleteNoSegments'; import { deleteLockCategories } from './deleteLockCategories';
import { getCategoryActionType } from '../utils/categoryInfo'; import { getCategoryActionType } from '../utils/categoryInfo';
interface APIVideoInfo { interface APIVideoInfo {
@@ -357,7 +357,7 @@ export async function postSkipSegments(req: Request, res: Response) {
return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?'); return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
} }
let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => { let lockedCategoryList = (await db.prepare('all', 'SELECT category from "lockCategories" where "videoID" = ?', [videoID])).map((list: any) => {
return list.category; return list.category;
}); });
@@ -389,9 +389,9 @@ export async function postSkipSegments(req: Request, res: Response) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]); await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
} }
// Reset no segments // Reset lock categories
noSegmentList = []; lockedCategoryList = [];
deleteNoSegments(videoID, null); deleteLockCategories(videoID, null);
} }
// Check if all submissions are correct // Check if all submissions are correct
@@ -407,8 +407,8 @@ export async function postSkipSegments(req: Request, res: Response) {
return; return;
} }
// Reject segemnt if it's in the no segments list // Reject segment if it's in the locked categories list
if (!isVIP && noSegmentList.indexOf(segments[i].category) !== -1) { if (!isVIP && lockedCategoryList.indexOf(segments[i].category) !== -1) {
// TODO: Do something about the fradulent submission // TODO: Do something about the fradulent submission
Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'"); Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
res.status(403).send( res.status(403).send(

10
src/routes/setUsername.ts
View File

@@ -22,6 +22,10 @@ export async function setUsername(req: Request, res: Response) {
return; return;
} }
// remove unicode control characters from username (example: \n, \r, \t etc.)
// source: https://en.wikipedia.org/wiki/Control_character#In_Unicode
userName = userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, '');
if (adminUserIDInput != undefined) { if (adminUserIDInput != undefined) {
//this is the admin controlling the other users account, don't hash the controling account's ID //this is the admin controlling the other users account, don't hash the controling account's ID
adminUserIDInput = getHash(adminUserIDInput); adminUserIDInput = getHash(adminUserIDInput);
@@ -36,6 +40,12 @@ export async function setUsername(req: Request, res: Response) {
userID = getHash(userID); userID = getHash(userID);
} }
if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148"].includes(userID)) {
// Don't allow
res.sendStatus(200);
return;
}
try { try {
//check if username is already set //check if username is already set
let row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ?`, [userID]); let row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ?`, [userID]);

6
src/routes/shadowBanUser.ts
View File

@@ -43,8 +43,8 @@ export async function shadowBanUser(req: Request, res: Response) {
//find all previous submissions and hide them //find all previous submissions and hide them
if (unHideOldSubmissions) { if (unHideOldSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ?
AND NOT EXISTS ( SELECT "videoID", "category" FROM "noSegments" WHERE AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
"sponsorTimes"."videoID" = "noSegments"."videoID" AND "sponsorTimes"."category" = "noSegments"."category")`, [userID]); "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
} }
} else if (!enabled && row.userCount > 0) { } else if (!enabled && row.userCount > 0) {
//remove them from the shadow ban list //remove them from the shadow ban list
@@ -53,7 +53,7 @@ export async function shadowBanUser(req: Request, res: Response) {
//find all previous submissions and unhide them //find all previous submissions and unhide them
if (unHideOldSubmissions) { if (unHideOldSubmissions) {
let segmentsToIgnore = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st let segmentsToIgnore = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st
JOIN "noSegments" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?` JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?`
, [userID])).map((item: {UUID: string}) => item.UUID); , [userID])).map((item: {UUID: string}) => item.UUID);
let allSegments = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID])) let allSegments = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID]))
.map((item: {UUID: string}) => item.UUID); .map((item: {UUID: string}) => item.UUID);

92
src/routes/voteOnSponsorTime.ts
View File

@@ -180,51 +180,54 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
const timeSubmitted = Date.now(); const timeSubmitted = Date.now();
const voteAmount = isVIP ? 500 : 1; const voteAmount = isVIP ? 500 : 1;
const ableToVote = isVIP || finalResponse.finalStatus === 200 || true;
// Add the vote if (ableToVote) {
if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) { // Add the vote
// Update the already existing db entry if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]); // Update the already existing db entry
} else { await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
// Add a db entry } else {
await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]); // Add a db entry
} await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
}
// Add the info into the private db // Add the info into the private db
if (usersLastVoteInfo?.votes > 0) { if (usersLastVoteInfo?.votes > 0) {
// Reverse the previous vote // Reverse the previous vote
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]); await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);
await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]); await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
} else { } else {
await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]); await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
} }
// See if the submissions category is ready to change // See if the submissions category is ready to change
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]); const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
const startingVotes = isSubmissionVIP ? 10000 : 1; const startingVotes = isSubmissionVIP ? 10000 : 1;
// Change this value from 1 in the future to make it harder to change categories // Change this value from 1 in the future to make it harder to change categories
// Done this way without ORs incase the value is zero // Done this way without ORs incase the value is zero
const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes; const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
// Add submission as vote // Add submission as vote
if (!currentCategoryInfo && submissionInfo) { if (!currentCategoryInfo && submissionInfo) {
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]); await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]); await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
} }
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
//TODO: In the future, raise this number from zero to make it harder to change categories //TODO: In the future, raise this number from zero to make it harder to change categories
// VIPs change it every time // VIPs change it every time
if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) { if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
// Replace the category // Replace the category
await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]); await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
}
} }
clearRedisCache(videoInfo); clearRedisCache(videoInfo);
@@ -273,8 +276,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
// If not upvote // If not upvote
if (!isVIP && type !== 1) { if (!isVIP && type !== 1) {
const isSegmentLocked = async () => !!(await db.prepare('get', `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked; const isSegmentLocked = async () => !!(await db.prepare('get', `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked;
const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "noSegments".category from "noSegments" left join "sponsorTimes"' + const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "lockCategories".category from "lockCategories" left join "sponsorTimes"' +
' on ("noSegments"."videoID" = "sponsorTimes"."videoID" and "noSegments".category = "sponsorTimes".category)' + ' on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and "lockCategories".category = "sponsorTimes".category)' +
' where "UUID" = ?', [UUID])); ' where "UUID" = ?', [UUID]));
if (await isSegmentLocked() || await isVideoLocked()) { if (await isSegmentLocked() || await isVideoLocked()) {
@@ -287,13 +290,19 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res); return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res);
} }
if (type == 1 && !isVIP && !isOwnSubmission) { if (type !== undefined && !isVIP && !isOwnSubmission) {
// Check if upvoting hidden segment // Check if upvoting hidden segment
const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
if (voteInfo && voteInfo.votes <= -2) { if (voteInfo && voteInfo.votes <= -2) {
res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP."); if (type == 1) {
return; res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
return;
} else if (type == 0) {
// Already downvoted enough, ignore
res.status(200).send();
return;
}
} }
} }
@@ -378,7 +387,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
const ableToVote = isVIP const ableToVote = isVIP
|| ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined || ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined); && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
&& finalResponse.finalStatus === 200;
if (ableToVote) { if (ableToVote) {
//update the votes table //update the votes table

15
src/types/config.model.ts
View File

@@ -37,7 +37,9 @@ export interface SBSConfig {
minimumPrefix?: string; minimumPrefix?: string;
maximumPrefix?: string; maximumPrefix?: string;
redis?: redis.ClientOpts; redis?: redis.ClientOpts;
maxRewardTimePerSegmentInSeconds?: number;
postgres?: PoolConfig; postgres?: PoolConfig;
dumpDatabase?: DumpDatabase;
} }
export interface WebhookConfig { export interface WebhookConfig {
@@ -62,3 +64,16 @@ export interface PostgresConfig {
enableWalCheckpointNumber: boolean; enableWalCheckpointNumber: boolean;
postgres: PoolConfig; postgres: PoolConfig;
} }
export interface DumpDatabase {
enabled: boolean;
minTimeBetweenMs: number;
appExportPath: string;
postgresExportPath: string;
tables: DumpDatabaseTable[];
}
export interface DumpDatabaseTable {
name: string;
order?: string;
}

4
src/utils/redis.ts
View File

@@ -29,6 +29,10 @@ if (config.redis) {
exportObject.getAsync = (key) => new Promise((resolve) => client.get(key, (err, reply) => resolve({err, reply}))); exportObject.getAsync = (key) => new Promise((resolve) => client.get(key, (err, reply) => resolve({err, reply})));
exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({err, reply}))); exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({err, reply})));
exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err))); exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
client.on("error", function(error) {
Logger.error(error);
});
} }
export default exportObject; export default exportObject;

4
test.json
View File

@@ -10,8 +10,8 @@
"discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook", "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook",
"neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock", "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock",
"behindProxy": true, "behindProxy": true,
"db": "./test/databases/sponsorTimes.db", "db": ":memory:",
"privateDB": "./test/databases/private.db", "privateDB": ":memory:",
"createDatabaseIfNotExist": true, "createDatabaseIfNotExist": true,
"schemaFolder": "./databases", "schemaFolder": "./databases",
"dbSchema": "./databases/_sponsorTimes.db.sql", "dbSchema": "./databases/_sponsorTimes.db.sql",

86
test/cases/noSegmentRecords.ts → test/cases/lockCategoriesRecords.ts
View File

@@ -4,21 +4,21 @@ import {getHash} from '../../src/utils/getHash';
import {db} from '../../src/databases/databases'; import {db} from '../../src/databases/databases';
describe('noSegmentRecords', () => { describe('lockCategoriesRecords', () => {
before(async () => { before(async () => {
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-noSegments") + "')"); await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-lockCategories") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id', 'intro')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id-1', 'intro')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'lockCategoryVideo', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'intro')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record-1', 'intro')");
}); });
it('Should update the database version when starting the application', async () => { it('Should update the database version when starting the application', async () => {
@@ -30,7 +30,7 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories not in video (http response)', (done: Done) => { it('Should be able to submit categories not in video (http response)', (done: Done) => {
let json = { let json = {
videoID: 'no-segments-video-id', videoID: 'no-segments-video-id',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'outro', 'outro',
'shilling', 'shilling',
@@ -48,7 +48,7 @@ describe('noSegmentRecords', () => {
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -75,7 +75,7 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories not in video (sql check)', (done: Done) => { it('Should be able to submit categories not in video (sql check)', (done: Done) => {
let json = { let json = {
videoID: 'no-segments-video-id-1', videoID: 'no-segments-video-id-1',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'outro', 'outro',
'shilling', 'shilling',
@@ -86,7 +86,7 @@ describe('noSegmentRecords', () => {
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -95,7 +95,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['no-segments-video-id-1']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']);
if (result.length !== 4) { if (result.length !== 4) {
console.log(result); console.log(result);
done("Expected 4 entrys in db, got " + result.length); done("Expected 4 entrys in db, got " + result.length);
@@ -114,13 +114,13 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories with _ in the category', (done: Done) => { it('Should be able to submit categories with _ in the category', (done: Done) => {
let json = { let json = {
videoID: 'underscore', videoID: 'underscore',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'word_word', 'word_word',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -129,7 +129,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['underscore']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['underscore']);
if (result.length !== 1) { if (result.length !== 1) {
console.log(result); console.log(result);
done("Expected 1 entrys in db, got " + result.length); done("Expected 1 entrys in db, got " + result.length);
@@ -148,13 +148,13 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories with upper and lower case in the category', (done: Done) => { it('Should be able to submit categories with upper and lower case in the category', (done: Done) => {
let json = { let json = {
videoID: 'bothCases', videoID: 'bothCases',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'wordWord', 'wordWord',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -163,7 +163,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['bothCases']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['bothCases']);
if (result.length !== 1) { if (result.length !== 1) {
console.log(result); console.log(result);
done("Expected 1 entrys in db, got " + result.length); done("Expected 1 entrys in db, got " + result.length);
@@ -182,13 +182,13 @@ describe('noSegmentRecords', () => {
it('Should not be able to submit categories with $ in the category', (done: Done) => { it('Should not be able to submit categories with $ in the category', (done: Done) => {
let json = { let json = {
videoID: 'specialChar', videoID: 'specialChar',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'word&word', 'word&word',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -197,7 +197,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['specialChar']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['specialChar']);
if (result.length !== 0) { if (result.length !== 0) {
console.log(result); console.log(result);
done("Expected 0 entrys in db, got " + result.length); done("Expected 0 entrys in db, got " + result.length);
@@ -214,7 +214,7 @@ describe('noSegmentRecords', () => {
}); });
it('Should return 400 for missing params', (done: Done) => { it('Should return 400 for missing params', (done: Done) => {
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -238,7 +238,7 @@ describe('noSegmentRecords', () => {
categories: [], categories: [],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -262,7 +262,7 @@ describe('noSegmentRecords', () => {
categories: ['sponsor'], categories: ['sponsor'],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -286,7 +286,7 @@ describe('noSegmentRecords', () => {
categories: ['sponsor'], categories: ['sponsor'],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -310,7 +310,7 @@ describe('noSegmentRecords', () => {
categories: {}, categories: {},
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -334,7 +334,7 @@ describe('noSegmentRecords', () => {
categories: 'sponsor', categories: 'sponsor',
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -360,7 +360,7 @@ describe('noSegmentRecords', () => {
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -377,16 +377,16 @@ describe('noSegmentRecords', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to delete a noSegment record', (done: Done) => { it('Should be able to delete a lockCategories record', (done: Done) => {
let json = { let json = {
videoID: 'delete-record', videoID: 'delete-record',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'sponsor', 'sponsor',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'DELETE', method: 'DELETE',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -395,7 +395,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['delete-record']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['delete-record']);
if (result.length === 0) { if (result.length === 0) {
done(); done();
} else { } else {
@@ -408,16 +408,16 @@ describe('noSegmentRecords', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to delete one noSegment record without removing another', (done: Done) => { it('Should be able to delete one lockCategories record without removing another', (done: Done) => {
let json = { let json = {
videoID: 'delete-record-1', videoID: 'delete-record-1',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'sponsor', 'sponsor',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'DELETE', method: 'DELETE',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -426,7 +426,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['delete-record-1']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['delete-record-1']);
if (result.length === 1) { if (result.length === 1) {
done(); done();
} else { } else {
@@ -445,7 +445,7 @@ describe('noSegmentRecords', () => {
* To test the submission code properly see ./test/cases/postSkipSegments.js * To test the submission code properly see ./test/cases/postSkipSegments.js
*/ */
it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done: Done) => { it('Should not be able to submit a segment to a video with a lock-category record (single submission)', (done: Done) => {
fetch(getbaseURL() + "/api/postVideoSponsorTimes", { fetch(getbaseURL() + "/api/postVideoSponsorTimes", {
method: 'POST', method: 'POST',
headers: { headers: {
@@ -453,7 +453,7 @@ describe('noSegmentRecords', () => {
}, },
body: JSON.stringify({ body: JSON.stringify({
userID: "testman42", userID: "testman42",
videoID: "noSubmitVideo", videoID: "lockCategoryVideo",
segments: [{ segments: [{
segment: [20, 40], segment: [20, 40],
category: "sponsor", category: "sponsor",
@@ -478,7 +478,7 @@ describe('noSegmentRecords', () => {
}, },
body: JSON.stringify({ body: JSON.stringify({
userID: "testman42", userID: "testman42",
videoID: "noSubmitVideo", videoID: "lockCategoryVideo",
segments: [{ segments: [{
segment: [20, 40], segment: [20, 40],
category: "sponsor", category: "sponsor",
@@ -507,7 +507,7 @@ describe('noSegmentRecords', () => {
}, },
body: JSON.stringify({ body: JSON.stringify({
userID: "testman42", userID: "testman42",
videoID: "noSubmitVideo", videoID: "lockCategoryVideo",
segments: [{ segments: [{
segment: [20, 40], segment: [20, 40],
category: "intro", category: "intro",

8
test/cases/postSkipSegments.ts
View File

@@ -195,8 +195,8 @@ describe('postSkipSegments', () => {
}); });
it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => { it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => {
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category")
VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')"); VALUES ('` + getHash("VIPUser-lockCategories") + "', 'noDuration', 'sponsor')");
try { try {
const res = await fetch(getbaseURL() const res = await fetch(getbaseURL()
@@ -217,13 +217,13 @@ describe('postSkipSegments', () => {
}); });
if (res.status === 200) { if (res.status === 200) {
const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]); const lockCategoriesRow = await db.prepare('get', `SELECT * from "lockCategories" WHERE videoID = ?`, ["noDuration"]);
const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]); FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]);
const videoRow = videoRows[0]; const videoRow = videoRows[0];
const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]); FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]);
if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10 if (lockCategoriesRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10
&& videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) { && videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) {
return; return;
} else { } else {

6
test/cases/unBan.ts
View File

@@ -13,7 +13,7 @@ describe('unBan', () => {
await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testEntity-unBan')`); await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testEntity-unBan')`);
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-unBan") + "')"); await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-unBan") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')");
let startOfInsertSegmentQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; let startOfInsertSegmentQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfInsertSegmentQuery + "('unBan-videoID-0', 1, 11, 2, 'unBan-uuid-0', 'testMan-unBan', 0, 50, 'sponsor', 1, '" + getHash('unBan-videoID-0', 1) + "')"); await db.prepare("run", startOfInsertSegmentQuery + "('unBan-videoID-0', 1, 11, 2, 'unBan-uuid-0', 'testMan-unBan', 0, 50, 'sponsor', 1, '" + getHash('unBan-videoID-0', 1) + "')");
@@ -47,7 +47,7 @@ describe('unBan', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to unban a user and re-enable shadow banned segments without noSegment entrys', (done) => { it('Should be able to unban a user and re-enable shadow banned segments without lockCategories entrys', (done) => {
fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testWoman-unBan&adminUserID=VIPUser-unBan&enabled=false", { fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testWoman-unBan&adminUserID=VIPUser-unBan&enabled=false", {
method: 'POST', method: 'POST',
headers: { headers: {
@@ -72,7 +72,7 @@ describe('unBan', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to unban a user and re-enable shadow banned segments with a mix of noSegment entrys', (done) => { it('Should be able to unban a user and re-enable shadow banned segments with a mix of lockCategories entrys', (done) => {
fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testEntity-unBan&adminUserID=VIPUser-unBan&enabled=false", { fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testEntity-unBan&adminUserID=VIPUser-unBan&enabled=false", {
method: 'POST', method: 'POST',
headers: { headers: {

42
test/cases/voteOnSponsorTime.ts
View File

@@ -59,7 +59,7 @@ describe('voteOnSponsorTime', () => {
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser") + "')"); await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser") + "')");
await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES ('` + getHash("randomID4") + "')"); await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES ('` + getHash("randomID4") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("videoID", "userID", "category") VALUES ('no-sponsor-segments-video', 'someUser', 'sponsor')`); await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES ('no-sponsor-segments-video', 'someUser', 'sponsor')`);
}); });
@@ -386,10 +386,25 @@ describe('voteOnSponsorTime', () => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1") + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1")
.then(async res => { .then(async res => {
if (res.status === 403) { let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
if (res.status === 403 && row.votes === -3) {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 403"); done("Status code was " + res.status + ", row is " + JSON.stringify(row));
}
})
.catch(err => done(err));
});
it('Non-VIP should not be able to downvote "dead" submission', (done: Done) => {
fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=0")
.then(async res => {
let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
if (res.status === 200 && row.votes === -3) {
done();
} else {
done("Status code was " + res.status + ", row is " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));
@@ -428,12 +443,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should not be able to downvote on a segment with no-segments category', (done: Done) => { it('Non-VIP should not be able to downvote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=0") + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=0")
.then(async res => { .then(async res => {
if (res.status === 403) { let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 403 && row.votes === 2) {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 403"); done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));
@@ -441,12 +457,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should be able to upvote on a segment with no-segments category', (done: Done) => { it('Non-VIP should be able to upvote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=1") + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=1")
.then(async res => { .then(async res => {
if (res.status === 200) { let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 200 && row.votes === 3) {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 200"); done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));
@@ -454,12 +471,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should not be able to category vote on a segment with no-segments category', (done: Done) => { it('Non-VIP should not be able to category vote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&category=outro") + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&category=outro")
.then(async res => { .then(async res => {
if (res.status === 403) { let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 403 && row.category === "sponsor") {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 403"); done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));