Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-07 20:17:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into export

Browse Source
This commit is contained in:
Ajay Ramachandran
2021-05-23 11:17:10 -04:00
parent f6f5570d0c e71399f5af
commit 590ed037dd
29 changed files with 5190 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.MD
View File

@@ -8,7 +8,7 @@ This is the server backend for it
This uses a Postgres or Sqlite database to hold all the timing data. This uses a Postgres or Sqlite database to hold all the timing data.
To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me. To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea. Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.

27
config.json.example
View File

@@ -38,5 +38,32 @@
"max": 20, // 20 requests in 15min time window "max": 20, // 20 requests in 15min time window
"statusCode": 200 "statusCode": 200
} }
},
"maxRewardTimePerSegmentInSeconds": 86400, // maximum time a user get rewarded in the leaderboard for a single segment
"dumpDatabase": {
"enabled": true,
"minTimeBetweenMs": 60000, // 1 minute between dumps
"appExportPath": "./docker/database-export",
"postgresExportPath": "/opt/exports",
"tables": [{
"name": "sponsorTimes",
"order": "timeSubmitted"
},
{
"name": "userNames"
},
{
"name": "categoryVotes"
},
{
"name": "lockCategories"
},
{
"name": "warnings",
"order": "issueTime"
},
{
"name": "vipUsers"
}]
} }
} }

4
databases/_sponsorTimes_indexes.sql
View File

@@ -51,10 +51,10 @@ CREATE INDEX IF NOT EXISTS "warnings_issueTime"
("issueTime" ASC NULLS LAST) ("issueTime" ASC NULLS LAST)
TABLESPACE pg_default; TABLESPACE pg_default;
-- noSegments -- lockCategories
CREATE INDEX IF NOT EXISTS "noSegments_videoID" CREATE INDEX IF NOT EXISTS "noSegments_videoID"
ON public."noSegments" USING btree ON public."lockCategories" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST) ("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default; TABLESPACE pg_default;

8
databases/_upgrade_sponsorTimes_11.sql Normal file
View File

@@ -0,0 +1,8 @@
BEGIN TRANSACTION;
/* Rename table: noSegments to lockCategories */
ALTER TABLE "noSegments" RENAME TO "lockCategories";
UPDATE "config" SET value = 11 WHERE key = 'version';
COMMIT;

10
docker/docker-compose.yml
View File

@@ -7,9 +7,10 @@ services:
- database.env - database.env
volumes: volumes:
- database-data:/var/lib/postgresql/data - database-data:/var/lib/postgresql/data
- ./database-export/:/opt/exports - ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
ports: ports:
- 127.0.0.1:5432:5432 - 5432:5432
restart: always
redis: redis:
container_name: redis container_name: redis
image: redis image: redis
@@ -17,7 +18,8 @@ services:
volumes: volumes:
- ./redis/redis.conf:/usr/local/etc/redis/redis.conf - ./redis/redis.conf:/usr/local/etc/redis/redis.conf
ports: ports:
- 127.0.0.1:32773:6379 - 32773:6379
restart: always
volumes: volumes:
database-data: database-data:

173
nginx/nginx.conf
View File

@@ -2,7 +2,7 @@ worker_processes 8;
worker_rlimit_nofile 8192; worker_rlimit_nofile 8192;
events { events {
worker_connections 32768; ## Default: 1024 worker_connections 132768; ## Default: 1024
} }
http { http {
@@ -12,28 +12,41 @@ http {
upstream backend_GET { upstream backend_GET {
least_conn; least_conn;
server localhost:4442; server localhost:4441;
server localhost:4443; server localhost:4442;
server localhost:4444; #server localhost:4443;
server localhost:4445; #server localhost:4444;
server localhost:4446; #server localhost:4445;
#server localhost:4446;
#server localhost:4447; #server localhost:4447;
#server localhost:4448; #server localhost:4448;
server 10.0.0.3:4441;
server 10.0.0.3:4442;
#server 134.209.69.251:80 backup;
server 116.203.32.253:80 backup;
#server 116.203.32.253:80;
} }
upstream backend_POST { upstream backend_POST {
#server localhost:4441;
#server localhost:4442;
server 10.0.0.3:4441;
#server 10.0.0.3:4442;
}
upstream backend_db {
server localhost:4441; server localhost:4441;
#server 10.0.0.3:4441;
} }
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m; proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
proxy_cache_key "$scheme$request_method$host$request_uri"; proxy_cache_key "$scheme$request_method$host$request_uri";
add_header X-Cache $upstream_cache_status; add_header X-Cache $upstream_cache_status;
server { server {
server_name sponsor.ajay.app api.sponsor.ajay.app; server_name sponsor.ajay.app api.sponsor.ajay.app;
access_log off;
error_log /dev/null;
error_page 404 /404.html; error_page 404 /404.html;
error_page 500 @myerrordirective_500; error_page 500 @myerrordirective_500;
error_page 502 @myerrordirective_502; error_page 502 @myerrordirective_502;
@@ -43,14 +56,16 @@ http {
# internal; # internal;
#} #}
#proxy_send_timeout 120s;
location @myerrordirective_500 { location @myerrordirective_500 {
return 502 "Internal Server Error"; return 400 "Internal Server Error";
} }
location @myerrordirective_502 { location @myerrordirective_502 {
return 502 "Bad Gateway"; return 400 "Bad Gateway";
} }
location @myerrordirective_504 { location @myerrordirective_504 {
return 502 "Gateway Timeout"; return 400 "Gateway Timeout";
} }
@@ -62,17 +77,16 @@ http {
return 301 https://sb.ltn.fi; return 301 https://sb.ltn.fi;
} }
location /invidious/ {
proxy_pass https://invidious.fdn.fr/;
}
location /test/ { location /test/ {
proxy_pass http://localhost:4440/; proxy_pass http://localhost:4440/;
#proxy_pass https://sbtest.etcinit.com/; #proxy_pass https://sbtest.etcinit.com/;
} }
location /api/skipSegments { location /api/skipSegments {
proxy_pass http://backend_$request_method; #return 200 "[]";
proxy_pass http://backend_$request_method;
#proxy_cache CACHEZONE;
#proxy_cache_valid 2m;
} }
location /api/getTopUsers { location /api/getTopUsers {
@@ -83,24 +97,49 @@ http {
location /api/getTotalStats { location /api/getTotalStats {
proxy_pass http://backend_GET; proxy_pass http://backend_GET;
} #return 200 "";
}
location /api/getVideoSponsorTimes { location /api/getVideoSponsorTimes {
proxy_pass http://backend_GET; proxy_pass http://backend_GET;
} }
location = /database.db { location /download/ {
alias /home/sbadmin/sponsor/databases/sponsorTimes.db; #alias /home/sbadmin/sponsor/docker/database-export/;
return 307 https://cdnsponsor.ajay.app$request_uri;
}
location /database {
proxy_pass http://backend_db;
#return 200 "Disabled for load reasons";
} }
location = /database.db {
#return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
#alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
alias /home/sbadmin/test-db/database.db;
}
#location = /database/sponsorTimes.csv {
# alias /home/sbadmin/sponsorTimes.csv;
#}
#location /api/voteOnSponsorTime {
# return 200 "Success";
#}
#location /api/viewedVideoSponsorTime {
# return 200 "Success";
#}
location /api { location /api {
proxy_pass http://backend_POST; proxy_pass http://backend_POST;
} }
location / { location / {
root /home/sbadmin/caddy/SponsorBlockSite/public-prod; root /home/sbadmin/SponsorBlockSite/public-prod;
### CORS ### CORS
if ($request_method = 'OPTIONS') { if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Origin' '*';
@@ -132,14 +171,78 @@ http {
} }
listen 443 ssl; # managed by Certbot listen 443 default_server ssl http2; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot #listen 443 http3 reuseport;
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot #ssl_protocols TLSv1.2 TLSv1.3;
#listen 80;
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
} }
server {
server_name cdnsponsor.ajay.app;
error_page 404 /404.html;
#location /database/ {
# alias /home/sbadmin/sponsor/docker/database-export/;
#}
location /download/ {
alias /home/sbadmin/sponsor/docker/database-export/;
}
location / {
root /home/sbadmin/SponsorBlockSite/public-prod;
### CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
#
# Custom headers and headers various browsers *should* be OK with but aren't
#
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
#
# Tell client that this pre-flight info is valid for 20 days
#
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain; charset=utf-8';
add_header 'Content-Length' 0;
return 204;
}
if ($request_method = 'POST') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
if ($request_method = 'GET') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
}
listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
@@ -148,6 +251,7 @@ http {
access_log off; access_log off;
error_log /dev/null; error_log /dev/null;
if ($host = api.sponsor.ajay.app) { if ($host = api.sponsor.ajay.app) {
return 301 https://$host$request_uri; return 301 https://$host$request_uri;
@@ -166,4 +270,17 @@ http {
}
server {
if ($host = cdnsponsor.ajay.app) {
return 301 https://$host$request_uri;
} # managed by Certbot
server_name cdnsponsor.ajay.app;
listen 80;
return 404; # managed by Certbot
}} }}

4652
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
package.json
View File

@@ -13,7 +13,7 @@
"author": "Ajay Ramachandran", "author": "Ajay Ramachandran",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"better-sqlite3": "^5.4.3", "better-sqlite3": "^7.1.5",
"dotenv": "^8.2.0", "dotenv": "^8.2.0",
"express": "^4.17.1", "express": "^4.17.1",
"express-rate-limit": "^5.1.3", "express-rate-limit": "^5.1.3",

17
src/app.ts
View File

@@ -5,8 +5,8 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
import {postSegmentShift} from './routes/postSegmentShift'; import {postSegmentShift} from './routes/postSegmentShift';
import {postWarning} from './routes/postWarning'; import {postWarning} from './routes/postWarning';
import {getIsUserVIP} from './routes/getIsUserVIP'; import {getIsUserVIP} from './routes/getIsUserVIP';
import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments'; import {deleteLockCategoriesEndpoint} from './routes/deleteLockCategories';
import {postNoSegments} from './routes/postNoSegments'; import {postLockCategories} from './routes/postLockCategories';
import {getUserInfo} from './routes/getUserInfo'; import {getUserInfo} from './routes/getUserInfo';
import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted'; import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
import {getTotalStats} from './routes/getTotalStats'; import {getTotalStats} from './routes/getTotalStats';
@@ -25,8 +25,9 @@ import {endpoint as getSkipSegments} from './routes/getSkipSegments';
import {userCounter} from './middleware/userCounter'; import {userCounter} from './middleware/userCounter';
import {loggerMiddleware} from './middleware/logger'; import {loggerMiddleware} from './middleware/logger';
import {corsMiddleware} from './middleware/cors'; import {corsMiddleware} from './middleware/cors';
import {apiCspMiddleware} from './middleware/apiCsp';
import {rateLimitMiddleware} from './middleware/requestRateLimit'; import {rateLimitMiddleware} from './middleware/requestRateLimit';
import dumpDatabase from './routes/dumpDatabase'; import dumpDatabase, {redirectLink} from './routes/dumpDatabase';
export function createServer(callback: () => void) { export function createServer(callback: () => void) {
@@ -36,6 +37,7 @@ export function createServer(callback: () => void) {
//setup CORS correctly //setup CORS correctly
app.use(corsMiddleware); app.use(corsMiddleware);
app.use(loggerMiddleware); app.use(loggerMiddleware);
app.use("/api/", apiCspMiddleware);
app.use(express.json()); app.use(express.json());
if (config.userCounterURL) app.use(userCounter); if (config.userCounterURL) app.use(userCounter);
@@ -114,10 +116,12 @@ function setupRoutes(app: Express) {
//send out a formatted time saved total //send out a formatted time saved total
app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted); app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
//submit video containing no segments //submit video to lock categories
app.post('/api/noSegments', postNoSegments); app.post('/api/noSegments', postLockCategories);
app.post('/api/lockCategories', postLockCategories);
app.delete('/api/noSegments', deleteNoSegmentsEndpoint); app.delete('/api/noSegments', deleteLockCategoriesEndpoint);
app.delete('/api/lockCategories', deleteLockCategoriesEndpoint);
//get if user is a vip //get if user is a vip
app.get('/api/isUserVIP', getIsUserVIP); app.get('/api/isUserVIP', getIsUserVIP);
@@ -131,6 +135,7 @@ function setupRoutes(app: Express) {
if (config.postgres) { if (config.postgres) {
app.get('/database', (req, res) => dumpDatabase(req, res, true)); app.get('/database', (req, res) => dumpDatabase(req, res, true));
app.get('/database.json', (req, res) => dumpDatabase(req, res, false)); app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
app.get('/database/*', redirectLink)
} else { } else {
app.get('/database.db', function (req: Request, res: Response) { app.get('/database.db', function (req: Request, res: Response) {
res.sendFile("./databases/sponsorTimes.db", {root: "./"}); res.sendFile("./databases/sponsorTimes.db", {root: "./"});

29
src/config.ts
View File

@@ -45,7 +45,34 @@ addDefaults(config, {
}, },
userCounterURL: null, userCounterURL: null,
youtubeAPIKey: null, youtubeAPIKey: null,
postgres: null maxRewardTimePerSegmentInSeconds: 86400,
postgres: null,
dumpDatabase: {
enabled: false,
minTimeBetweenMs: 60000,
appExportPath: './docker/database-export',
postgresExportPath: '/opt/exports',
tables: [{
name: "sponsorTimes",
order: "timeSubmitted"
},
{
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "lockCategories",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}]
}
}); });
// Add defaults // Add defaults

6
src/middleware/apiCsp.ts Normal file
View File

@@ -0,0 +1,6 @@
import {NextFunction, Request, Response} from 'express';
export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) {
res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
next();
}

12
src/routes/deleteNoSegments.ts → src/routes/deleteLockCategories.ts
View File

@@ -5,7 +5,7 @@ import {db} from '../databases/databases';
import { Category, VideoID } from '../types/segments.model'; import { Category, VideoID } from '../types/segments.model';
import { UserID } from '../types/user.model'; import { UserID } from '../types/user.model';
export async function deleteNoSegmentsEndpoint(req: Request, res: Response) { export async function deleteLockCategoriesEndpoint(req: Request, res: Response) {
// Collect user input data // Collect user input data
const videoID = req.body.videoID as VideoID; const videoID = req.body.videoID as VideoID;
const userID = req.body.userID as UserID; const userID = req.body.userID as UserID;
@@ -35,9 +35,9 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
return; return;
} }
deleteNoSegments(videoID, categories); deleteLockCategories(videoID, categories);
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID}); res.status(200).json({message: 'Removed lock categories entrys for video ' + videoID});
} }
/** /**
@@ -45,12 +45,12 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
* @param videoID * @param videoID
* @param categories If null, will remove all * @param categories If null, will remove all
*/ */
export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise<void> { export async function deleteLockCategories(videoID: VideoID, categories: Category[]): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => { const entries = (await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
return categories === null || categories.indexOf(entry.category) !== -1; return categories === null || categories.indexOf(entry.category) !== -1;
}); });
for (const entry of entries) { for (const entry of entries) {
await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]); await db.prepare('run', 'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
} }
} }

205
src/routes/dumpDatabase.ts
View File

@@ -2,51 +2,111 @@ import {db} from '../databases/databases';
import {Logger} from '../utils/logger'; import {Logger} from '../utils/logger';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
import { config } from '../config'; import { config } from '../config';
import util from 'util';
import fs from 'fs';
import path from 'path';
const unlink = util.promisify(fs.unlink);
const ONE_MINUTE = 1000 * 60; const ONE_MINUTE = 1000 * 60;
const styleHeader = `<style>body{font-family: sans-serif}</style>` const styleHeader = `<style>
body {
font-family: sans-serif
}
table th,
table td {
padding: 7px;
}
table th {
text-align: left;
}
table tbody tr:nth-child(odd) {
background: #efefef;
}
</style>`
const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p> const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p> <p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`; <p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
const tables = [{ const tables = config?.dumpDatabase?.tables ?? [];
name: "sponsorTimes", const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
order: "timeSubmitted" const appExportPath = config?.dumpDatabase?.appExportPath ?? './docker/database-export';
}, const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? '/opt/exports';
{ const tableNames = tables.map(table => table.name);
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "noSegments",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}];
const links: string[] = tables.map((table) => `/database/${table.name}.csv`); interface TableDumpList {
fileName: string;
tableName: string;
};
let latestDumpFiles: TableDumpList[] = [];
const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`) interface TableFile {
.reduce((acc, url) => acc + url, ""); file: string,
timestamp: number
};
if (tables.length === 0) {
Logger.warn('[dumpDatabase] No tables configured');
}
let lastUpdate = 0; let lastUpdate = 0;
let updateQueued = false;
let updateRunning = false;
export default function dumpDatabase(req: Request, res: Response, showPage: boolean) { function removeOutdatedDumps(exportPath: string): Promise<void> {
return new Promise((resolve, reject) => {
// Get list of table names
// Create array for each table
const tableFiles: Record<string, TableFile[]> = tableNames.reduce((obj: any, tableName) => {
obj[tableName] = [];
return obj;
}, {});
// read files in export directory
fs.readdir(exportPath, async (err: any, files: string[]) => {
if (err) Logger.error(err);
if (err) return resolve();
files.forEach(file => {
// we only care about files that start with "<tablename>_" and ends with .csv
tableNames.forEach(tableName => {
if (file.startsWith(`${tableName}`) && file.endsWith('.csv')) {
const filePath = path.join(exportPath, file);
tableFiles[tableName].push({
file: filePath,
timestamp: fs.statSync(filePath).mtime.getTime()
});
}
});
});
for (let tableName in tableFiles) {
const files = tableFiles[tableName].sort((a, b) => b.timestamp - a.timestamp);
for (let i = 2; i < files.length; i++) {
// remove old file
await unlink(files[i].file).catch((error: any) => {
Logger.error(`[dumpDatabase] Garbage collection failed ${error}`);
});
}
}
resolve();
});
});
}
export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) { if (!config.postgres) {
res.status(404).send("Not supported on this instance"); res.status(404).send("Not supported on this instance");
return; return;
} }
const now = Date.now(); updateQueueTime();
const updateQueued = now - lastUpdate > ONE_MINUTE;
res.status(200) res.status(200)
@@ -54,25 +114,100 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
res.send(`${styleHeader} res.send(`${styleHeader}
<h1>SponsorBlock database dumps</h1>${licenseHeader} <h1>SponsorBlock database dumps</h1>${licenseHeader}
<h3>How this works</h3> <h3>How this works</h3>
Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
Then, you can download the csv files below, or use the links returned from the JSON request. Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to get a list of urls and the update status database dump to run.
Then, you can download the csv files below, or use the links returned from the JSON request.
A dump will also be triggered by making a request to one of these urls.
<h3>Links</h3> <h3>Links</h3>
${linksHTML}<br/> <table>
<thead>
<tr>
<th>Table</th>
<th>CSV</th>
</tr>
</thead>
<tbody>
${latestDumpFiles.map((item:any) => {
return `
<tr>
<td>${item.tableName}</td>
<td><a href="/database/${item.tableName}.csv">${item.tableName}.csv</a></td>
</tr>
`;
}).join('')}
${latestDumpFiles.length === 0 ? '<tr><td colspan="2">Please wait: Generating files</td></tr>' : ''}
</tbody>
</table>
<hr/>
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`); ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
} else { } else {
res.send({ res.send({
lastUpdated: lastUpdate, lastUpdated: lastUpdate,
updateQueued, updateQueued,
links links: latestDumpFiles.map((item:any) => {
return {
table: item.tableName,
url: `/database/${item.tableName}.csv`,
size: item.fileSize,
};
}),
}) })
} }
if (updateQueued) { await queueDump();
lastUpdate = Date.now(); }
export async function redirectLink(req: Request, res: Response): Promise<void> {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) {
res.status(404).send("Not supported on this instance");
return;
}
const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path);
updateQueueTime();
if (file) {
res.redirect("/download/" + file.fileName);
} else {
res.status(404).send();
}
await queueDump();
}
function updateQueueTime(): void {
updateQueued ||= Date.now() - lastUpdate > MILLISECONDS_BETWEEN_DUMPS;
}
async function queueDump(): Promise<void> {
if (updateQueued && !updateRunning) {
const startTime = Date.now();
updateRunning = true;
await removeOutdatedDumps(appExportPath);
const dumpFiles = [];
for (const table of tables) { for (const table of tables) {
db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) const fileName = `${table.name}_${startTime}.csv`;
TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`); const file = `${postgresExportPath}/${fileName}`;
await db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '${file}' WITH (FORMAT CSV, HEADER true);`);
dumpFiles.push({
fileName,
tableName: table.name,
});
} }
latestDumpFiles = [...dumpFiles];
updateQueued = false;
updateRunning = false;
lastUpdate = startTime;
} }
} }

5
src/routes/getSavedTimeForUser.ts
View File

@@ -1,8 +1,11 @@
import {db} from '../databases/databases'; import {db} from '../databases/databases';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
import {getHash} from '../utils/getHash'; import {getHash} from '../utils/getHash';
import {config} from '../config';
import { Logger } from '../utils/logger'; import { Logger } from '../utils/logger';
const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
export async function getSavedTimeForUser(req: Request, res: Response) { export async function getSavedTimeForUser(req: Request, res: Response) {
let userID = req.query.userID as string; let userID = req.query.userID as string;
@@ -16,7 +19,7 @@ export async function getSavedTimeForUser(req: Request, res: Response) {
userID = getHash(userID); userID = getHash(userID);
try { try {
let row = await db.prepare("get", 'SELECT SUM(("endTime" - "startTime") / 60 * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]); let row = await db.prepare("get", 'SELECT SUM(((CASE WHEN "endTime" - "startTime" > ' + maxRewardTimePerSegmentInSeconds + ' THEN ' + maxRewardTimePerSegmentInSeconds + ' ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]);
if (row.minutesSaved != null) { if (row.minutesSaved != null) {
res.send({ res.send({

4
src/routes/getSkipSegments.ts
View File

@@ -268,6 +268,10 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
: req.query.category : req.query.category
? [req.query.category] ? [req.query.category]
: ['sponsor']; : ['sponsor'];
if (!Array.isArray(categories)) {
res.status(400).send("Categories parameter does not match format requirements.");
return false;
}
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube; let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) { if (!Object.values(Service).some((val) => val == service)) {

9
src/routes/getTopUsers.ts
View File

@@ -5,6 +5,7 @@ import {Request, Response} from 'express';
const MILLISECONDS_IN_MINUTE = 60000; const MILLISECONDS_IN_MINUTE = 60000;
const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE); const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = false) { async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = false) {
const userNames = []; const userNames = [];
@@ -24,14 +25,14 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
} }
const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount", const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
SUM(("sponsorTimes"."endTime" - "sponsorTimes"."startTime") / 60 * "sponsorTimes"."views") as "minutesSaved", SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ${maxRewardTimePerSegmentInSeconds} THEN ${maxRewardTimePerSegmentInSeconds} ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
SUM("votes") as "userVotes", ` + SUM("votes") as "userVotes", ` +
additionalFields + additionalFields +
`IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID" `IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID" LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID"
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL
GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20 GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
ORDER BY "` + sortBy + `" DESC LIMIT 100`, []); ORDER BY "${sortBy}" DESC LIMIT 100`, []);
for (let i = 0; i < rows.length; i++) { for (let i = 0; i < rows.length; i++) {
userNames[i] = rows[i].userName; userNames[i] = rows[i].userName;
@@ -70,6 +71,10 @@ export async function getTopUsers(req: Request, res: Response) {
return; return;
} }
//TODO: remove. This is broken for now
res.status(200).send();
return;
//setup which sort type to use //setup which sort type to use
let sortBy = ''; let sortBy = '';
if (sortType == 0) { if (sortType == 0) {

42
src/routes/getUserInfo.ts
View File

@@ -1,11 +1,15 @@
import {db} from '../databases/databases'; import {db} from '../databases/databases';
import {getHash} from '../utils/getHash'; import {getHash} from '../utils/getHash';
import {isUserVIP} from '../utils/isUserVIP';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
import {Logger} from '../utils/logger' import {Logger} from '../utils/logger';
import { HashedUserID, UserID } from '../types/user.model';
async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSaved: number, segmentCount: number }> { async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> {
try { try {
let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved", count(*) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved",
count(*) as "segmentCount" FROM "sponsorTimes"
WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
if (row.minutesSaved != null) { if (row.minutesSaved != null) {
return { return {
minutesSaved: row.minutesSaved, minutesSaved: row.minutesSaved,
@@ -22,7 +26,7 @@ async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSa
} }
} }
async function dbGetUsername(userID: string) { async function dbGetUsername(userID: HashedUserID) {
try { try {
let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]); let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
if (row !== undefined) { if (row !== undefined) {
@@ -36,24 +40,19 @@ async function dbGetUsername(userID: string) {
} }
} }
async function dbGetViewsForUser(userID: string) { async function dbGetViewsForUser(userID: HashedUserID) {
try { try {
let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]); let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
//increase the view count by one return row?.viewCount ?? 0;
if (row.viewCount != null) {
return row.viewCount;
} else {
return 0;
}
} catch (err) { } catch (err) {
return false; return false;
} }
} }
async function dbGetWarningsForUser(userID: string): Promise<number> { async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
try { try {
let rows = await db.prepare('all', `SELECT * FROM "warnings" WHERE "userID" = ?`, [userID]); let row = await db.prepare('get', `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]);
return rows.length; return row?.total ?? 0;
} catch (err) { } catch (err) {
Logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0'); Logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0');
return 0; return 0;
@@ -61,7 +60,7 @@ async function dbGetWarningsForUser(userID: string): Promise<number> {
} }
export async function getUserInfo(req: Request, res: Response) { export async function getUserInfo(req: Request, res: Response) {
let userID = req.query.userID as string; let userID = req.query.userID as UserID;
if (userID == undefined) { if (userID == undefined) {
//invalid request //invalid request
@@ -70,17 +69,18 @@ export async function getUserInfo(req: Request, res: Response) {
} }
//hash the userID //hash the userID
userID = getHash(userID); const hashedUserID: HashedUserID = getHash(userID);
const segmentsSummary = await dbGetSubmittedSegmentSummary(userID); const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID);
if (segmentsSummary) { if (segmentsSummary) {
res.send({ res.send({
userID, userID: hashedUserID,
userName: await dbGetUsername(userID), userName: await dbGetUsername(hashedUserID),
minutesSaved: segmentsSummary.minutesSaved, minutesSaved: segmentsSummary.minutesSaved,
segmentCount: segmentsSummary.segmentCount, segmentCount: segmentsSummary.segmentCount,
viewCount: await dbGetViewsForUser(userID), viewCount: await dbGetViewsForUser(hashedUserID),
warnings: await dbGetWarningsForUser(userID), warnings: await dbGetWarningsForUser(hashedUserID),
vip: await isUserVIP(hashedUserID),
}); });
} else { } else {
res.status(400).send(); res.status(400).send();

18
src/routes/postNoSegments.ts → src/routes/postLockCategories.ts
View File

@@ -4,7 +4,7 @@ import {isUserVIP} from '../utils/isUserVIP';
import {db} from '../databases/databases'; import {db} from '../databases/databases';
import {Request, Response} from 'express'; import {Request, Response} from 'express';
export async function postNoSegments(req: Request, res: Response) { export async function postLockCategories(req: Request, res: Response) {
// Collect user input data // Collect user input data
let videoID = req.body.videoID; let videoID = req.body.videoID;
let userID = req.body.userID; let userID = req.body.userID;
@@ -34,12 +34,12 @@ export async function postNoSegments(req: Request, res: Response) {
return; return;
} }
// Get existing no segment markers // Get existing lock categories markers
let noSegmentList = await db.prepare('all', 'SELECT "category" from "noSegments" where "videoID" = ?', [videoID]); let noCategoryList = await db.prepare('all', 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]);
if (!noSegmentList || noSegmentList.length === 0) { if (!noCategoryList || noCategoryList.length === 0) {
noSegmentList = []; noCategoryList = [];
} else { } else {
noSegmentList = noSegmentList.map((obj: any) => { noCategoryList = noCategoryList.map((obj: any) => {
return obj.category; return obj.category;
}); });
} }
@@ -48,7 +48,7 @@ export async function postNoSegments(req: Request, res: Response) {
let categoriesToMark = categories.filter((category) => { let categoriesToMark = categories.filter((category) => {
return !!category.match(/^[_a-zA-Z]+$/); return !!category.match(/^[_a-zA-Z]+$/);
}).filter((category) => { }).filter((category) => {
return noSegmentList.indexOf(category) === -1; return noCategoryList.indexOf(category) === -1;
}); });
// remove any duplicates // remove any duplicates
@@ -59,9 +59,9 @@ export async function postNoSegments(req: Request, res: Response) {
// create database entry // create database entry
for (const category of categoriesToMark) { for (const category of categoriesToMark) {
try { try {
await db.prepare('run', `INSERT INTO "noSegments" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]); await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]);
} catch (err) { } catch (err) {
Logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'"); Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
Logger.error(err); Logger.error(err);
res.status(500).json({ res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.", message: "Internal Server Error: Could not write marker to the database.",

14
src/routes/postSkipSegments.ts
View File

@@ -14,7 +14,7 @@ import {Request, Response} from 'express';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys'; import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import redis from '../utils/redis'; import redis from '../utils/redis';
import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model'; import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { deleteNoSegments } from './deleteNoSegments'; import { deleteLockCategories } from './deleteLockCategories';
import { getCategoryActionType } from '../utils/categoryInfo'; import { getCategoryActionType } from '../utils/categoryInfo';
interface APIVideoInfo { interface APIVideoInfo {
@@ -357,7 +357,7 @@ export async function postSkipSegments(req: Request, res: Response) {
return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?'); return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
} }
let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => { let lockedCategoryList = (await db.prepare('all', 'SELECT category from "lockCategories" where "videoID" = ?', [videoID])).map((list: any) => {
return list.category; return list.category;
}); });
@@ -389,9 +389,9 @@ export async function postSkipSegments(req: Request, res: Response) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]); await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
} }
// Reset no segments // Reset lock categories
noSegmentList = []; lockedCategoryList = [];
deleteNoSegments(videoID, null); deleteLockCategories(videoID, null);
} }
// Check if all submissions are correct // Check if all submissions are correct
@@ -407,8 +407,8 @@ export async function postSkipSegments(req: Request, res: Response) {
return; return;
} }
// Reject segemnt if it's in the no segments list // Reject segment if it's in the locked categories list
if (!isVIP && noSegmentList.indexOf(segments[i].category) !== -1) { if (!isVIP && lockedCategoryList.indexOf(segments[i].category) !== -1) {
// TODO: Do something about the fradulent submission // TODO: Do something about the fradulent submission
Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'"); Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
res.status(403).send( res.status(403).send(

10
src/routes/setUsername.ts
View File

@@ -21,6 +21,10 @@ export async function setUsername(req: Request, res: Response) {
res.sendStatus(200); res.sendStatus(200);
return; return;
} }
// remove unicode control characters from username (example: \n, \r, \t etc.)
// source: https://en.wikipedia.org/wiki/Control_character#In_Unicode
userName = userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, '');
if (adminUserIDInput != undefined) { if (adminUserIDInput != undefined) {
//this is the admin controlling the other users account, don't hash the controling account's ID //this is the admin controlling the other users account, don't hash the controling account's ID
@@ -35,6 +39,12 @@ export async function setUsername(req: Request, res: Response) {
//hash the userID //hash the userID
userID = getHash(userID); userID = getHash(userID);
} }
if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148"].includes(userID)) {
// Don't allow
res.sendStatus(200);
return;
}
try { try {
//check if username is already set //check if username is already set

6
src/routes/shadowBanUser.ts
View File

@@ -43,8 +43,8 @@ export async function shadowBanUser(req: Request, res: Response) {
//find all previous submissions and hide them //find all previous submissions and hide them
if (unHideOldSubmissions) { if (unHideOldSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ? await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ?
AND NOT EXISTS ( SELECT "videoID", "category" FROM "noSegments" WHERE AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
"sponsorTimes"."videoID" = "noSegments"."videoID" AND "sponsorTimes"."category" = "noSegments"."category")`, [userID]); "sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
} }
} else if (!enabled && row.userCount > 0) { } else if (!enabled && row.userCount > 0) {
//remove them from the shadow ban list //remove them from the shadow ban list
@@ -53,7 +53,7 @@ export async function shadowBanUser(req: Request, res: Response) {
//find all previous submissions and unhide them //find all previous submissions and unhide them
if (unHideOldSubmissions) { if (unHideOldSubmissions) {
let segmentsToIgnore = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st let segmentsToIgnore = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st
JOIN "noSegments" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?` JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?`
, [userID])).map((item: {UUID: string}) => item.UUID); , [userID])).map((item: {UUID: string}) => item.UUID);
let allSegments = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID])) let allSegments = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID]))
.map((item: {UUID: string}) => item.UUID); .map((item: {UUID: string}) => item.UUID);

92
src/routes/voteOnSponsorTime.ts
View File

@@ -180,51 +180,54 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
const timeSubmitted = Date.now(); const timeSubmitted = Date.now();
const voteAmount = isVIP ? 500 : 1; const voteAmount = isVIP ? 500 : 1;
const ableToVote = isVIP || finalResponse.finalStatus === 200 || true;
// Add the vote if (ableToVote) {
if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) { // Add the vote
// Update the already existing db entry if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]); // Update the already existing db entry
} else { await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
// Add a db entry } else {
await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]); // Add a db entry
} await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
}
// Add the info into the private db // Add the info into the private db
if (usersLastVoteInfo?.votes > 0) { if (usersLastVoteInfo?.votes > 0) {
// Reverse the previous vote // Reverse the previous vote
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]); await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);
await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]); await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
} else { } else {
await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]); await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
} }
// See if the submissions category is ready to change // See if the submissions category is ready to change
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]); const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
const startingVotes = isSubmissionVIP ? 10000 : 1; const startingVotes = isSubmissionVIP ? 10000 : 1;
// Change this value from 1 in the future to make it harder to change categories // Change this value from 1 in the future to make it harder to change categories
// Done this way without ORs incase the value is zero // Done this way without ORs incase the value is zero
const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes; const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
// Add submission as vote // Add submission as vote
if (!currentCategoryInfo && submissionInfo) { if (!currentCategoryInfo && submissionInfo) {
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]); await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]); await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
} }
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
//TODO: In the future, raise this number from zero to make it harder to change categories //TODO: In the future, raise this number from zero to make it harder to change categories
// VIPs change it every time // VIPs change it every time
if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) { if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
// Replace the category // Replace the category
await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]); await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
}
} }
clearRedisCache(videoInfo); clearRedisCache(videoInfo);
@@ -273,8 +276,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
// If not upvote // If not upvote
if (!isVIP && type !== 1) { if (!isVIP && type !== 1) {
const isSegmentLocked = async () => !!(await db.prepare('get', `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked; const isSegmentLocked = async () => !!(await db.prepare('get', `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked;
const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "noSegments".category from "noSegments" left join "sponsorTimes"' + const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "lockCategories".category from "lockCategories" left join "sponsorTimes"' +
' on ("noSegments"."videoID" = "sponsorTimes"."videoID" and "noSegments".category = "sponsorTimes".category)' + ' on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and "lockCategories".category = "sponsorTimes".category)' +
' where "UUID" = ?', [UUID])); ' where "UUID" = ?', [UUID]));
if (await isSegmentLocked() || await isVideoLocked()) { if (await isSegmentLocked() || await isVideoLocked()) {
@@ -287,13 +290,19 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res); return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res);
} }
if (type == 1 && !isVIP && !isOwnSubmission) { if (type !== undefined && !isVIP && !isOwnSubmission) {
// Check if upvoting hidden segment // Check if upvoting hidden segment
const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]); const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
if (voteInfo && voteInfo.votes <= -2) { if (voteInfo && voteInfo.votes <= -2) {
res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP."); if (type == 1) {
return; res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
return;
} else if (type == 0) {
// Already downvoted enough, ignore
res.status(200).send();
return;
}
} }
} }
@@ -378,7 +387,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
const ableToVote = isVIP const ableToVote = isVIP
|| ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined || ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined && (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined); && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
&& finalResponse.finalStatus === 200;
if (ableToVote) { if (ableToVote) {
//update the votes table //update the votes table

17
src/types/config.model.ts
View File

@@ -37,7 +37,9 @@ export interface SBSConfig {
minimumPrefix?: string; minimumPrefix?: string;
maximumPrefix?: string; maximumPrefix?: string;
redis?: redis.ClientOpts; redis?: redis.ClientOpts;
maxRewardTimePerSegmentInSeconds?: number;
postgres?: PoolConfig; postgres?: PoolConfig;
dumpDatabase?: DumpDatabase;
} }
export interface WebhookConfig { export interface WebhookConfig {
@@ -61,4 +63,17 @@ export interface PostgresConfig {
createDbIfNotExists: boolean; createDbIfNotExists: boolean;
enableWalCheckpointNumber: boolean; enableWalCheckpointNumber: boolean;
postgres: PoolConfig; postgres: PoolConfig;
} }
export interface DumpDatabase {
enabled: boolean;
minTimeBetweenMs: number;
appExportPath: string;
postgresExportPath: string;
tables: DumpDatabaseTable[];
}
export interface DumpDatabaseTable {
name: string;
order?: string;
}

4
src/utils/redis.ts
View File

@@ -29,6 +29,10 @@ if (config.redis) {
exportObject.getAsync = (key) => new Promise((resolve) => client.get(key, (err, reply) => resolve({err, reply}))); exportObject.getAsync = (key) => new Promise((resolve) => client.get(key, (err, reply) => resolve({err, reply})));
exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({err, reply}))); exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({err, reply})));
exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err))); exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
client.on("error", function(error) {
Logger.error(error);
});
} }
export default exportObject; export default exportObject;

4
test.json
View File

@@ -10,8 +10,8 @@
"discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook", "discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook",
"neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock", "neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock",
"behindProxy": true, "behindProxy": true,
"db": "./test/databases/sponsorTimes.db", "db": ":memory:",
"privateDB": "./test/databases/private.db", "privateDB": ":memory:",
"createDatabaseIfNotExist": true, "createDatabaseIfNotExist": true,
"schemaFolder": "./databases", "schemaFolder": "./databases",
"dbSchema": "./databases/_sponsorTimes.db.sql", "dbSchema": "./databases/_sponsorTimes.db.sql",

86
test/cases/noSegmentRecords.ts → test/cases/lockCategoriesRecords.ts
View File

@@ -4,21 +4,21 @@ import {getHash} from '../../src/utils/getHash';
import {db} from '../../src/databases/databases'; import {db} from '../../src/databases/databases';
describe('noSegmentRecords', () => { describe('lockCategoriesRecords', () => {
before(async () => { before(async () => {
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-noSegments") + "')"); await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-lockCategories") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id', 'intro')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id-1', 'intro')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'lockCategoryVideo', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'intro')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record-1', 'intro')");
}); });
it('Should update the database version when starting the application', async () => { it('Should update the database version when starting the application', async () => {
@@ -30,7 +30,7 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories not in video (http response)', (done: Done) => { it('Should be able to submit categories not in video (http response)', (done: Done) => {
let json = { let json = {
videoID: 'no-segments-video-id', videoID: 'no-segments-video-id',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'outro', 'outro',
'shilling', 'shilling',
@@ -48,7 +48,7 @@ describe('noSegmentRecords', () => {
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -75,7 +75,7 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories not in video (sql check)', (done: Done) => { it('Should be able to submit categories not in video (sql check)', (done: Done) => {
let json = { let json = {
videoID: 'no-segments-video-id-1', videoID: 'no-segments-video-id-1',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'outro', 'outro',
'shilling', 'shilling',
@@ -86,7 +86,7 @@ describe('noSegmentRecords', () => {
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -95,7 +95,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['no-segments-video-id-1']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']);
if (result.length !== 4) { if (result.length !== 4) {
console.log(result); console.log(result);
done("Expected 4 entrys in db, got " + result.length); done("Expected 4 entrys in db, got " + result.length);
@@ -114,13 +114,13 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories with _ in the category', (done: Done) => { it('Should be able to submit categories with _ in the category', (done: Done) => {
let json = { let json = {
videoID: 'underscore', videoID: 'underscore',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'word_word', 'word_word',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -129,7 +129,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['underscore']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['underscore']);
if (result.length !== 1) { if (result.length !== 1) {
console.log(result); console.log(result);
done("Expected 1 entrys in db, got " + result.length); done("Expected 1 entrys in db, got " + result.length);
@@ -148,13 +148,13 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories with upper and lower case in the category', (done: Done) => { it('Should be able to submit categories with upper and lower case in the category', (done: Done) => {
let json = { let json = {
videoID: 'bothCases', videoID: 'bothCases',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'wordWord', 'wordWord',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -163,7 +163,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['bothCases']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['bothCases']);
if (result.length !== 1) { if (result.length !== 1) {
console.log(result); console.log(result);
done("Expected 1 entrys in db, got " + result.length); done("Expected 1 entrys in db, got " + result.length);
@@ -182,13 +182,13 @@ describe('noSegmentRecords', () => {
it('Should not be able to submit categories with $ in the category', (done: Done) => { it('Should not be able to submit categories with $ in the category', (done: Done) => {
let json = { let json = {
videoID: 'specialChar', videoID: 'specialChar',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'word&word', 'word&word',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -197,7 +197,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['specialChar']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['specialChar']);
if (result.length !== 0) { if (result.length !== 0) {
console.log(result); console.log(result);
done("Expected 0 entrys in db, got " + result.length); done("Expected 0 entrys in db, got " + result.length);
@@ -214,7 +214,7 @@ describe('noSegmentRecords', () => {
}); });
it('Should return 400 for missing params', (done: Done) => { it('Should return 400 for missing params', (done: Done) => {
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -238,7 +238,7 @@ describe('noSegmentRecords', () => {
categories: [], categories: [],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -262,7 +262,7 @@ describe('noSegmentRecords', () => {
categories: ['sponsor'], categories: ['sponsor'],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -286,7 +286,7 @@ describe('noSegmentRecords', () => {
categories: ['sponsor'], categories: ['sponsor'],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -310,7 +310,7 @@ describe('noSegmentRecords', () => {
categories: {}, categories: {},
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -334,7 +334,7 @@ describe('noSegmentRecords', () => {
categories: 'sponsor', categories: 'sponsor',
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -360,7 +360,7 @@ describe('noSegmentRecords', () => {
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -377,16 +377,16 @@ describe('noSegmentRecords', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to delete a noSegment record', (done: Done) => { it('Should be able to delete a lockCategories record', (done: Done) => {
let json = { let json = {
videoID: 'delete-record', videoID: 'delete-record',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'sponsor', 'sponsor',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'DELETE', method: 'DELETE',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -395,7 +395,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['delete-record']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['delete-record']);
if (result.length === 0) { if (result.length === 0) {
done(); done();
} else { } else {
@@ -408,16 +408,16 @@ describe('noSegmentRecords', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to delete one noSegment record without removing another', (done: Done) => { it('Should be able to delete one lockCategories record without removing another', (done: Done) => {
let json = { let json = {
videoID: 'delete-record-1', videoID: 'delete-record-1',
userID: 'VIPUser-noSegments', userID: 'VIPUser-lockCategories',
categories: [ categories: [
'sponsor', 'sponsor',
], ],
}; };
fetch(getbaseURL() + "/api/noSegments", { fetch(getbaseURL() + "/api/lockCategories", {
method: 'DELETE', method: 'DELETE',
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@@ -426,7 +426,7 @@ describe('noSegmentRecords', () => {
}) })
.then(async res => { .then(async res => {
if (res.status === 200) { if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['delete-record-1']); let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['delete-record-1']);
if (result.length === 1) { if (result.length === 1) {
done(); done();
} else { } else {
@@ -445,7 +445,7 @@ describe('noSegmentRecords', () => {
* To test the submission code properly see ./test/cases/postSkipSegments.js * To test the submission code properly see ./test/cases/postSkipSegments.js
*/ */
it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done: Done) => { it('Should not be able to submit a segment to a video with a lock-category record (single submission)', (done: Done) => {
fetch(getbaseURL() + "/api/postVideoSponsorTimes", { fetch(getbaseURL() + "/api/postVideoSponsorTimes", {
method: 'POST', method: 'POST',
headers: { headers: {
@@ -453,7 +453,7 @@ describe('noSegmentRecords', () => {
}, },
body: JSON.stringify({ body: JSON.stringify({
userID: "testman42", userID: "testman42",
videoID: "noSubmitVideo", videoID: "lockCategoryVideo",
segments: [{ segments: [{
segment: [20, 40], segment: [20, 40],
category: "sponsor", category: "sponsor",
@@ -478,7 +478,7 @@ describe('noSegmentRecords', () => {
}, },
body: JSON.stringify({ body: JSON.stringify({
userID: "testman42", userID: "testman42",
videoID: "noSubmitVideo", videoID: "lockCategoryVideo",
segments: [{ segments: [{
segment: [20, 40], segment: [20, 40],
category: "sponsor", category: "sponsor",
@@ -507,7 +507,7 @@ describe('noSegmentRecords', () => {
}, },
body: JSON.stringify({ body: JSON.stringify({
userID: "testman42", userID: "testman42",
videoID: "noSubmitVideo", videoID: "lockCategoryVideo",
segments: [{ segments: [{
segment: [20, 40], segment: [20, 40],
category: "intro", category: "intro",

8
test/cases/postSkipSegments.ts
View File

@@ -195,8 +195,8 @@ describe('postSkipSegments', () => {
}); });
it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => { it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => {
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category")
VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')"); VALUES ('` + getHash("VIPUser-lockCategories") + "', 'noDuration', 'sponsor')");
try { try {
const res = await fetch(getbaseURL() const res = await fetch(getbaseURL()
@@ -217,13 +217,13 @@ describe('postSkipSegments', () => {
}); });
if (res.status === 200) { if (res.status === 200) {
const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]); const lockCategoriesRow = await db.prepare('get', `SELECT * from "lockCategories" WHERE videoID = ?`, ["noDuration"]);
const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]); FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]);
const videoRow = videoRows[0]; const videoRow = videoRows[0];
const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]); FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]);
if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10 if (lockCategoriesRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10
&& videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) { && videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) {
return; return;
} else { } else {

6
test/cases/unBan.ts
View File

@@ -13,7 +13,7 @@ describe('unBan', () => {
await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testEntity-unBan')`); await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testEntity-unBan')`);
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-unBan") + "')"); await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-unBan") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')"); await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')");
let startOfInsertSegmentQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES'; let startOfInsertSegmentQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfInsertSegmentQuery + "('unBan-videoID-0', 1, 11, 2, 'unBan-uuid-0', 'testMan-unBan', 0, 50, 'sponsor', 1, '" + getHash('unBan-videoID-0', 1) + "')"); await db.prepare("run", startOfInsertSegmentQuery + "('unBan-videoID-0', 1, 11, 2, 'unBan-uuid-0', 'testMan-unBan', 0, 50, 'sponsor', 1, '" + getHash('unBan-videoID-0', 1) + "')");
@@ -47,7 +47,7 @@ describe('unBan', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to unban a user and re-enable shadow banned segments without noSegment entrys', (done) => { it('Should be able to unban a user and re-enable shadow banned segments without lockCategories entrys', (done) => {
fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testWoman-unBan&adminUserID=VIPUser-unBan&enabled=false", { fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testWoman-unBan&adminUserID=VIPUser-unBan&enabled=false", {
method: 'POST', method: 'POST',
headers: { headers: {
@@ -72,7 +72,7 @@ describe('unBan', () => {
.catch(err => done(err)); .catch(err => done(err));
}); });
it('Should be able to unban a user and re-enable shadow banned segments with a mix of noSegment entrys', (done) => { it('Should be able to unban a user and re-enable shadow banned segments with a mix of lockCategories entrys', (done) => {
fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testEntity-unBan&adminUserID=VIPUser-unBan&enabled=false", { fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testEntity-unBan&adminUserID=VIPUser-unBan&enabled=false", {
method: 'POST', method: 'POST',
headers: { headers: {

42
test/cases/voteOnSponsorTime.ts
View File

@@ -59,7 +59,7 @@ describe('voteOnSponsorTime', () => {
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser") + "')"); await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser") + "')");
await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES ('` + getHash("randomID4") + "')"); await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES ('` + getHash("randomID4") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("videoID", "userID", "category") VALUES ('no-sponsor-segments-video', 'someUser', 'sponsor')`); await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES ('no-sponsor-segments-video', 'someUser', 'sponsor')`);
}); });
@@ -386,10 +386,25 @@ describe('voteOnSponsorTime', () => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1") + "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1")
.then(async res => { .then(async res => {
if (res.status === 403) { let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
if (res.status === 403 && row.votes === -3) {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 403"); done("Status code was " + res.status + ", row is " + JSON.stringify(row));
}
})
.catch(err => done(err));
});
it('Non-VIP should not be able to downvote "dead" submission', (done: Done) => {
fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=0")
.then(async res => {
let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
if (res.status === 200 && row.votes === -3) {
done();
} else {
done("Status code was " + res.status + ", row is " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));
@@ -428,12 +443,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should not be able to downvote on a segment with no-segments category', (done: Done) => { it('Non-VIP should not be able to downvote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=0") + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=0")
.then(async res => { .then(async res => {
if (res.status === 403) { let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 403 && row.votes === 2) {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 403"); done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));
@@ -441,12 +457,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should be able to upvote on a segment with no-segments category', (done: Done) => { it('Non-VIP should be able to upvote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=1") + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=1")
.then(async res => { .then(async res => {
if (res.status === 200) { let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 200 && row.votes === 3) {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 200"); done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));
@@ -454,12 +471,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should not be able to category vote on a segment with no-segments category', (done: Done) => { it('Non-VIP should not be able to category vote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL() fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&category=outro") + "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&category=outro")
.then(async res => { .then(async res => {
if (res.status === 403) { let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 403 && row.category === "sponsor") {
done(); done();
} else { } else {
done("Status code was " + res.status + " instead of 403"); done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
} }
}) })
.catch(err => done(err)); .catch(err => done(err));