Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-06 03:26:59 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into export

Browse Source
This commit is contained in:
Ajay Ramachandran
2021-05-23 11:17:10 -04:00
parent f6f5570d0c e71399f5af
commit 590ed037dd
29 changed files with 5190 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.MD
View File

@@ -8,7 +8,7 @@ This is the server backend for it
This uses a Postgres or Sqlite database to hold all the timing data.
To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database.db. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
To make sure that this project doesn't die, I have made the database publicly downloadable at https://sponsor.ajay.app/database. You can download a backup or get archive.org to take a backup if you do desire. The database is under [this license](https://creativecommons.org/licenses/by-nc-sa/4.0/) unless you get explicit permission from me.
Hopefully this project can be combined with projects like [this](https://github.com/Sponsoff/sponsorship_remover) and use this data to create a neural network to predict when sponsored segments happen. That project is sadly abandoned now, so I have decided to attempt to revive this idea.

27
config.json.example
View File

@@ -38,5 +38,32 @@
"max": 20, // 20 requests in 15min time window
"statusCode": 200
}
},
"maxRewardTimePerSegmentInSeconds": 86400, // maximum time a user get rewarded in the leaderboard for a single segment
"dumpDatabase": {
"enabled": true,
"minTimeBetweenMs": 60000, // 1 minute between dumps
"appExportPath": "./docker/database-export",
"postgresExportPath": "/opt/exports",
"tables": [{
"name": "sponsorTimes",
"order": "timeSubmitted"
},
{
"name": "userNames"
},
{
"name": "categoryVotes"
},
{
"name": "lockCategories"
},
{
"name": "warnings",
"order": "issueTime"
},
{
"name": "vipUsers"
}]
}
}

4
databases/_sponsorTimes_indexes.sql
View File

@@ -51,10 +51,10 @@ CREATE INDEX IF NOT EXISTS "warnings_issueTime"
("issueTime" ASC NULLS LAST)
TABLESPACE pg_default;
-- noSegments
-- lockCategories
CREATE INDEX IF NOT EXISTS "noSegments_videoID"
ON public."noSegments" USING btree
ON public."lockCategories" USING btree
("videoID" COLLATE pg_catalog."default" ASC NULLS LAST, category COLLATE pg_catalog."default" ASC NULLS LAST)
TABLESPACE pg_default;

8
databases/_upgrade_sponsorTimes_11.sql Normal file
View File

@@ -0,0 +1,8 @@
BEGIN TRANSACTION;
/* Rename table: noSegments to lockCategories */
ALTER TABLE "noSegments" RENAME TO "lockCategories";
UPDATE "config" SET value = 11 WHERE key = 'version';
COMMIT;

10
docker/docker-compose.yml
View File

@@ -7,9 +7,10 @@ services:
- database.env
volumes:
- database-data:/var/lib/postgresql/data
- ./database-export/:/opt/exports
- ./database-export/:/opt/exports # To make this work, run chmod 777 ./database-exports
ports:
- 127.0.0.1:5432:5432
- 5432:5432
restart: always
redis:
container_name: redis
image: redis
@@ -17,7 +18,8 @@ services:
volumes:
- ./redis/redis.conf:/usr/local/etc/redis/redis.conf
ports:
- 127.0.0.1:32773:6379
- 32773:6379
restart: always
volumes:
database-data:
database-data:

173
nginx/nginx.conf
View File

@@ -2,7 +2,7 @@ worker_processes 8;
worker_rlimit_nofile 8192;
events {
worker_connections 32768; ## Default: 1024
worker_connections 132768; ## Default: 1024
}
http {
@@ -12,28 +12,41 @@ http {
upstream backend_GET {
least_conn;
server localhost:4442;
server localhost:4443;
server localhost:4444;
server localhost:4445;
server localhost:4446;
server localhost:4441;
server localhost:4442;
#server localhost:4443;
#server localhost:4444;
#server localhost:4445;
#server localhost:4446;
#server localhost:4447;
#server localhost:4448;
server 10.0.0.3:4441;
server 10.0.0.3:4442;
#server 134.209.69.251:80 backup;
server 116.203.32.253:80 backup;
#server 116.203.32.253:80;
}
upstream backend_POST {
#server localhost:4441;
#server localhost:4442;
server 10.0.0.3:4441;
#server 10.0.0.3:4442;
}
upstream backend_db {
server localhost:4441;
#server 10.0.0.3:4441;
}
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=40m;
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHEZONE:10m inactive=60m max_size=400m;
proxy_cache_key "$scheme$request_method$host$request_uri";
add_header X-Cache $upstream_cache_status;
server {
server_name sponsor.ajay.app api.sponsor.ajay.app;
access_log off;
error_log /dev/null;
error_page 404 /404.html;
error_page 500 @myerrordirective_500;
error_page 502 @myerrordirective_502;
@@ -43,14 +56,16 @@ http {
# internal;
#}
#proxy_send_timeout 120s;
location @myerrordirective_500 {
return 502 "Internal Server Error";
return 400 "Internal Server Error";
}
location @myerrordirective_502 {
return 502 "Bad Gateway";
return 400 "Bad Gateway";
}
location @myerrordirective_504 {
return 502 "Gateway Timeout";
return 400 "Gateway Timeout";
}
@@ -62,17 +77,16 @@ http {
return 301 https://sb.ltn.fi;
}
location /invidious/ {
proxy_pass https://invidious.fdn.fr/;
}
location /test/ {
proxy_pass http://localhost:4440/;
#proxy_pass https://sbtest.etcinit.com/;
}
location /api/skipSegments {
proxy_pass http://backend_$request_method;
#return 200 "[]";
proxy_pass http://backend_$request_method;
#proxy_cache CACHEZONE;
#proxy_cache_valid 2m;
}
location /api/getTopUsers {
@@ -83,24 +97,49 @@ http {
location /api/getTotalStats {
proxy_pass http://backend_GET;
}
#return 200 "";
}
location /api/getVideoSponsorTimes {
proxy_pass http://backend_GET;
}
location = /database.db {
alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
location /download/ {
#alias /home/sbadmin/sponsor/docker/database-export/;
return 307 https://cdnsponsor.ajay.app$request_uri;
}
location /database {
proxy_pass http://backend_db;
#return 200 "Disabled for load reasons";
}
location = /database.db {
#return 404 "Sqlite database has been replaced with csv exports at https://sponsor.ajay.app/database. Sqlite exports might come back soon, but exported at longer intervals.";
#alias /home/sbadmin/sponsor/databases/sponsorTimes.db;
alias /home/sbadmin/test-db/database.db;
}
#location = /database/sponsorTimes.csv {
# alias /home/sbadmin/sponsorTimes.csv;
#}
#location /api/voteOnSponsorTime {
# return 200 "Success";
#}
#location /api/viewedVideoSponsorTime {
# return 200 "Success";
#}
location /api {
proxy_pass http://backend_POST;
}
location / {
root /home/sbadmin/caddy/SponsorBlockSite/public-prod;
root /home/sbadmin/SponsorBlockSite/public-prod;
### CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*';
@@ -132,14 +171,78 @@ http {
}
listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app/privkey.pem; # managed by Certbot
listen 443 default_server ssl http2; # managed by Certbot
#listen 443 http3 reuseport;
#ssl_protocols TLSv1.2 TLSv1.3;
#listen 80;
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
server {
server_name cdnsponsor.ajay.app;
error_page 404 /404.html;
#location /database/ {
# alias /home/sbadmin/sponsor/docker/database-export/;
#}
location /download/ {
alias /home/sbadmin/sponsor/docker/database-export/;
}
location / {
root /home/sbadmin/SponsorBlockSite/public-prod;
### CORS
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
#
# Custom headers and headers various browsers *should* be OK with but aren't
#
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
#
# Tell client that this pre-flight info is valid for 20 days
#
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Type' 'text/plain; charset=utf-8';
add_header 'Content-Length' 0;
return 204;
}
if ($request_method = 'POST') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
if ($request_method = 'GET') {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
}
}
listen 443 ssl; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/sponsor.ajay.app-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/sponsor.ajay.app-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
@@ -148,6 +251,7 @@ http {
access_log off;
error_log /dev/null;
if ($host = api.sponsor.ajay.app) {
return 301 https://$host$request_uri;
@@ -166,4 +270,17 @@ http {
}
server {
if ($host = cdnsponsor.ajay.app) {
return 301 https://$host$request_uri;
} # managed by Certbot
server_name cdnsponsor.ajay.app;
listen 80;
return 404; # managed by Certbot
}}

4652
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

2
package.json
View File

@@ -13,7 +13,7 @@
"author": "Ajay Ramachandran",
"license": "MIT",
"dependencies": {
"better-sqlite3": "^5.4.3",
"better-sqlite3": "^7.1.5",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"express-rate-limit": "^5.1.3",

17
src/app.ts
View File

@@ -5,8 +5,8 @@ import {oldGetVideoSponsorTimes} from './routes/oldGetVideoSponsorTimes';
import {postSegmentShift} from './routes/postSegmentShift';
import {postWarning} from './routes/postWarning';
import {getIsUserVIP} from './routes/getIsUserVIP';
import {deleteNoSegmentsEndpoint} from './routes/deleteNoSegments';
import {postNoSegments} from './routes/postNoSegments';
import {deleteLockCategoriesEndpoint} from './routes/deleteLockCategories';
import {postLockCategories} from './routes/postLockCategories';
import {getUserInfo} from './routes/getUserInfo';
import {getDaysSavedFormatted} from './routes/getDaysSavedFormatted';
import {getTotalStats} from './routes/getTotalStats';
@@ -25,8 +25,9 @@ import {endpoint as getSkipSegments} from './routes/getSkipSegments';
import {userCounter} from './middleware/userCounter';
import {loggerMiddleware} from './middleware/logger';
import {corsMiddleware} from './middleware/cors';
import {apiCspMiddleware} from './middleware/apiCsp';
import {rateLimitMiddleware} from './middleware/requestRateLimit';
import dumpDatabase from './routes/dumpDatabase';
import dumpDatabase, {redirectLink} from './routes/dumpDatabase';
export function createServer(callback: () => void) {
@@ -36,6 +37,7 @@ export function createServer(callback: () => void) {
//setup CORS correctly
app.use(corsMiddleware);
app.use(loggerMiddleware);
app.use("/api/", apiCspMiddleware);
app.use(express.json());
if (config.userCounterURL) app.use(userCounter);
@@ -114,10 +116,12 @@ function setupRoutes(app: Express) {
//send out a formatted time saved total
app.get('/api/getDaysSavedFormatted', getDaysSavedFormatted);
//submit video containing no segments
app.post('/api/noSegments', postNoSegments);
//submit video to lock categories
app.post('/api/noSegments', postLockCategories);
app.post('/api/lockCategories', postLockCategories);
app.delete('/api/noSegments', deleteNoSegmentsEndpoint);
app.delete('/api/noSegments', deleteLockCategoriesEndpoint);
app.delete('/api/lockCategories', deleteLockCategoriesEndpoint);
//get if user is a vip
app.get('/api/isUserVIP', getIsUserVIP);
@@ -131,6 +135,7 @@ function setupRoutes(app: Express) {
if (config.postgres) {
app.get('/database', (req, res) => dumpDatabase(req, res, true));
app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
app.get('/database/*', redirectLink)
} else {
app.get('/database.db', function (req: Request, res: Response) {
res.sendFile("./databases/sponsorTimes.db", {root: "./"});

29
src/config.ts
View File

@@ -45,7 +45,34 @@ addDefaults(config, {
},
userCounterURL: null,
youtubeAPIKey: null,
postgres: null
maxRewardTimePerSegmentInSeconds: 86400,
postgres: null,
dumpDatabase: {
enabled: false,
minTimeBetweenMs: 60000,
appExportPath: './docker/database-export',
postgresExportPath: '/opt/exports',
tables: [{
name: "sponsorTimes",
order: "timeSubmitted"
},
{
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "lockCategories",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}]
}
});
// Add defaults

6
src/middleware/apiCsp.ts Normal file
View File

@@ -0,0 +1,6 @@
import {NextFunction, Request, Response} from 'express';
export function apiCspMiddleware(req: Request, res: Response, next: NextFunction) {
res.header("Content-Security-Policy", "script-src 'none'; object-src 'none'");
next();
}

12
src/routes/deleteNoSegments.ts → src/routes/deleteLockCategories.ts
View File

@@ -5,7 +5,7 @@ import {db} from '../databases/databases';
import { Category, VideoID } from '../types/segments.model';
import { UserID } from '../types/user.model';
export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
export async function deleteLockCategoriesEndpoint(req: Request, res: Response) {
// Collect user input data
const videoID = req.body.videoID as VideoID;
const userID = req.body.userID as UserID;
@@ -35,9 +35,9 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
return;
}
deleteNoSegments(videoID, categories);
deleteLockCategories(videoID, categories);
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
res.status(200).json({message: 'Removed lock categories entrys for video ' + videoID});
}
/**
@@ -45,12 +45,12 @@ export async function deleteNoSegmentsEndpoint(req: Request, res: Response) {
* @param videoID
* @param categories If null, will remove all
*/
export async function deleteNoSegments(videoID: VideoID, categories: Category[]): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "noSegments" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
export async function deleteLockCategories(videoID: VideoID, categories: Category[]): Promise<void> {
const entries = (await db.prepare("all", 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', [videoID])).filter((entry: any) => {
return categories === null || categories.indexOf(entry.category) !== -1;
});
for (const entry of entries) {
await db.prepare('run', 'DELETE FROM "noSegments" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
await db.prepare('run', 'DELETE FROM "lockCategories" WHERE "videoID" = ? AND "category" = ?', [videoID, entry.category]);
}
}

205
src/routes/dumpDatabase.ts
View File

@@ -2,51 +2,111 @@ import {db} from '../databases/databases';
import {Logger} from '../utils/logger';
import {Request, Response} from 'express';
import { config } from '../config';
import util from 'util';
import fs from 'fs';
import path from 'path';
const unlink = util.promisify(fs.unlink);
const ONE_MINUTE = 1000 * 60;
const styleHeader = `<style>body{font-family: sans-serif}</style>`
const styleHeader = `<style>
body {
font-family: sans-serif
}
table th,
table td {
padding: 7px;
}
table th {
text-align: left;
}
table tbody tr:nth-child(odd) {
background: #efefef;
}
</style>`
const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
const tables = [{
name: "sponsorTimes",
order: "timeSubmitted"
},
{
name: "userNames"
},
{
name: "categoryVotes"
},
{
name: "noSegments",
},
{
name: "warnings",
order: "issueTime"
},
{
name: "vipUsers"
}];
const tables = config?.dumpDatabase?.tables ?? [];
const MILLISECONDS_BETWEEN_DUMPS = config?.dumpDatabase?.minTimeBetweenMs ?? ONE_MINUTE;
const appExportPath = config?.dumpDatabase?.appExportPath ?? './docker/database-export';
const postgresExportPath = config?.dumpDatabase?.postgresExportPath ?? '/opt/exports';
const tableNames = tables.map(table => table.name);
const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
interface TableDumpList {
fileName: string;
tableName: string;
};
let latestDumpFiles: TableDumpList[] = [];
const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
.reduce((acc, url) => acc + url, "");
interface TableFile {
file: string,
timestamp: number
};
if (tables.length === 0) {
Logger.warn('[dumpDatabase] No tables configured');
}
let lastUpdate = 0;
let updateQueued = false;
let updateRunning = false;
export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
function removeOutdatedDumps(exportPath: string): Promise<void> {
return new Promise((resolve, reject) => {
// Get list of table names
// Create array for each table
const tableFiles: Record<string, TableFile[]> = tableNames.reduce((obj: any, tableName) => {
obj[tableName] = [];
return obj;
}, {});
// read files in export directory
fs.readdir(exportPath, async (err: any, files: string[]) => {
if (err) Logger.error(err);
if (err) return resolve();
files.forEach(file => {
// we only care about files that start with "<tablename>_" and ends with .csv
tableNames.forEach(tableName => {
if (file.startsWith(`${tableName}`) && file.endsWith('.csv')) {
const filePath = path.join(exportPath, file);
tableFiles[tableName].push({
file: filePath,
timestamp: fs.statSync(filePath).mtime.getTime()
});
}
});
});
for (let tableName in tableFiles) {
const files = tableFiles[tableName].sort((a, b) => b.timestamp - a.timestamp);
for (let i = 2; i < files.length; i++) {
// remove old file
await unlink(files[i].file).catch((error: any) => {
Logger.error(`[dumpDatabase] Garbage collection failed ${error}`);
});
}
}
resolve();
});
});
}
export default async function dumpDatabase(req: Request, res: Response, showPage: boolean) {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) {
res.status(404).send("Not supported on this instance");
return;
}
const now = Date.now();
const updateQueued = now - lastUpdate > ONE_MINUTE;
updateQueueTime();
res.status(200)
@@ -54,25 +114,100 @@ export default function dumpDatabase(req: Request, res: Response, showPage: bool
res.send(`${styleHeader}
<h1>SponsorBlock database dumps</h1>${licenseHeader}
<h3>How this works</h3>
Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
Then, you can download the csv files below, or use the links returned from the JSON request.
Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to get a list of urls and the update status database dump to run.
Then, you can download the csv files below, or use the links returned from the JSON request.
A dump will also be triggered by making a request to one of these urls.
<h3>Links</h3>
${linksHTML}<br/>
<table>
<thead>
<tr>
<th>Table</th>
<th>CSV</th>
</tr>
</thead>
<tbody>
${latestDumpFiles.map((item:any) => {
return `
<tr>
<td>${item.tableName}</td>
<td><a href="/database/${item.tableName}.csv">${item.tableName}.csv</a></td>
</tr>
`;
}).join('')}
${latestDumpFiles.length === 0 ? '<tr><td colspan="2">Please wait: Generating files</td></tr>' : ''}
</tbody>
</table>
<hr/>
${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
} else {
res.send({
lastUpdated: lastUpdate,
updateQueued,
links
links: latestDumpFiles.map((item:any) => {
return {
table: item.tableName,
url: `/database/${item.tableName}.csv`,
size: item.fileSize,
};
}),
})
}
if (updateQueued) {
lastUpdate = Date.now();
await queueDump();
}
export async function redirectLink(req: Request, res: Response): Promise<void> {
if (!config?.dumpDatabase?.enabled) {
res.status(404).send("Database dump is disabled");
return;
}
if (!config.postgres) {
res.status(404).send("Not supported on this instance");
return;
}
const file = latestDumpFiles.find((value) => `/database/${value.tableName}.csv` === req.path);
updateQueueTime();
if (file) {
res.redirect("/download/" + file.fileName);
} else {
res.status(404).send();
}
await queueDump();
}
function updateQueueTime(): void {
updateQueued ||= Date.now() - lastUpdate > MILLISECONDS_BETWEEN_DUMPS;
}
async function queueDump(): Promise<void> {
if (updateQueued && !updateRunning) {
const startTime = Date.now();
updateRunning = true;
await removeOutdatedDumps(appExportPath);
const dumpFiles = [];
for (const table of tables) {
db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
const fileName = `${table.name}_${startTime}.csv`;
const file = `${postgresExportPath}/${fileName}`;
await db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``})
TO '${file}' WITH (FORMAT CSV, HEADER true);`);
dumpFiles.push({
fileName,
tableName: table.name,
});
}
latestDumpFiles = [...dumpFiles];
updateQueued = false;
updateRunning = false;
lastUpdate = startTime;
}
}

5
src/routes/getSavedTimeForUser.ts
View File

@@ -1,8 +1,11 @@
import {db} from '../databases/databases';
import {Request, Response} from 'express';
import {getHash} from '../utils/getHash';
import {config} from '../config';
import { Logger } from '../utils/logger';
const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
export async function getSavedTimeForUser(req: Request, res: Response) {
let userID = req.query.userID as string;
@@ -16,7 +19,7 @@ export async function getSavedTimeForUser(req: Request, res: Response) {
userID = getHash(userID);
try {
let row = await db.prepare("get", 'SELECT SUM(("endTime" - "startTime") / 60 * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]);
let row = await db.prepare("get", 'SELECT SUM(((CASE WHEN "endTime" - "startTime" > ' + maxRewardTimePerSegmentInSeconds + ' THEN ' + maxRewardTimePerSegmentInSeconds + ' ELSE "endTime" - "startTime" END) / 60) * "views") as "minutesSaved" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -1 AND "shadowHidden" != 1 ', [userID]);
if (row.minutesSaved != null) {
res.send({

4
src/routes/getSkipSegments.ts
View File

@@ -268,6 +268,10 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
: req.query.category
? [req.query.category]
: ['sponsor'];
if (!Array.isArray(categories)) {
res.status(400).send("Categories parameter does not match format requirements.");
return false;
}
let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
if (!Object.values(Service).some((val) => val == service)) {

9
src/routes/getTopUsers.ts
View File

@@ -5,6 +5,7 @@ import {Request, Response} from 'express';
const MILLISECONDS_IN_MINUTE = 60000;
const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
const maxRewardTimePerSegmentInSeconds = config.maxRewardTimePerSegmentInSeconds ?? 86400;
async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boolean = false) {
const userNames = [];
@@ -24,14 +25,14 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled: boole
}
const rows = await db.prepare('all', `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
SUM(("sponsorTimes"."endTime" - "sponsorTimes"."startTime") / 60 * "sponsorTimes"."views") as "minutesSaved",
SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ${maxRewardTimePerSegmentInSeconds} THEN ${maxRewardTimePerSegmentInSeconds} ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
SUM("votes") as "userVotes", ` +
additionalFields +
`IFNULL("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
LEFT JOIN "privateDB"."shadowBannedUsers" ON "sponsorTimes"."userID"="privateDB"."shadowBannedUsers"."userID"
WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "privateDB"."shadowBannedUsers"."userID" IS NULL
GROUP BY IFNULL("userName", "sponsorTimes"."userID") HAVING "userVotes" > 20
ORDER BY "` + sortBy + `" DESC LIMIT 100`, []);
ORDER BY "${sortBy}" DESC LIMIT 100`, []);
for (let i = 0; i < rows.length; i++) {
userNames[i] = rows[i].userName;
@@ -70,6 +71,10 @@ export async function getTopUsers(req: Request, res: Response) {
return;
}
//TODO: remove. This is broken for now
res.status(200).send();
return;
//setup which sort type to use
let sortBy = '';
if (sortType == 0) {

42
src/routes/getUserInfo.ts
View File

@@ -1,11 +1,15 @@
import {db} from '../databases/databases';
import {getHash} from '../utils/getHash';
import {isUserVIP} from '../utils/isUserVIP';
import {Request, Response} from 'express';
import {Logger} from '../utils/logger'
import {Logger} from '../utils/logger';
import { HashedUserID, UserID } from '../types/user.model';
async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSaved: number, segmentCount: number }> {
async function dbGetSubmittedSegmentSummary(userID: HashedUserID): Promise<{ minutesSaved: number, segmentCount: number }> {
try {
let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved", count(*) as "segmentCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
let row = await db.prepare("get", `SELECT SUM((("endTime" - "startTime") / 60) * "views") as "minutesSaved",
count(*) as "segmentCount" FROM "sponsorTimes"
WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
if (row.minutesSaved != null) {
return {
minutesSaved: row.minutesSaved,
@@ -22,7 +26,7 @@ async function dbGetSubmittedSegmentSummary(userID: string): Promise<{ minutesSa
}
}
async function dbGetUsername(userID: string) {
async function dbGetUsername(userID: HashedUserID) {
try {
let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
if (row !== undefined) {
@@ -36,24 +40,19 @@ async function dbGetUsername(userID: string) {
}
}
async function dbGetViewsForUser(userID: string) {
async function dbGetViewsForUser(userID: HashedUserID) {
try {
let row = await db.prepare('get', `SELECT SUM("views") as "viewCount" FROM "sponsorTimes" WHERE "userID" = ? AND "votes" > -2 AND "shadowHidden" != 1`, [userID]);
//increase the view count by one
if (row.viewCount != null) {
return row.viewCount;
} else {
return 0;
}
return row?.viewCount ?? 0;
} catch (err) {
return false;
}
}
async function dbGetWarningsForUser(userID: string): Promise<number> {
async function dbGetWarningsForUser(userID: HashedUserID): Promise<number> {
try {
let rows = await db.prepare('all', `SELECT * FROM "warnings" WHERE "userID" = ?`, [userID]);
return rows.length;
let row = await db.prepare('get', `SELECT COUNT(*) as total FROM "warnings" WHERE "userID" = ? AND "enabled" = 1`, [userID]);
return row?.total ?? 0;
} catch (err) {
Logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0');
return 0;
@@ -61,7 +60,7 @@ async function dbGetWarningsForUser(userID: string): Promise<number> {
}
export async function getUserInfo(req: Request, res: Response) {
let userID = req.query.userID as string;
let userID = req.query.userID as UserID;
if (userID == undefined) {
//invalid request
@@ -70,17 +69,18 @@ export async function getUserInfo(req: Request, res: Response) {
}
//hash the userID
userID = getHash(userID);
const hashedUserID: HashedUserID = getHash(userID);
const segmentsSummary = await dbGetSubmittedSegmentSummary(userID);
const segmentsSummary = await dbGetSubmittedSegmentSummary(hashedUserID);
if (segmentsSummary) {
res.send({
userID,
userName: await dbGetUsername(userID),
userID: hashedUserID,
userName: await dbGetUsername(hashedUserID),
minutesSaved: segmentsSummary.minutesSaved,
segmentCount: segmentsSummary.segmentCount,
viewCount: await dbGetViewsForUser(userID),
warnings: await dbGetWarningsForUser(userID),
viewCount: await dbGetViewsForUser(hashedUserID),
warnings: await dbGetWarningsForUser(hashedUserID),
vip: await isUserVIP(hashedUserID),
});
} else {
res.status(400).send();

18
src/routes/postNoSegments.ts → src/routes/postLockCategories.ts
View File

@@ -4,7 +4,7 @@ import {isUserVIP} from '../utils/isUserVIP';
import {db} from '../databases/databases';
import {Request, Response} from 'express';
export async function postNoSegments(req: Request, res: Response) {
export async function postLockCategories(req: Request, res: Response) {
// Collect user input data
let videoID = req.body.videoID;
let userID = req.body.userID;
@@ -34,12 +34,12 @@ export async function postNoSegments(req: Request, res: Response) {
return;
}
// Get existing no segment markers
let noSegmentList = await db.prepare('all', 'SELECT "category" from "noSegments" where "videoID" = ?', [videoID]);
if (!noSegmentList || noSegmentList.length === 0) {
noSegmentList = [];
// Get existing lock categories markers
let noCategoryList = await db.prepare('all', 'SELECT "category" from "lockCategories" where "videoID" = ?', [videoID]);
if (!noCategoryList || noCategoryList.length === 0) {
noCategoryList = [];
} else {
noSegmentList = noSegmentList.map((obj: any) => {
noCategoryList = noCategoryList.map((obj: any) => {
return obj.category;
});
}
@@ -48,7 +48,7 @@ export async function postNoSegments(req: Request, res: Response) {
let categoriesToMark = categories.filter((category) => {
return !!category.match(/^[_a-zA-Z]+$/);
}).filter((category) => {
return noSegmentList.indexOf(category) === -1;
return noCategoryList.indexOf(category) === -1;
});
// remove any duplicates
@@ -59,9 +59,9 @@ export async function postNoSegments(req: Request, res: Response) {
// create database entry
for (const category of categoriesToMark) {
try {
await db.prepare('run', `INSERT INTO "noSegments" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]);
await db.prepare('run', `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES(?, ?, ?)`, [videoID, userID, category]);
} catch (err) {
Logger.error("Error submitting 'noSegment' marker for category '" + category + "' for video '" + videoID + "'");
Logger.error("Error submitting 'lockCategories' marker for category '" + category + "' for video '" + videoID + "'");
Logger.error(err);
res.status(500).json({
message: "Internal Server Error: Could not write marker to the database.",

14
src/routes/postSkipSegments.ts
View File

@@ -14,7 +14,7 @@ import {Request, Response} from 'express';
import { skipSegmentsHashKey, skipSegmentsKey } from '../middleware/redisKeys';
import redis from '../utils/redis';
import { Category, CategoryActionType, IncomingSegment, Segment, SegmentUUID, Service, VideoDuration, VideoID } from '../types/segments.model';
import { deleteNoSegments } from './deleteNoSegments';
import { deleteLockCategories } from './deleteLockCategories';
import { getCategoryActionType } from '../utils/categoryInfo';
interface APIVideoInfo {
@@ -357,7 +357,7 @@ export async function postSkipSegments(req: Request, res: Response) {
return res.status(403).send('Submission rejected due to a warning from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. Could you please send a message in Discord or Matrix so we can further help you?');
}
let noSegmentList = (await db.prepare('all', 'SELECT category from "noSegments" where "videoID" = ?', [videoID])).map((list: any) => {
let lockedCategoryList = (await db.prepare('all', 'SELECT category from "lockCategories" where "videoID" = ?', [videoID])).map((list: any) => {
return list.category;
});
@@ -389,9 +389,9 @@ export async function postSkipSegments(req: Request, res: Response) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "UUID" = ?`, [submission.UUID]);
}
// Reset no segments
noSegmentList = [];
deleteNoSegments(videoID, null);
// Reset lock categories
lockedCategoryList = [];
deleteLockCategories(videoID, null);
}
// Check if all submissions are correct
@@ -407,8 +407,8 @@ export async function postSkipSegments(req: Request, res: Response) {
return;
}
// Reject segemnt if it's in the no segments list
if (!isVIP && noSegmentList.indexOf(segments[i].category) !== -1) {
// Reject segment if it's in the locked categories list
if (!isVIP && lockedCategoryList.indexOf(segments[i].category) !== -1) {
// TODO: Do something about the fradulent submission
Logger.warn("Caught a no-segment submission. userID: '" + userID + "', videoID: '" + videoID + "', category: '" + segments[i].category + "'");
res.status(403).send(

10
src/routes/setUsername.ts
View File

@@ -21,6 +21,10 @@ export async function setUsername(req: Request, res: Response) {
res.sendStatus(200);
return;
}
// remove unicode control characters from username (example: \n, \r, \t etc.)
// source: https://en.wikipedia.org/wiki/Control_character#In_Unicode
userName = userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, '');
if (adminUserIDInput != undefined) {
//this is the admin controlling the other users account, don't hash the controling account's ID
@@ -35,6 +39,12 @@ export async function setUsername(req: Request, res: Response) {
//hash the userID
userID = getHash(userID);
}
if (["7e7eb6c6dbbdba6a106a38e87eae29ed8689d0033cb629bb324a8dab615c5a97", "e1839ce056d185f176f30a3d04a79242110fe46ad6e9bd1a9170f56857d1b148"].includes(userID)) {
// Don't allow
res.sendStatus(200);
return;
}
try {
//check if username is already set

6
src/routes/shadowBanUser.ts
View File

@@ -43,8 +43,8 @@ export async function shadowBanUser(req: Request, res: Response) {
//find all previous submissions and hide them
if (unHideOldSubmissions) {
await db.prepare('run', `UPDATE "sponsorTimes" SET "shadowHidden" = 1 WHERE "userID" = ?
AND NOT EXISTS ( SELECT "videoID", "category" FROM "noSegments" WHERE
"sponsorTimes"."videoID" = "noSegments"."videoID" AND "sponsorTimes"."category" = "noSegments"."category")`, [userID]);
AND NOT EXISTS ( SELECT "videoID", "category" FROM "lockCategories" WHERE
"sponsorTimes"."videoID" = "lockCategories"."videoID" AND "sponsorTimes"."category" = "lockCategories"."category")`, [userID]);
}
} else if (!enabled && row.userCount > 0) {
//remove them from the shadow ban list
@@ -53,7 +53,7 @@ export async function shadowBanUser(req: Request, res: Response) {
//find all previous submissions and unhide them
if (unHideOldSubmissions) {
let segmentsToIgnore = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st
JOIN "noSegments" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?`
JOIN "lockCategories" ns on "st"."videoID" = "ns"."videoID" AND st.category = ns.category WHERE "st"."userID" = ?`
, [userID])).map((item: {UUID: string}) => item.UUID);
let allSegments = (await db.prepare('all', `SELECT "UUID" FROM "sponsorTimes" st WHERE "st"."userID" = ?`, [userID]))
.map((item: {UUID: string}) => item.UUID);

92
src/routes/voteOnSponsorTime.ts
View File

@@ -180,51 +180,54 @@ async function categoryVote(UUID: SegmentUUID, userID: UserID, isVIP: boolean, i
const timeSubmitted = Date.now();
const voteAmount = isVIP ? 500 : 1;
const ableToVote = isVIP || finalResponse.finalStatus === 200 || true;
// Add the vote
if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
// Update the already existing db entry
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
} else {
// Add a db entry
await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
}
if (ableToVote) {
// Add the vote
if ((await db.prepare('get', `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) {
// Update the already existing db entry
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]);
} else {
// Add a db entry
await db.prepare('run', `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]);
}
// Add the info into the private db
if (usersLastVoteInfo?.votes > 0) {
// Reverse the previous vote
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);
// Add the info into the private db
if (usersLastVoteInfo?.votes > 0) {
// Reverse the previous vote
await db.prepare('run', `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]);
await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
} else {
await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
}
await privateDB.prepare('run', `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]);
} else {
await privateDB.prepare('run', `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]);
}
// See if the submissions category is ready to change
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
// See if the submissions category is ready to change
const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, videoInfo.category]);
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
const startingVotes = isSubmissionVIP ? 10000 : 1;
const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID);
const startingVotes = isSubmissionVIP ? 10000 : 1;
// Change this value from 1 in the future to make it harder to change categories
// Done this way without ORs incase the value is zero
const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
// Change this value from 1 in the future to make it harder to change categories
// Done this way without ORs incase the value is zero
const currentCategoryCount = (currentCategoryInfo === undefined || currentCategoryInfo === null) ? startingVotes : currentCategoryInfo.votes;
// Add submission as vote
if (!currentCategoryInfo && submissionInfo) {
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
// Add submission as vote
if (!currentCategoryInfo && submissionInfo) {
await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, videoInfo.category, currentCategoryCount]);
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
}
await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", videoInfo.category, submissionInfo.timeSubmitted]);
}
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount;
//TODO: In the future, raise this number from zero to make it harder to change categories
// VIPs change it every time
if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
// Replace the category
await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
//TODO: In the future, raise this number from zero to make it harder to change categories
// VIPs change it every time
if (nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) || isVIP || isOwnSubmission) {
// Replace the category
await db.prepare('run', `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]);
}
}
clearRedisCache(videoInfo);
@@ -273,8 +276,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
// If not upvote
if (!isVIP && type !== 1) {
const isSegmentLocked = async () => !!(await db.prepare('get', `SELECT "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]))?.locked;
const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "noSegments".category from "noSegments" left join "sponsorTimes"' +
' on ("noSegments"."videoID" = "sponsorTimes"."videoID" and "noSegments".category = "sponsorTimes".category)' +
const isVideoLocked = async () => !!(await db.prepare('get', 'SELECT "lockCategories".category from "lockCategories" left join "sponsorTimes"' +
' on ("lockCategories"."videoID" = "sponsorTimes"."videoID" and "lockCategories".category = "sponsorTimes".category)' +
' where "UUID" = ?', [UUID]));
if (await isSegmentLocked() || await isVideoLocked()) {
@@ -287,13 +290,19 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
return categoryVote(UUID, nonAnonUserID, isVIP, isOwnSubmission, category, hashedIP, finalResponse, res);
}
if (type == 1 && !isVIP && !isOwnSubmission) {
if (type !== undefined && !isVIP && !isOwnSubmission) {
// Check if upvoting hidden segment
const voteInfo = await db.prepare('get', `SELECT votes FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID]);
if (voteInfo && voteInfo.votes <= -2) {
res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
return;
if (type == 1) {
res.status(403).send("Not allowed to upvote segment with too many downvotes unless you are VIP.");
return;
} else if (type == 0) {
// Already downvoted enough, ignore
res.status(200).send();
return;
}
}
}
@@ -378,7 +387,8 @@ export async function voteOnSponsorTime(req: Request, res: Response) {
const ableToVote = isVIP
|| ((await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ?`, [nonAnonUserID])) !== undefined
&& (await privateDB.prepare("get", `SELECT "userID" FROM "shadowBannedUsers" WHERE "userID" = ?`, [nonAnonUserID])) === undefined
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined);
&& (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID])) === undefined)
&& finalResponse.finalStatus === 200;
if (ableToVote) {
//update the votes table

17
src/types/config.model.ts
View File

@@ -37,7 +37,9 @@ export interface SBSConfig {
minimumPrefix?: string;
maximumPrefix?: string;
redis?: redis.ClientOpts;
maxRewardTimePerSegmentInSeconds?: number;
postgres?: PoolConfig;
dumpDatabase?: DumpDatabase;
}
export interface WebhookConfig {
@@ -61,4 +63,17 @@ export interface PostgresConfig {
createDbIfNotExists: boolean;
enableWalCheckpointNumber: boolean;
postgres: PoolConfig;
}
}
export interface DumpDatabase {
enabled: boolean;
minTimeBetweenMs: number;
appExportPath: string;
postgresExportPath: string;
tables: DumpDatabaseTable[];
}
export interface DumpDatabaseTable {
name: string;
order?: string;
}

4
src/utils/redis.ts
View File

@@ -29,6 +29,10 @@ if (config.redis) {
exportObject.getAsync = (key) => new Promise((resolve) => client.get(key, (err, reply) => resolve({err, reply})));
exportObject.setAsync = (key, value) => new Promise((resolve) => client.set(key, value, (err, reply) => resolve({err, reply})));
exportObject.delAsync = (...keys) => new Promise((resolve) => client.del(keys, (err) => resolve(err)));
client.on("error", function(error) {
Logger.error(error);
});
}
export default exportObject;

4
test.json
View File

@@ -10,8 +10,8 @@
"discordNeuralBlockRejectWebhookURL": "http://127.0.0.1:8081/NeuralBlockRejectWebhook",
"neuralBlockURL": "http://127.0.0.1:8081/NeuralBlock",
"behindProxy": true,
"db": "./test/databases/sponsorTimes.db",
"privateDB": "./test/databases/private.db",
"db": ":memory:",
"privateDB": ":memory:",
"createDatabaseIfNotExist": true,
"schemaFolder": "./databases",
"dbSchema": "./databases/_sponsorTimes.db.sql",

86
test/cases/noSegmentRecords.ts → test/cases/lockCategoriesRecords.ts
View File

@@ -4,21 +4,21 @@ import {getHash} from '../../src/utils/getHash';
import {db} from '../../src/databases/databases';
describe('noSegmentRecords', () => {
describe('lockCategoriesRecords', () => {
before(async () => {
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-noSegments") + "')");
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-lockCategories") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id', 'intro')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id', 'sponsor')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id', 'intro')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'no-segments-video-id-1', 'intro')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'noSubmitVideo', 'sponsor')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'no-segments-video-id-1', 'intro')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'lockCategoryVideo', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record', 'sponsor')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-noSegments") + "', 'delete-record-1', 'intro')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-lockCategories") + "', 'delete-record-1', 'intro')");
});
it('Should update the database version when starting the application', async () => {
@@ -30,7 +30,7 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories not in video (http response)', (done: Done) => {
let json = {
videoID: 'no-segments-video-id',
userID: 'VIPUser-noSegments',
userID: 'VIPUser-lockCategories',
categories: [
'outro',
'shilling',
@@ -48,7 +48,7 @@ describe('noSegmentRecords', () => {
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -75,7 +75,7 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories not in video (sql check)', (done: Done) => {
let json = {
videoID: 'no-segments-video-id-1',
userID: 'VIPUser-noSegments',
userID: 'VIPUser-lockCategories',
categories: [
'outro',
'shilling',
@@ -86,7 +86,7 @@ describe('noSegmentRecords', () => {
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -95,7 +95,7 @@ describe('noSegmentRecords', () => {
})
.then(async res => {
if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['no-segments-video-id-1']);
let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['no-segments-video-id-1']);
if (result.length !== 4) {
console.log(result);
done("Expected 4 entrys in db, got " + result.length);
@@ -114,13 +114,13 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories with _ in the category', (done: Done) => {
let json = {
videoID: 'underscore',
userID: 'VIPUser-noSegments',
userID: 'VIPUser-lockCategories',
categories: [
'word_word',
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -129,7 +129,7 @@ describe('noSegmentRecords', () => {
})
.then(async res => {
if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['underscore']);
let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['underscore']);
if (result.length !== 1) {
console.log(result);
done("Expected 1 entrys in db, got " + result.length);
@@ -148,13 +148,13 @@ describe('noSegmentRecords', () => {
it('Should be able to submit categories with upper and lower case in the category', (done: Done) => {
let json = {
videoID: 'bothCases',
userID: 'VIPUser-noSegments',
userID: 'VIPUser-lockCategories',
categories: [
'wordWord',
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -163,7 +163,7 @@ describe('noSegmentRecords', () => {
})
.then(async res => {
if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['bothCases']);
let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['bothCases']);
if (result.length !== 1) {
console.log(result);
done("Expected 1 entrys in db, got " + result.length);
@@ -182,13 +182,13 @@ describe('noSegmentRecords', () => {
it('Should not be able to submit categories with $ in the category', (done: Done) => {
let json = {
videoID: 'specialChar',
userID: 'VIPUser-noSegments',
userID: 'VIPUser-lockCategories',
categories: [
'word&word',
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -197,7 +197,7 @@ describe('noSegmentRecords', () => {
})
.then(async res => {
if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['specialChar']);
let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['specialChar']);
if (result.length !== 0) {
console.log(result);
done("Expected 0 entrys in db, got " + result.length);
@@ -214,7 +214,7 @@ describe('noSegmentRecords', () => {
});
it('Should return 400 for missing params', (done: Done) => {
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -238,7 +238,7 @@ describe('noSegmentRecords', () => {
categories: [],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -262,7 +262,7 @@ describe('noSegmentRecords', () => {
categories: ['sponsor'],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -286,7 +286,7 @@ describe('noSegmentRecords', () => {
categories: ['sponsor'],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -310,7 +310,7 @@ describe('noSegmentRecords', () => {
categories: {},
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -334,7 +334,7 @@ describe('noSegmentRecords', () => {
categories: 'sponsor',
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -360,7 +360,7 @@ describe('noSegmentRecords', () => {
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -377,16 +377,16 @@ describe('noSegmentRecords', () => {
.catch(err => done(err));
});
it('Should be able to delete a noSegment record', (done: Done) => {
it('Should be able to delete a lockCategories record', (done: Done) => {
let json = {
videoID: 'delete-record',
userID: 'VIPUser-noSegments',
userID: 'VIPUser-lockCategories',
categories: [
'sponsor',
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'DELETE',
headers: {
'Content-Type': 'application/json',
@@ -395,7 +395,7 @@ describe('noSegmentRecords', () => {
})
.then(async res => {
if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['delete-record']);
let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['delete-record']);
if (result.length === 0) {
done();
} else {
@@ -408,16 +408,16 @@ describe('noSegmentRecords', () => {
.catch(err => done(err));
});
it('Should be able to delete one noSegment record without removing another', (done: Done) => {
it('Should be able to delete one lockCategories record without removing another', (done: Done) => {
let json = {
videoID: 'delete-record-1',
userID: 'VIPUser-noSegments',
userID: 'VIPUser-lockCategories',
categories: [
'sponsor',
],
};
fetch(getbaseURL() + "/api/noSegments", {
fetch(getbaseURL() + "/api/lockCategories", {
method: 'DELETE',
headers: {
'Content-Type': 'application/json',
@@ -426,7 +426,7 @@ describe('noSegmentRecords', () => {
})
.then(async res => {
if (res.status === 200) {
let result = await db.prepare('all', 'SELECT * FROM "noSegments" WHERE "videoID" = ?', ['delete-record-1']);
let result = await db.prepare('all', 'SELECT * FROM "lockCategories" WHERE "videoID" = ?', ['delete-record-1']);
if (result.length === 1) {
done();
} else {
@@ -445,7 +445,7 @@ describe('noSegmentRecords', () => {
* To test the submission code properly see ./test/cases/postSkipSegments.js
*/
it('Should not be able to submit a segment to a video with a no-segment record (single submission)', (done: Done) => {
it('Should not be able to submit a segment to a video with a lock-category record (single submission)', (done: Done) => {
fetch(getbaseURL() + "/api/postVideoSponsorTimes", {
method: 'POST',
headers: {
@@ -453,7 +453,7 @@ describe('noSegmentRecords', () => {
},
body: JSON.stringify({
userID: "testman42",
videoID: "noSubmitVideo",
videoID: "lockCategoryVideo",
segments: [{
segment: [20, 40],
category: "sponsor",
@@ -478,7 +478,7 @@ describe('noSegmentRecords', () => {
},
body: JSON.stringify({
userID: "testman42",
videoID: "noSubmitVideo",
videoID: "lockCategoryVideo",
segments: [{
segment: [20, 40],
category: "sponsor",
@@ -507,7 +507,7 @@ describe('noSegmentRecords', () => {
},
body: JSON.stringify({
userID: "testman42",
videoID: "noSubmitVideo",
videoID: "lockCategoryVideo",
segments: [{
segment: [20, 40],
category: "intro",

8
test/cases/postSkipSegments.ts
View File

@@ -195,8 +195,8 @@ describe('postSkipSegments', () => {
});
it('Should be able to submit with a new duration, and hide old submissions and remove segment locks', async () => {
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category")
VALUES ('` + getHash("VIPUser-noSegments") + "', 'noDuration', 'sponsor')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category")
VALUES ('` + getHash("VIPUser-lockCategories") + "', 'noDuration', 'sponsor')");
try {
const res = await fetch(getbaseURL()
@@ -217,13 +217,13 @@ describe('postSkipSegments', () => {
});
if (res.status === 200) {
const noSegmentsRow = await db.prepare('get', `SELECT * from "noSegments" WHERE videoID = ?`, ["noDuration"]);
const lockCategoriesRow = await db.prepare('get', `SELECT * from "lockCategories" WHERE videoID = ?`, ["noDuration"]);
const videoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 0`, ["noDuration"]);
const videoRow = videoRows[0];
const hiddenVideoRows = await db.prepare('all', `SELECT "startTime", "endTime", "locked", "category", "videoDuration"
FROM "sponsorTimes" WHERE "videoID" = ? AND hidden = 1`, ["noDuration"]);
if (noSegmentsRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10
if (lockCategoriesRow === undefined && videoRows.length === 1 && hiddenVideoRows.length === 1 && videoRow.startTime === 1 && videoRow.endTime === 10
&& videoRow.locked === 0 && videoRow.category === "sponsor" && videoRow.videoDuration === 100) {
return;
} else {

6
test/cases/unBan.ts
View File

@@ -13,7 +13,7 @@ describe('unBan', () => {
await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" VALUES('testEntity-unBan')`);
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser-unBan") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')");
await db.prepare("run", `INSERT INTO "lockCategories" ("userID", "videoID", "category") VALUES ('` + getHash("VIPUser-unBan") + "', 'unBan-videoID-1', 'sponsor')");
let startOfInsertSegmentQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
await db.prepare("run", startOfInsertSegmentQuery + "('unBan-videoID-0', 1, 11, 2, 'unBan-uuid-0', 'testMan-unBan', 0, 50, 'sponsor', 1, '" + getHash('unBan-videoID-0', 1) + "')");
@@ -47,7 +47,7 @@ describe('unBan', () => {
.catch(err => done(err));
});
it('Should be able to unban a user and re-enable shadow banned segments without noSegment entrys', (done) => {
it('Should be able to unban a user and re-enable shadow banned segments without lockCategories entrys', (done) => {
fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testWoman-unBan&adminUserID=VIPUser-unBan&enabled=false", {
method: 'POST',
headers: {
@@ -72,7 +72,7 @@ describe('unBan', () => {
.catch(err => done(err));
});
it('Should be able to unban a user and re-enable shadow banned segments with a mix of noSegment entrys', (done) => {
it('Should be able to unban a user and re-enable shadow banned segments with a mix of lockCategories entrys', (done) => {
fetch(utils.getbaseURL() + "/api/shadowBanUser?userID=testEntity-unBan&adminUserID=VIPUser-unBan&enabled=false", {
method: 'POST',
headers: {

42
test/cases/voteOnSponsorTime.ts
View File

@@ -59,7 +59,7 @@ describe('voteOnSponsorTime', () => {
await db.prepare("run", `INSERT INTO "vipUsers" ("userID") VALUES ('` + getHash("VIPUser") + "')");
await privateDB.prepare("run", `INSERT INTO "shadowBannedUsers" ("userID") VALUES ('` + getHash("randomID4") + "')");
await db.prepare("run", `INSERT INTO "noSegments" ("videoID", "userID", "category") VALUES ('no-sponsor-segments-video', 'someUser', 'sponsor')`);
await db.prepare("run", `INSERT INTO "lockCategories" ("videoID", "userID", "category") VALUES ('no-sponsor-segments-video', 'someUser', 'sponsor')`);
});
@@ -386,10 +386,25 @@ describe('voteOnSponsorTime', () => {
fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=1")
.then(async res => {
if (res.status === 403) {
let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
if (res.status === 403 && row.votes === -3) {
done();
} else {
done("Status code was " + res.status + " instead of 403");
done("Status code was " + res.status + ", row is " + JSON.stringify(row));
}
})
.catch(err => done(err));
});
it('Non-VIP should not be able to downvote "dead" submission', (done: Done) => {
fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=randomID2&UUID=vote-uuid-5&type=0")
.then(async res => {
let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["vote-uuid-5"]);
if (res.status === 200 && row.votes === -3) {
done();
} else {
done("Status code was " + res.status + ", row is " + JSON.stringify(row));
}
})
.catch(err => done(err));
@@ -428,12 +443,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should not be able to downvote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=0")
+ "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=0")
.then(async res => {
if (res.status === 403) {
let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 403 && row.votes === 2) {
done();
} else {
done("Status code was " + res.status + " instead of 403");
done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
}
})
.catch(err => done(err));
@@ -441,12 +457,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should be able to upvote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&type=1")
+ "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&type=1")
.then(async res => {
if (res.status === 200) {
let row = await db.prepare('get', `SELECT "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 200 && row.votes === 3) {
done();
} else {
done("Status code was " + res.status + " instead of 200");
done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
}
})
.catch(err => done(err));
@@ -454,12 +471,13 @@ describe('voteOnSponsorTime', () => {
it('Non-VIP should not be able to category vote on a segment with no-segments category', (done: Done) => {
fetch(getbaseURL()
+ "/api/voteOnSponsorTime?userID=no-segments-voter&UUID=no-sponsor-segments-uuid-0&category=outro")
+ "/api/voteOnSponsorTime?userID=randomID&UUID=no-sponsor-segments-uuid-0&category=outro")
.then(async res => {
if (res.status === 403) {
let row = await db.prepare('get', `SELECT "category" FROM "sponsorTimes" WHERE "UUID" = ?`, ["no-sponsor-segments-uuid-0"]);
if (res.status === 403 && row.category === "sponsor") {
done();
} else {
done("Status code was " + res.status + " instead of 403");
done("Status code was " + res.status + " instead of 403, row was " + JSON.stringify(row));
}
})
.catch(err => done(err));