Merge pull request #215 from ajayyy/export

Add new "Service" option

.gitignore

@@ -99,6 +99,7 @@ test/databases/sponsorTimes.db
 test/databases/sponsorTimes.db-shm
 test/databases/sponsorTimes.db-wal
 test/databases/private.db
+docker/database-export
 
 # Config files
 config.json

databases/_upgrade_sponsorTimes_7.sql

@@ -0,0 +1,28 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_7" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "locked" INTEGER NOT NULL default '0',
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+    "service"	TEXT NOT NULL DEFAULT 'YouTube',
+	"shadowHidden"	INTEGER NOT NULL,
+    "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_7 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category",'YouTube', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_7 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 7 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_sponsorTimes_8.sql

@@ -0,0 +1,29 @@
+BEGIN TRANSACTION;
+
+/* Add Service field */
+CREATE TABLE "sqlb_temp_table_8" (
+	"videoID"	TEXT NOT NULL,
+	"startTime"	REAL NOT NULL,
+	"endTime"	REAL NOT NULL,
+	"votes"	INTEGER NOT NULL,
+    "locked" INTEGER NOT NULL default '0',
+    "incorrectVotes" INTEGER NOT NULL default '1',
+	"UUID"	TEXT NOT NULL UNIQUE,
+	"userID"	TEXT NOT NULL,
+	"timeSubmitted"	INTEGER NOT NULL,
+	"views"	INTEGER NOT NULL,
+	"category"	TEXT NOT NULL DEFAULT 'sponsor',
+    "service"	TEXT NOT NULL DEFAULT 'YouTube',
+    "videoDuration"	INTEGER NOT NULL DEFAULT '0',
+	"shadowHidden"	INTEGER NOT NULL,
+    "hashedVideoID" TEXT NOT NULL default ''
+);
+
+INSERT INTO sqlb_temp_table_8 SELECT "videoID","startTime","endTime","votes","locked","incorrectVotes","UUID","userID","timeSubmitted","views","category","service",'0', "shadowHidden","hashedVideoID" FROM "sponsorTimes";
+
+DROP TABLE "sponsorTimes";
+ALTER TABLE sqlb_temp_table_8 RENAME TO "sponsorTimes";
+
+UPDATE "config" SET value = 8 WHERE key = 'version';
+
+COMMIT;

docker/docker-compose.yml

@@ -7,6 +7,7 @@ services:
       - database.env
     volumes:
       - database-data:/var/lib/postgresql/data
+      - ./database-export/:/opt/exports
     ports:
       - 127.0.0.1:5432:5432
   redis:

src/app.ts

@@ -26,6 +26,7 @@ import {userCounter} from './middleware/userCounter';
 import {loggerMiddleware} from './middleware/logger';
 import {corsMiddleware} from './middleware/cors';
 import {rateLimitMiddleware} from './middleware/requestRateLimit';
+import dumpDatabase from './routes/dumpDatabase';
 
 
 export function createServer(callback: () => void) {
@@ -127,7 +128,12 @@ function setupRoutes(app: Express) {
     //get if user is a vip
     app.post('/api/segmentShift', postSegmentShift);
 
+    if (config.postgres) {
+        app.get('/database', (req, res) => dumpDatabase(req, res, true));
+        app.get('/database.json', (req, res) => dumpDatabase(req, res, false));
+    } else {
         app.get('/database.db', function (req: Request, res: Response) {
             res.sendFile("./databases/sponsorTimes.db", {root: "./"});
         });
     }
+}

src/routes/dumpDatabase.ts

@@ -0,0 +1,78 @@
+import {db} from '../databases/databases';
+import {Logger} from '../utils/logger';
+import {Request, Response} from 'express';
+import { config } from '../config';
+
+const ONE_MINUTE = 1000 * 60;
+
+const styleHeader = `<style>body{font-family: sans-serif}</style>`
+
+const licenseHeader = `<p>The API and database follow <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="nofollow">CC BY-NC-SA 4.0</a> unless you have explicit permission.</p>
+<p><a href="https://gist.github.com/ajayyy/4b27dfc66e33941a45aeaadccb51de71">Attribution Template</a></p>
+<p>If you need to use the database or API in a way that violates this license, contact me with your reason and I may grant you access under a different license.</p></a></p>`;
+
+const tables = [{
+    name: "sponsorTimes",
+    order: "timeSubmitted"
+},
+{
+    name: "userNames"
+},
+{
+    name: "categoryVotes"
+},
+{
+    name: "noSegments",
+},
+{
+    name: "warnings",
+    order: "issueTime"
+},
+{
+    name: "vipUsers"
+}];
+
+const links: string[] = tables.map((table) => `/database/${table.name}.csv`);
+
+const linksHTML: string = tables.map((table) => `<p><a href="/database/${table.name}.csv">${table.name}.csv</a></p>`)
+                        .reduce((acc, url) => acc + url, "");
+
+let lastUpdate = 0;
+
+export default function dumpDatabase(req: Request, res: Response, showPage: boolean) {
+    if (!config.postgres) {
+        res.status(404).send("Not supported on this instance");
+        return;
+    }
+
+    const now = Date.now();
+    const updateQueued = now - lastUpdate > ONE_MINUTE;
+
+    res.status(200)
+    
+    if (showPage) {
+        res.send(`${styleHeader}
+            <h1>SponsorBlock database dumps</h1>${licenseHeader}
+            <h3>How this works</h3>
+            Send a request to <code>https://sponsor.ajay.app/database.json</code>, or visit this page to trigger the database dump to run.
+            Then, you can download the csv files below, or use the links returned from the JSON request.
+            <h3>Links</h3>
+            ${linksHTML}<br/>
+            ${updateQueued ? `Update queued.` : ``} Last updated: ${lastUpdate ? new Date(lastUpdate).toUTCString() : `Unknown`}`);
+    } else {
+        res.send({
+            lastUpdated: lastUpdate,
+            updateQueued,
+            links
+        })
+    }
+
+    if (updateQueued) {
+        lastUpdate = Date.now();
+
+        for (const table of tables) {
+            db.prepare('run', `COPY (SELECT * FROM "${table.name}"${table.order ? ` ORDER BY "${table.order}"` : ``}) 
+                    TO '/opt/exports/${table.name}.csv' WITH (FORMAT CSV, HEADER true);`);
+        }
+    }
+}

src/routes/getSkipSegments.ts

@@ -4,7 +4,7 @@ import { config } from '../config';
 import { db, privateDB } from '../databases/databases';
 import { skipSegmentsKey } from '../middleware/redisKeys';
 import { SBRecord } from '../types/lib.model';
-import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
+import { Category, DBSegment, HashedIP, IPAddress, OverlappingSegmentGroup, Segment, SegmentCache, Service, VideoData, VideoID, VideoIDHash, Visibility, VotableObject } from "../types/segments.model";
 import { getHash } from '../utils/getHash';
 import { getIP } from '../utils/getIP';
 import { Logger } from '../utils/logger';
@@ -43,11 +43,12 @@ async function prepareCategorySegments(req: Request, videoID: VideoID, category:
     return chooseSegments(filteredSegments).map((chosenSegment) => ({
         category,
         segment: [chosenSegment.startTime, chosenSegment.endTime],
-        UUID: chosenSegment.UUID
+        UUID: chosenSegment.UUID,
+        videoDuration: chosenSegment.videoDuration
     }));
 }
 
-async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[]): Promise<Segment[]> {
+async function getSegmentsByVideoID(req: Request, videoID: string, categories: Category[], service: Service): Promise<Segment[]> {
     const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
     const segments: Segment[] = [];
 
@@ -58,9 +59,9 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
         const segmentsByCategory: SBRecord<Category, DBSegment[]> = (await db
             .prepare(
                 'all',
-                `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden" FROM "sponsorTimes" 
+                `SELECT "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden" FROM "sponsorTimes" 
-                WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
+                WHERE "videoID" = ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
-                [videoID]
+                [videoID, service]
             )).reduce((acc: SBRecord<Category, DBSegment[]>, segment: DBSegment) => {
                 acc[segment.category] = acc[segment.category] || [];
                 acc[segment.category].push(segment);
@@ -81,7 +82,7 @@ async function getSegmentsByVideoID(req: Request, videoID: string, categories: C
     }
 }
 
-async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[]): Promise<SBRecord<VideoID, VideoData>> {
+async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash, categories: Category[], service: Service): Promise<SBRecord<VideoID, VideoData>> {
     const cache: SegmentCache = {shadowHiddenSegmentIPs: {}};
     const segments: SBRecord<VideoID, VideoData> = {};
 
@@ -94,9 +95,9 @@ async function getSegmentsByHash(req: Request, hashedVideoIDPrefix: VideoIDHash,
         const segmentPerVideoID: SegmentWithHashPerVideoID = (await db
             .prepare(
                 'all',
-                `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
+                `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "category", "videoDuration", "shadowHidden", "hashedVideoID" FROM "sponsorTimes"
-                WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) ORDER BY "startTime"`,
+                WHERE "hashedVideoID" LIKE ? AND "category" IN (${categories.map((c) => "'" + c + "'")}) AND "service" = ? ORDER BY "startTime"`,
-                [hashedVideoIDPrefix + '%']
+                [hashedVideoIDPrefix + '%', service]
             )).reduce((acc: SegmentWithHashPerVideoID, segment: DBSegment) => {
                 acc[segment.videoID] = acc[segment.videoID] || {
                     hash: segment.hashedVideoID,
@@ -239,6 +240,11 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
             ? [req.query.category]
             : ['sponsor'];
 
+    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    if (!Object.values(Service).some((val) => val == service)) {
+        service = Service.YouTube;
+    }
+
     // Only 404s are cached at the moment
     const redisResult = await redis.getAsync(skipSegmentsKey(videoID));
     
@@ -251,7 +257,7 @@ async function handleGetSegments(req: Request, res: Response): Promise<Segment[]
         }
     }
 
-    const segments = await getSegmentsByVideoID(req, videoID, categories);
+    const segments = await getSegmentsByVideoID(req, videoID, categories, service);
 
     if (segments === null || segments === undefined) {
         res.sendStatus(500);

src/routes/getSkipSegmentsByHash.ts

@@ -1,7 +1,7 @@
 import {hashPrefixTester} from '../utils/hashPrefixTester';
 import {getSegmentsByHash} from './getSkipSegments';
 import {Request, Response} from 'express';
-import { Category, VideoIDHash } from '../types/segments.model';
+import { Category, Service, VideoIDHash } from '../types/segments.model';
 
 export async function getSkipSegmentsByHash(req: Request, res: Response) {
     let hashPrefix = req.params.prefix as VideoIDHash;
@@ -26,11 +26,16 @@ export async function getSkipSegmentsByHash(req: Request, res: Response) {
         return res.status(400).send("Bad parameter: categories (invalid JSON)");
     }
 
+    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    if (!Object.values(Service).some((val) => val == service)) {
+        service = Service.YouTube;
+    }
+    
     // filter out none string elements, only flat array with strings is valid
     categories = categories.filter((item: any) => typeof item === "string");
 
     // Get all video id's that match hash prefix
-    const segments = await getSegmentsByHash(req, hashPrefix, categories);
+    const segments = await getSegmentsByHash(req, hashPrefix, categories, service);
 
     if (!segments) return res.status(404).json([]);
 

src/routes/postSkipSegments.ts

@@ -13,7 +13,12 @@ import {dispatchEvent} from '../utils/webhookUtils';
 import {Request, Response} from 'express';
 import { skipSegmentsKey } from '../middleware/redisKeys';
 import redis from '../utils/redis';
+import { Category, IncomingSegment, Segment, Service, VideoDuration, VideoID } from '../types/segments.model';
 
+interface APIVideoInfo {
+    err: string | boolean,
+    data: any
+}
 
 async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: any, {submissionStart, submissionEnd}: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
     const row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
@@ -45,15 +50,12 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
     });
 }
 
-async function sendWebhooks(userID: string, videoID: string, UUID: string, segmentInfo: any) {
+async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
-    if (config.youtubeAPIKey !== null) {
+    if (apiVideoInfo && service == Service.YouTube) {
         const userSubmissionCountRow = await db.prepare('get', `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
 
-        YouTubeAPI.listVideos(videoID, (err: any, data: any) => {
+        const {data, err} = apiVideoInfo;
-            if (err || data.items.length === 0) {
+        if (err) return;
-                err && Logger.error(err);
-                return;
-            }
 
         const startTime = parseFloat(segmentInfo.segment[0]);
         const endTime = parseFloat(segmentInfo.segment[1]);
@@ -101,7 +103,6 @@ async function sendWebhooks(userID: string, videoID: string, UUID: string, segme
             Logger.error(JSON.stringify(err));
             Logger.error("\n");
         });
-        });
     }
 }
 
@@ -166,34 +167,25 @@ async function sendWebhooksNB(userID: string, videoID: string, UUID: string, sta
 // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
 //   false for a pass - it was confusing and lead to this bug - any use of this function in
 //   the future could have the same problem.
-async function autoModerateSubmission(submission: { videoID: any; userID: any; segments: any }) {
+async function autoModerateSubmission(apiVideoInfo: APIVideoInfo, 
-    // Get the video information from the youtube API
+            submission: { videoID: any; userID: any; segments: any }) {
-    if (config.youtubeAPIKey !== null) {
+    if (apiVideoInfo) {
-        const {err, data} = await new Promise((resolve) => {
+        const {err, data} = apiVideoInfo;
-            YouTubeAPI.listVideos(submission.videoID, (err: any, data: any) => resolve({err, data}));
+        if (err) return false;
-        });
 
-        if (err) {
-            return false;
-        } else {
         // Check to see if video exists
-            if (data.pageInfo.totalResults === 0) {
+        if (data.pageInfo.totalResults === 0) return "No video exists with id " + submission.videoID;
-                return "No video exists with id " + submission.videoID;
+
-            } else {
+        const duration = getYouTubeVideoDuration(apiVideoInfo);
         const segments = submission.segments;
         let nbString = "";
         for (let i = 0; i < segments.length; i++) {
             const startTime = parseFloat(segments[i].segment[0]);
             const endTime = parseFloat(segments[i].segment[1]);
 
-                    let duration = data.items[0].contentDetails.duration;
-                    duration = isoDurations.toSeconds(isoDurations.parse(duration));
             if (duration == 0) {
                 // Allow submission if the duration is 0 (bug in youtube api)
                 return false;
-                    } else if ((endTime - startTime) > (duration / 100) * 80) {
-                        // Reject submission if over 80% of the video
-                        return "One of your submitted segments is over 80% of the video.";
             } else {
                 if (segments[i].category === "sponsor") {
                     //Prepare timestamps to send to NB all at once
@@ -201,6 +193,41 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s
                 }
             }
         }
+
+        // Get all submissions for this user
+        const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [submission.userID, submission.videoID]);
+        const allSegmentTimes = [];
+        if (allSubmittedByUser !== undefined) {
+            //add segments the user has previously submitted
+            for (const segmentInfo of allSubmittedByUser) {
+                allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
+            }
+        }
+
+        //add segments they are trying to add in this submission
+        for (let i = 0; i < segments.length; i++) {
+            let startTime = parseFloat(segments[i].segment[0]);
+            let endTime = parseFloat(segments[i].segment[1]);
+            allSegmentTimes.push([startTime, endTime]);
+        }
+
+        //merge all the times into non-overlapping arrays
+        const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
+            return a[0] - b[0] || a[1] - b[1];
+        }));
+
+        let videoDuration = data.items[0].contentDetails.duration;
+        videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
+        if (videoDuration != 0) {
+            let allSegmentDuration = 0;
+            //sum all segment times together
+            allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
+            if (allSegmentDuration > (videoDuration / 100) * 80) {
+                // Reject submission if all segments combine are over 80% of the video
+                return "Total length of your submitted segments are over 80% of the video.";
+            }
+        }
+
         // Check NeuralBlock
         const neuralBlockURL = config.neuralBlockURL;
         if (!neuralBlockURL) return false;
@@ -227,13 +254,12 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s
             }
 
         }
+
         if (nbDecision) {
             return "Rejected based on NeuralBlock predictions.";
         } else {
             return false;
         }
-            }
-        }
     } else {
         Logger.debug("Skipped YouTube API");
 
@@ -243,6 +269,21 @@ async function autoModerateSubmission(submission: { videoID: any; userID: any; s
     }
 }
 
+function getYouTubeVideoDuration(apiVideoInfo: APIVideoInfo): VideoDuration {
+    const duration = apiVideoInfo?.data?.items[0]?.contentDetails?.duration;
+    return duration ? isoDurations.toSeconds(isoDurations.parse(duration)) as VideoDuration : null;
+}
+
+async function getYouTubeVideoInfo(videoID: VideoID): Promise<APIVideoInfo> {
+    if (config.youtubeAPIKey !== null) {
+        return new Promise((resolve) => {
+            YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
+        });
+    } else {
+        return null;
+    }
+}
+
 function proxySubmission(req: Request) {
     fetch(config.proxySubmission + '/api/skipSegments?userID=' + req.query.userID + '&videoID=' + req.query.videoID, {
             method: 'POST',
@@ -267,14 +308,18 @@ export async function postSkipSegments(req: Request, res: Response) {
 
     const videoID = req.query.videoID || req.body.videoID;
     let userID = req.query.userID || req.body.userID;
+    let service: Service = req.query.service ?? req.body.service ?? Service.YouTube;
+    if (!Object.values(Service).some((val) => val == service)) {
+        service = Service.YouTube;
+    }
+    let videoDuration: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration;
 
-
+    let segments = req.body.segments as IncomingSegment[];
-    let segments = req.body.segments;
     if (segments === undefined) {
         // Use query instead
         segments = [{
-            segment: [req.query.startTime, req.query.endTime],
+            segment: [req.query.startTime as string, req.query.endTime as string],
-            category: req.query.category,
+            category: req.query.category as Category
         }];
     }
 
@@ -367,16 +412,22 @@ export async function postSkipSegments(req: Request, res: Response) {
 
         //check if this info has already been submitted before
         const duplicateCheck2Row = await db.prepare('get', `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "startTime" = ?
-            and "endTime" = ? and "category" = ? and "videoID" = ?`, [startTime, endTime, segments[i].category, videoID]);
+            and "endTime" = ? and "category" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, videoID, service]);
         if (duplicateCheck2Row.count > 0) {
             res.sendStatus(409);
             return;
         }
     }
 
+    let apiVideoInfo: APIVideoInfo = null;
+    if (service == Service.YouTube) {
+        apiVideoInfo = await getYouTubeVideoInfo(videoID);
+    }
+    videoDuration = getYouTubeVideoDuration(apiVideoInfo) || videoDuration;
+
     // Auto moderator check
-    if (!isVIP) {
+    if (!isVIP && service == Service.YouTube) {
-        const autoModerateResult = await autoModerateSubmission({userID, videoID, segments});//startTime, endTime, category: segments[i].category});
+        const autoModerateResult = await autoModerateSubmission(apiVideoInfo, {userID, videoID, segments});//startTime, endTime, category: segments[i].category});
         if (autoModerateResult == "Rejected based on NeuralBlock predictions.") {
             // If NB automod rejects, the submission will start with -2 votes.
             // Note, if one submission is bad all submissions will be affected.
@@ -437,63 +488,18 @@ export async function postSkipSegments(req: Request, res: Response) {
 
         let startingVotes = 0 + decreaseVotes;
 
-        if (config.youtubeAPIKey !== null) {
-            let {err, data} = await new Promise((resolve) => {
-                YouTubeAPI.listVideos(videoID, (err: any, data: any) => resolve({err, data}));
-            });
-
-            if (err) {
-                Logger.error("Error while submitting when connecting to YouTube API: " + err);
-            } else {
-                //get all segments for this video and user
-                const allSubmittedByUser = await db.prepare('all', `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ? and "votes" > -1`, [userID, videoID]);
-                const allSegmentTimes = [];
-                if (allSubmittedByUser !== undefined) {
-                    //add segments the user has previously submitted
-                    for (const segmentInfo of allSubmittedByUser) {
-                        allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)]);
-                    }
-                }
-
-                //add segments they are trying to add in this submission
-                for (let i = 0; i < segments.length; i++) {
-                    let startTime = parseFloat(segments[i].segment[0]);
-                    let endTime = parseFloat(segments[i].segment[1]);
-                    allSegmentTimes.push([startTime, endTime]);
-                }
-
-                //merge all the times into non-overlapping arrays
-                const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function (a, b) {
-                    return a[0] - b[0] || a[1] - b[1];
-                }));
-
-                let videoDuration = data.items[0].contentDetails.duration;
-                videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
-                if (videoDuration != 0) {
-                    let allSegmentDuration = 0;
-                    //sum all segment times together
-                    allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
-                    if (allSegmentDuration > (videoDuration / 100) * 80) {
-                        // Reject submission if all segments combine are over 80% of the video
-                        res.status(400).send("Total length of your submitted segments are over 80% of the video.");
-                        return;
-                    }
-                }
-            }
-        }
-
         for (const segmentInfo of segments) {
             //this can just be a hash of the data
             //it's better than generating an actual UUID like what was used before
             //also better for duplication checking
-            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, segmentInfo.segment[0], segmentInfo.segment[1]);
+            const UUID = getSubmissionUUID(videoID, segmentInfo.category, userID, parseFloat(segmentInfo.segment[0]), parseFloat(segmentInfo.segment[1]));
 
             const startingLocked = isVIP ? 1 : 0;
             try {
                 await db.prepare('run', `INSERT INTO "sponsorTimes" 
-                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "shadowHidden", "hashedVideoID")
+                    ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", "views", "category", "service", "videoDuration", "shadowHidden", "hashedVideoID")
-                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+                    VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
-                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, shadowBanned, getHash(videoID, 1),
+                        videoID, segmentInfo.segment[0], segmentInfo.segment[1], startingVotes, startingLocked, UUID, userID, timeSubmitted, 0, segmentInfo.category, service, videoDuration, shadowBanned, getHash(videoID, 1),
                     ],
                 );
 
@@ -504,7 +510,7 @@ export async function postSkipSegments(req: Request, res: Response) {
                 redis.delAsync(skipSegmentsKey(videoID));
             } catch (err) {
                 //a DB change probably occurred
-                res.sendStatus(502);
+                res.sendStatus(500);
                 Logger.error("Error when putting sponsorTime in the DB: " + videoID + ", " + segmentInfo.segment[0] + ", " +
                     segmentInfo.segment[1] + ", " + userID + ", " + segmentInfo.category + ". " + err);
 
@@ -529,7 +535,7 @@ export async function postSkipSegments(req: Request, res: Response) {
     res.json(newSegments);
 
     for (let i = 0; i < segments.length; i++) {
-        sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
+        sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service);
     }
 }
 

src/types/segments.model.ts

@@ -3,15 +3,32 @@ import { SBRecord } from "./lib.model";
 
 export type SegmentUUID = string  & { __segmentUUIDBrand: unknown };
 export type VideoID = string & { __videoIDBrand: unknown };
+export type VideoDuration = number & { __videoDurationBrand: unknown };
 export type Category = string & { __categoryBrand: unknown };
 export type VideoIDHash = VideoID & HashedValue;
 export type IPAddress = string & { __ipAddressBrand: unknown };
 export type HashedIP = IPAddress & HashedValue;
 
+// Uncomment as needed
+export enum Service {
+    YouTube = 'YouTube',
+    // Nebula = 'Nebula',
+    PeerTube = 'PeerTube',
+    // RSS = 'RSS',
+    // Corridor = 'Corridor',
+    // Lbry = 'Lbry'
+}
+
+export interface IncomingSegment { 
+    category: Category; 
+    segment: string[]; 
+}
+
 export interface Segment { 
     category: Category; 
     segment: number[]; 
     UUID: SegmentUUID;
+    videoDuration: VideoDuration;
 }
 
 export enum Visibility {
@@ -28,6 +45,7 @@ export interface DBSegment {
     locked: boolean;
     shadowHidden: Visibility;
     videoID: VideoID;
+    videoDuration: VideoDuration;
     hashedVideoID: VideoIDHash;
 }
 

test/cases/getSkipSegments.ts

@@ -5,16 +5,17 @@ import {getHash} from '../../src/utils/getHash';
 
 describe('getSkipSegments', () => {
     before(async () => {
-        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "timeSubmitted", views, category, "service", "videoDuration", "shadowHidden", "hashedVideoID") VALUES';
-        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest', 1, 11, 2, 0, '1-uuid-0', 'testman', 0, 50, 'sponsor', 'YouTube', 100, 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 0, '" + getHash('testtesttest', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest2', 1, 11, 2, 0, '1-uuid-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 120, 0, '" + getHash('testtesttest2', 1) + "')");
-        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('testtesttest,test', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest', 20, 33, 2, 0, '1-uuid-2', 'testman', 0, 50, 'intro', 'YouTube', 101, 0, '" + getHash('testtesttest', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('testtesttest,test', 1, 11, 2, 0, '1-uuid-1', 'testman', 0, 50, 'sponsor', 'YouTube', 140, 0, '" + getHash('testtesttest,test', 1) + "')");
-        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 0, '" + getHash('test3', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 1, 11, 2, 0, '1-uuid-4', 'testman', 0, 50, 'sponsor', 'YouTube', 200, 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('test3', 7, 22, -3, 0, '1-uuid-5', 'testman', 0, 50, 'sponsor', 'YouTube', 300, 0, '" + getHash('test3', 1) + "')");
-        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 0, '" + getHash('multiple', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 1, 11, 2, 0, '1-uuid-6', 'testman', 0, 50, 'intro', 'YouTube', 400, 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
+        await db.prepare("run", startOfQuery + "('multiple', 20, 33, 2, 0, '1-uuid-7', 'testman', 0, 50, 'intro', 'YouTube', 500, 0, '" + getHash('multiple', 1) + "')");
-        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 0, '" + getHash('locked', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 33, 2, 1, '1-uuid-locked-8', 'testman', 0, 50, 'intro', 'YouTube', 230, 0, '" + getHash('locked', 1) + "')");
+        await db.prepare("run", startOfQuery + "('locked', 20, 34, 100000, 0, '1-uuid-9', 'testman', 0, 50, 'intro', 'YouTube', 190, 0, '" + getHash('locked', 1) + "')");
     
         return;
     });
@@ -27,7 +28,24 @@ describe('getSkipSegments', () => {
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
-                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
+                    return;
+                } else {
+                    return ("Received incorrect body: " + (await res.text()));
+                }
+            }
+        })
+        .catch(err => "Couldn't call endpoint");
+    });
+
+    it('Should be able to get a time by category for a different service 1', () => {
+        fetch(getbaseURL() + "/api/skipSegments?videoID=testtesttest&category=sponsor&service=PeerTube")
+        .then(async res => {
+            if (res.status !== 200) return ("Status code was: " + res.status);
+            else {
+                const data = await res.json();
+                if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0-1" && data[0].videoDuration === 120) {
                     return;
                 } else {
                     return ("Received incorrect body: " + (await res.text()));
@@ -61,7 +79,7 @@ describe('getSkipSegments', () => {
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 1 && data[0].segment[1] === 11
-                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0") {
+                    && data[0].category === "sponsor" && data[0].UUID === "1-uuid-0" && data[0].videoDuration === 100) {
                     return;
                 } else {
                     return ("Received incorrect body: " + (await res.text()));
@@ -78,7 +96,7 @@ describe('getSkipSegments', () => {
             else {
                 const data = await res.json();
                 if (data.length === 1 && data[0].segment[0] === 20 && data[0].segment[1] === 33
-                    && data[0].category === "intro" && data[0].UUID === "1-uuid-2") {
+                    && data[0].category === "intro" && data[0].UUID === "1-uuid-2" && data[0].videoDuration === 101) {
                     return;
                 } else {
                     return ("Received incorrect body: " + (await res.text()));
@@ -99,9 +117,9 @@ describe('getSkipSegments', () => {
                     let success = true;
                     for (const segment of data) {
                         if ((segment.segment[0] !== 20 || segment.segment[1] !== 33
-                            || segment.category !== "intro" || segment.UUID !== "1-uuid-7") &&
+                            || segment.category !== "intro" || segment.UUID !== "1-uuid-7" || segment.videoDuration === 500) &&
                             (segment.segment[0] !== 1 || segment.segment[1] !== 11
-                                || segment.category !== "intro" || segment.UUID !== "1-uuid-6")) {
+                                || segment.category !== "intro" || segment.UUID !== "1-uuid-6" || segment.videoDuration === 400)) {
                             success = false;
                             break;
                         }

test/cases/getSkipSegmentsByHash.ts

@@ -12,11 +12,12 @@ sinonStub.callsFake(YouTubeApiMock.listVideos);
 
 describe('getSegmentsByHash', () => {
     before(async () => {
-        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "shadowHidden", "hashedVideoID") VALUES';
+        let startOfQuery = 'INSERT INTO "sponsorTimes" ("videoID", "startTime", "endTime", "votes", "UUID", "userID", "timeSubmitted", views, category, "service", "shadowHidden", "hashedVideoID") VALUES';
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 1, 10, 2, 'getSegmentsByHash-0-0-1', 'testman', 0, 50, 'sponsor', 'PeerTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-0', 20, 30, 2, 'getSegmentsByHash-0-1', 'testman', 100, 150, 'intro', 'YouTube', 0, '" + getHash('getSegmentsByHash-0', 1) + "')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
-        await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-noMatchHash', 40, 50, 2, 'getSegmentsByHash-noMatchHash', 'testman', 0, 50, 'sponsor', 'YouTube', 0, 'fdaffnoMatchHash')"); // hash = fdaff4dee1043451faa7398324fb63d8618ebcd11bddfe0491c488db12c6c910
+        await db.prepare("run", startOfQuery + "('getSegmentsByHash-1', 60, 70, 2, 'getSegmentsByHash-1', 'testman', 0, 50, 'sponsor', 'YouTube', 0, '" + getHash('getSegmentsByHash-1', 1) + "')"); // hash = 3272fa85ee0927f6073ef6f07ad5f3146047c1abba794cfa364d65ab9921692b
     });
 
     it('Should be able to get a 200', (done: Done) => {
@@ -128,7 +129,24 @@ describe('getSegmentsByHash', () => {
                 if (body.length !== 2) done("expected 2 videos, got " + body.length);
                 else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
                 else if (body[1].segments.length !== 1) done("expected 1 segments for second video, got " + body[1].segments.length);
-                else if (body[0].segments[0].category !== 'sponsor' || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+                else if (body[0].segments[0].category !== 'sponsor' 
+                        || body[0].segments[0].UUID !== 'getSegmentsByHash-0-0' 
+                        || body[1].segments[0].category !== 'sponsor') done("both segments are not sponsor");
+                else done();
+            }
+        })
+        .catch(err => done("Couldn't call endpoint"));
+    });
+
+    it('Should be able to get 200 for no categories (default sponsor) for a non YouTube service', (done: Done) => {
+        fetch(getbaseURL() + '/api/skipSegments/fdaf?service=PeerTube')
+        .then(async res => {
+            if (res.status !== 200) done("non 200 status code, was " + res.status);
+            else {
+                const body = await res.json();
+                if (body.length !== 1) done("expected 2 videos, got " + body.length);
+                else if (body[0].segments.length !== 1) done("expected 1 segments for first video, got " + body[0].segments.length);
+                else if (body[0].segments[0].UUID !== 'getSegmentsByHash-0-0-1') done("both segments are not sponsor");
                 else done();
             }
         })

test/cases/postSkipSegments.ts

@@ -97,6 +97,102 @@ describe('postSkipSegments', () => {
         .catch(err => done(err));
     });
 
+    it('Should be able to submit a single time with a duration (JSON method)', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                userID: "test",
+                videoID: "dQw4w9WgXZX",
+                videoDuration: 100,
+                segments: [{
+                    segment: [0, 10],
+                    category: "sponsor",
+                }],
+            }),
+        })
+        .then(async res => {
+            if (res.status === 200) {
+                const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXZX"]);
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 5010) {
+                    done();
+                } else {
+                    done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should be able to submit a single time with a duration from the API (JSON method)', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                userID: "test",
+                videoID: "noDuration",
+                videoDuration: 100,
+                segments: [{
+                    segment: [0, 10],
+                    category: "sponsor",
+                }],
+            }),
+        })
+        .then(async res => {
+            if (res.status === 200) {
+                const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "videoDuration" FROM "sponsorTimes" WHERE "videoID" = ?`, ["noDuration"]);
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.videoDuration === 100) {
+                    done();
+                } else {
+                    done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
+    it('Should be able to submit a single time under a different service (JSON method)', (done: Done) => {
+        fetch(getbaseURL()
+            + "/api/postVideoSponsorTimes", {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                userID: "test",
+                videoID: "dQw4w9WgXcG",
+                service: "PeerTube",
+                segments: [{
+                    segment: [0, 10],
+                    category: "sponsor",
+                }],
+            }),
+        })
+        .then(async res => {
+            if (res.status === 200) {
+                const row = await db.prepare('get', `SELECT "startTime", "endTime", "locked", "category", "service" FROM "sponsorTimes" WHERE "videoID" = ?`, ["dQw4w9WgXcG"]);
+                if (row.startTime === 0 && row.endTime === 10 && row.locked === 0 && row.category === "sponsor" && row.service === "PeerTube") {
+                    done();
+                } else {
+                    done("Submitted times were not saved. Actual submission: " + JSON.stringify(row));
+                }
+            } else {
+                done("Status code was " + res.status);
+            }
+        })
+        .catch(err => done(err));
+    });
+
     it('VIP submission should start locked', (done: Done) => {
         fetch(getbaseURL()
             + "/api/postVideoSponsorTimes", {
@@ -244,7 +340,7 @@ describe('postSkipSegments', () => {
                 }),
             })
         .then(async res => {
-            if (res.status === 400) {
+            if (res.status === 403) {
                 const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["n9rIGdXnSJc"]);
                 let success = true;
                 if (rows.length === 4) {
@@ -292,7 +388,7 @@ describe('postSkipSegments', () => {
             }),
         })
         .then(async res => {
-            if (res.status === 400) {
+            if (res.status === 403) {
                 const rows = await db.prepare('all', `SELECT "startTime", "endTime", "category" FROM "sponsorTimes" WHERE "videoID" = ? and "votes" > -1`, ["80percent_video"]);
                 let success = rows.length == 2;
                 for (const row of rows) {