Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into nb-mod-fetch

# Conflicts:
#	src/routes/postSkipSegments.js
#	test/mocks.js

config.json.example

@@ -18,5 +18,6 @@
     "dbSchema": "./databases/_sponsorTimes.db.sql",
     "privateDBSchema": "./databases/_private.db.sql",
     "mode": "development",
-    "readOnly": false
+    "readOnly": false,
+    "webhooks": []
 }

src/routes/getTopUsers.js

@@ -47,7 +47,7 @@ module.exports = function getTopUsers (req, res) {
                   "IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " +
                   "LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " +
                   "WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " +
-                  "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 50 " +
+                  "GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " +
                   "ORDER BY " + sortBy + " DESC LIMIT 100", []);
   
   for (let i = 0; i < rows.length; i++) {

src/routes/postSkipSegments.js

@@ -13,15 +13,42 @@ const getHash = require('../utils/getHash.js');
 const getIP = require('../utils/getIP.js');
 const getFormattedTime = require('../utils/getFormattedTime.js');
 const isUserTrustworthy = require('../utils/isUserTrustworthy.js')
+const { dispatchEvent } = require('../utils/webhookUtils.js');
 
-function sendDiscordNotification(userID, videoID, UUID, segmentInfo) {
+function sendWebhookNotification(userID, videoID, UUID, submissionCount, youtubeData, {submissionStart, submissionEnd}, segmentInfo) {
-    //check if they are a first time user
+    let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]);
-    //if so, send a notification to discord
+    let userName = row !== undefined ? row.userName : null;
-    if (config.youtubeAPIKey !== null && config.discordFirstTimeSubmissionsWebhookURL !== null) {
+    let video = youtubeData.items[0];
+
+    let scopeName = "submissions.other";
+    if (submissionCount <= 1) {
+        scopeName = "submissions.new";
+    }
+
+    dispatchEvent(scopeName, {
+        "video": {
+            "id": videoID,
+            "title": video.snippet.title,
+            "thumbnail": video.snippet.thumbnails.maxres ? video.snippet.thumbnails.maxres : null,
+            "url": "https://www.youtube.com/watch?v=" + videoID
+        },
+        "submission": {
+            "UUID": UUID,
+            "category": segmentInfo.category,
+            "startTime": submissionStart,
+            "endTime": submissionEnd,
+            "user": {
+                "UUID": userID,
+                "username": userName
+            }
+        }
+    });
+}
+
+function sendWebhooks(userID, videoID, UUID, segmentInfo) {
+    if (config.youtubeAPIKey !== null) {
         let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]);
 
-        // If it is a first time submission
-        if (userSubmissionCountRow.submissionCount <= 1) {
         YouTubeAPI.videos.list({
             part: "snippet",
             id: videoID
@@ -33,7 +60,11 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) {
             
             let startTime = parseFloat(segmentInfo.segment[0]);
             let endTime = parseFloat(segmentInfo.segment[1]);
+            sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {submissionStart: startTime, submissionEnd: endTime}, segmentInfo);
             
+            // If it is a first time submission
+            // Then send a notification to discord
+            if (config.discordFirstTimeSubmissionsWebhookURL === null) return;
             request.post(config.discordFirstTimeSubmissionsWebhookURL, {
                 json: {
                     "embeds": [{
@@ -66,7 +97,6 @@ function sendDiscordNotification(userID, videoID, UUID, segmentInfo) {
         });
     }
 }
-}
 
 // callback:  function(reject: "String containing reason the submission was rejected")
 // returns: string when an error, false otherwise
@@ -216,6 +246,9 @@ module.exports = async function postSkipSegments(req, res) {
         }
     }
 
+    // Will be filled when submitting
+    let UUIDs = [];
+
     try {
         //get current time
         let timeSubmitted = Date.now();
@@ -289,7 +322,9 @@ module.exports = async function postSkipSegments(req, res) {
             }
 
             // Discord notification
-            sendDiscordNotification(userID, videoID, UUID, segmentInfo);
+            sendWebhooks(userID, videoID, UUID, segmentInfo);
+    
+            UUIDs.push(UUID);
         }
     } catch (err) {
         logger.error(err);
@@ -300,4 +335,8 @@ module.exports = async function postSkipSegments(req, res) {
     }
 
     res.sendStatus(200);
+
+    for (let i = 0; i < segments.length; i++) {
+        sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
+    }
 }

src/routes/setUsername.js

@@ -12,7 +12,7 @@ module.exports = function setUsername(req, res) {
 
   let adminUserIDInput = req.query.adminUserID;
 
-  if (userID == undefined || userName == undefined || userID === "undefined") {
+  if (userID == undefined || userName == undefined || userID === "undefined" || username.length > 50) {
       //invalid request
       res.sendStatus(400);
       return;

src/routes/voteOnSponsorTime.js

@@ -4,7 +4,8 @@ var config = require('../config.js');
 var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 var getFormattedTime = require('../utils/getFormattedTime.js');
-var isUserTrustworthy = require('../utils/isUserTrustworthy.js')
+var isUserTrustworthy = require('../utils/isUserTrustworthy.js');
+const {getVoteAuthor, getVoteAuthorRaw, dispatchEvent} = require('../utils/webhookUtils.js');
 
 var databases = require('../databases/databases.js');
 var db = databases.db;
@@ -13,16 +14,124 @@ var YouTubeAPI = require('../utils/youtubeAPI.js');
 var request = require('request');
 const logger = require('../utils/logger.js');
 
-function getVoteAuthor(submissionCount, isVIP, isOwnSubmission) {
+const voteTypes = {
-    if (submissionCount === 0) {
+    normal: 0,
-        return "Report by New User";
+    incorrect: 1
-    } else if (isVIP) {
-        return "Report by VIP User";
-    } else if (isOwnSubmission) {
-        return "Report by Submitter";
 }
 
-    return "";
+/**
+ * @param {Object} voteData
+ * @param {string} voteData.UUID
+ * @param {string} voteData.nonAnonUserID
+ * @param {number} voteData.voteTypeEnum
+ * @param {boolean} voteData.isVIP
+ * @param {boolean} voteData.isOwnSubmission
+ * @param voteData.row
+ * @param {string} voteData.category
+ * @param {number} voteData.incrementAmount
+ * @param {number} voteData.oldIncrementAmount
+ */
+function sendWebhooks(voteData) {
+    let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " +
+        "(select count(1) from sponsorTimes where userID = s.userID) count, " +
+        "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " +
+        "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?",
+    [voteData.UUID]);
+
+    let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [voteData.nonAnonUserID]);
+
+    if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) {
+        let webhookURL = null;
+        if (voteData.voteTypeEnum === voteTypes.normal) {
+            webhookURL = config.discordReportChannelWebhookURL;
+        } else if (voteData.voteTypeEnum === voteTypes.incorrect) {
+            webhookURL = config.discordCompletelyIncorrectReportWebhookURL;
+        }
+
+        if (config.youtubeAPIKey !== null) {
+            YouTubeAPI.videos.list({
+                part: "snippet",
+                id: submissionInfoRow.videoID
+            }, function (err, data) {
+                if (err || data.items.length === 0) {
+                    err && logger.error(err);
+                    return;
+                }
+                let isUpvote = voteData.incrementAmount > 0;
+                // Send custom webhooks
+                dispatchEvent(isUpvote ? "vote.up" : "vote.down", {
+                    "user": {
+                        "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission)
+                    },
+                    "video": {
+                        "id": submissionInfoRow.videoID,
+                        "title": data.items[0].snippet.title,
+                        "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID,
+                        "thumbnail": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : ""
+                    },
+                    "submission": {
+                        "UUID": voteData.UUID,
+                        "views": voteData.row.views,
+                        "category": voteData.category,
+                        "startTime": submissionInfoRow.startTime,
+                        "endTime": submissionInfoRow.endTime,
+                        "user": {
+                            "UUID": submissionInfoRow.userID,
+                            "username": submissionInfoRow.userName,
+                            "submissions": {
+                                "total": submissionInfoRow.count,
+                                "ignored": submissionInfoRow.disregarded
+                            }
+                        }
+                    },
+                    "votes": {
+                        "before": voteData.row.votes,
+                        "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)
+                    }
+                });
+                
+                // Send discord message
+                if (webhookURL !== null && !isUpvote) {
+                    request.post(webhookURL, {
+                        json: {
+                            "embeds": [{
+                                "title": data.items[0].snippet.title,
+                                "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID 
+                                    + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
+                                "description": "**" + voteData.row.votes + " Votes Prior | " + 
+                                    (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount) + " Votes Now | " + voteData.row.views 
+                                    + " Views**\n\n**Submission ID:** " + voteData.UUID 
+                                    + "\n**Category:** " + submissionInfoRow.category
+                                    + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID 
+                                    + "\n\n**Total User Submissions:** "+submissionInfoRow.count
+                                    + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded
+                                    +"\n\n**Timestamp:** " + 
+                                        getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
+                                "color": 10813440,
+                                "author": {
+                                    "name": getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isVIP, voteData.isOwnSubmission)
+                                },
+                                "thumbnail": {
+                                    "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
+                                }
+                            }]
+                        }
+                    }, (err, res) => {
+                        if (err) {
+                            logger.error("Failed to send reported submission Discord hook.");
+                            logger.error(JSON.stringify(err));
+                            logger.error("\n");
+                        } else if (res && res.statusCode >= 400) {
+                            logger.error("Error sending reported submission Discord hook");
+                            logger.error(JSON.stringify(res));
+                            logger.error("\n");
+                        }
+                    });
+                }
+
+            });
+        }
+    }
 }
 
 function categoryVote(UUID, userID, isVIP, category, hashedIP, res) {
@@ -126,11 +235,6 @@ async function voteOnSponsorTime(req, res) {
         }
     }
 
-    let voteTypes = {
-        normal: 0,
-        incorrect: 1
-    }
-
     let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect;
 
     try {
@@ -194,74 +298,6 @@ async function voteOnSponsorTime(req, res) {
             }
         }
 
-        // Send discord message
-        if (incrementAmount < 0) {
-            // Get video ID
-            let submissionInfoRow = db.prepare('get', "SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " +
-                "(select count(1) from sponsorTimes where userID = s.userID) count, " +
-                "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " +
-                "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?",
-            [UUID]);
-
-            let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [nonAnonUserID]);
-
-            if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) {
-                let webhookURL = null;
-                if (voteTypeEnum === voteTypes.normal) {
-                    webhookURL = config.discordReportChannelWebhookURL;
-                } else if (voteTypeEnum === voteTypes.incorrect) {
-                    webhookURL = config.discordCompletelyIncorrectReportWebhookURL;
-                }
-    
-                if (config.youtubeAPIKey !== null && webhookURL !== null) {
-                    YouTubeAPI.videos.list({
-                        part: "snippet",
-                        id: submissionInfoRow.videoID
-                    }, function (err, data) {
-                        if (err || data.items.length === 0) {
-                            err && logger.error(err);
-                            return;
-                        }
-                        
-                        request.post(webhookURL, {
-                            json: {
-                                "embeds": [{
-                                    "title": data.items[0].snippet.title,
-                                    "url": "https://www.youtube.com/watch?v=" + submissionInfoRow.videoID 
-                                      + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
-                                    "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views 
-                                        + " Views**\n\n**Submission ID:** " + UUID 
-                                        + "\n**Category:** " + submissionInfoRow.category
-                                        + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID 
-                                        + "\n\n**Total User Submissions:** "+submissionInfoRow.count
-                                        + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded
-                                        +"\n\n**Timestamp:** " + 
-                                            getFormattedTime(submissionInfoRow.startTime) + " to " + getFormattedTime(submissionInfoRow.endTime),
-                                    "color": 10813440,
-                                    "author": {
-                                        "name": getVoteAuthor(userSubmissionCountRow.submissionCount, isVIP, isOwnSubmission)
-                                    },
-                                    "thumbnail": {
-                                        "url": data.items[0].snippet.thumbnails.maxres ? data.items[0].snippet.thumbnails.maxres.url : "",
-                                    }
-                                }]
-                            }
-                        }, (err, res) => {
-                            if (err) {
-                                logger.error("Failed to send reported submission Discord hook.");
-                                logger.error(JSON.stringify(err));
-                                logger.error("\n");
-                            } else if (res && res.statusCode >= 400) {
-                                logger.error("Error sending reported submission Discord hook");
-                                logger.error(JSON.stringify(res));
-                                logger.error("\n");
-                            }
-                        });
-                    });
-                }
-            }
-        }
-
         // Only change the database if they have made a submission before and haven't voted recently
         let ableToVote = isVIP 
                         || (db.prepare("get", "SELECT userID FROM sponsorTimes WHERE userID = ?", [nonAnonUserID]) !== undefined
@@ -314,6 +350,18 @@ async function voteOnSponsorTime(req, res) {
         }
 
         res.sendStatus(200);
+
+        sendWebhooks({
+            UUID,
+            nonAnonUserID,
+            voteTypeEnum,
+            isVIP,
+            isOwnSubmission,
+            row,
+            category,
+            incrementAmount,
+            oldIncrementAmount
+        });
     } catch (err) {
         logger.error(err);
 

src/utils/logger.js

@@ -45,6 +45,8 @@ const settings = {
 if (config.mode === 'development') {
   settings.INFO = true;
   settings.DEBUG = true;
+} else if (config.mode === 'test') {
+    settings.WARN = false;
 }
 
 function log(level, string) {

src/utils/webhookUtils.js

@@ -0,0 +1,52 @@
+const config = require('../config.js');
+const logger = require('../utils/logger.js');
+const request = require('request');
+
+function getVoteAuthorRaw(submissionCount, isVIP, isOwnSubmission) {
+    if (isOwnSubmission) {
+        return "self";
+    } else if (isVIP) {
+        return "vip";
+    } else if (submissionCount === 0) {
+        return "new";
+    } else {
+        return "other";
+    };
+};
+
+function getVoteAuthor(submissionCount, isVIP, isOwnSubmission) {
+    if (submissionCount === 0) {
+        return "Report by New User";
+    } else if (isVIP) {
+        return "Report by VIP User";
+    } else if (isOwnSubmission) {
+        return "Report by Submitter";
+    }
+
+    return "";
+}
+
+function dispatchEvent(scope, data) {
+    let webhooks = config.webhooks;
+    if (webhooks === undefined || webhooks.length === 0) return;
+    logger.debug("Dispatching webhooks");
+    webhooks.forEach(webhook => {
+        let webhookURL = webhook.url;
+        let authKey = webhook.key;
+        let scopes = webhook.scopes || [];
+        if (!scopes.includes(scope.toLowerCase())) return;
+        request.post(webhookURL, {json: data, headers: {
+            "Authorization": authKey,
+            "Event-Type": scope // Maybe change this in the future? 
+        }}).on('error', (e) => {
+            logger.warn('Couldn\'t send webhook to ' + webhook.url);
+            logger.warn(e);
+        });
+    });
+}
+
+module.exports = {
+    getVoteAuthorRaw,
+    getVoteAuthor,
+    dispatchEvent
+}

test.json

@@ -16,5 +16,36 @@
     "dbSchema": "./databases/_sponsorTimes.db.sql",
     "privateDBSchema": "./databases/_private.db.sql",
     "mode": "test",
-    "readOnly": false
+    "readOnly": false,
+    "webhooks": [
+        {
+            "url": "http://127.0.0.1:8081/CustomWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8081/FailedWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://127.0.0.1:8099/WrongPort",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }, {
+            "url": "http://unresolvable.host:8081/FailedWebhook",
+            "key": "superSecretKey",
+            "scopes": [
+                "vote.up",
+                "vote.down"
+            ]
+        }
+    ]
 }

test/mocks.js

@@ -15,6 +15,8 @@ app.post('/CompletelyIncorrectReportWebhook', (req, res) => {
   res.sendStatus(200);
 });
 
+// Testing NeuralBlock
+
 app.get('/NeuralBlock/api/checkSponsorSegments', (req, res) => {
   if (req.query.vid === "LevkAjUE6d4") {
     res.json({
@@ -36,6 +38,11 @@ app.get('/NeuralBlock/api/getSponsorSegments', (req, res) => {
   res.sendStatus(500);
 });
 
+// Testing webhooks
+app.post('/CustomWebhook', (req, res) => {
+  res.sendStatus(200);
+});
+
 module.exports = function createMockServer(callback) {
   return app.listen(config.mockPort, callback);
 }