Merge pull request #18 from ajayyy/experimental

Username support

README.MD

@@ -1,6 +1,6 @@
-# SponsorBlocker Server
+# SponsorBlock Server
 
-SponsorBlocker is an extension that will skip over sponsored segments of YouTube videos. SponsorBlocker is a crowdsourced browser extension that let's anyone submit the start and end time's of sponsored segments of YouTube videos. Once one person submits this information, everyone else with this extension will skip right over the sponsored segment.
+SponsorBlock is an extension that will skip over sponsored segments of YouTube videos. SponsorBlock is a crowdsourced browser extension that let's anyone submit the start and end time's of sponsored segments of YouTube videos. Once one person submits this information, everyone else with this extension will skip right over the sponsored segment.
 
 This is the server backend for it
 
@@ -14,4 +14,182 @@ Hopefully this project can be combined with projects like [this](https://github.
 
 # Client
 
-The client web browser extension is available here: https://github.com/ajayyy/SponsorBlocker
+The client web browser extension is available here: https://github.com/ajayyy/SponsorBlock
+
+# API Docs
+
+Public API available at https://sponsor.ajay.app.
+
+________________________________________________________________________________
+
+`/api/getVideoSponsorTimes`
+
+**Input**:
+```
+{
+  videoID: string
+}
+```
+
+**Response**:
+```
+{
+  sponorTimes: array [float],
+  UUIDs: array [string] //The ID for this sponsor time, used to submit votes
+}
+```
+
+**Error codes**:
+
+404: Not Found
+
+__________________________________________________________________
+
+`/api/postVideoSponsorTimes`
+
+**Input**:
+```
+{
+  videoID: string,
+  startTime: float,
+  endTime: float,
+  userID: string //This should be a randomly generated UUID
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200)
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+429: Rate Limit (Too many for the same user or IP)
+
+409: Duplicate
+
+__________________________________________________________________
+
+`/api/voteOnSponsorTime`
+
+**Input**:
+```
+{
+  UUID: string, //id of the sponsor being voted on
+  userID: string,
+  type: int //0 for downvote, 1 for upvote
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200)
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+405: Duplicate
+
+__________________________________________________________________
+
+`/api/viewedVideoSponsorTime`
+
+**Input**:
+```
+{
+  UUID: string
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+__________________________________________________________________
+
+`/api/getViewsForUser`
+
+**Input**:
+```
+{
+  userID: string
+}
+```
+
+**Response**:
+```
+{
+  viewCount: int
+}
+```
+
+**Error codes**:
+
+404: Not Found
+
+__________________________________________________________________
+
+### Stats Calls
+
+`/api/getTopUsers`
+
+**Input**:
+```
+{
+  sortType: int //0 for by minutes saved, 1 for by view count, 2 for by total submissions
+}
+```
+
+**Response**:
+```
+{
+  userNames: array [string],
+  viewCounts: array [int],
+  totalSubmissions: array [int],
+  minutesSaved: array [float]
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+__________________________________________________________________
+
+`/api/getTotalStats`
+
+**Input**:
+```
+{
+  Nothing
+}
+```
+
+**Response**:
+```
+{
+  userCount: int,
+  viewCount: int,
+  totalSubmissions: int,
+  minutesSaved: float
+}
+```
+
+**Error codes**:
+
+None

index.js

@@ -1,18 +1,16 @@
 var express = require('express');
 var http = require('http');
-
 // Create a service (the app object is just a callback).
 var app = express();
 
-//uuid service
-var uuidv1 = require('uuid/v1');
-
 //hashing service
 var crypto = require('crypto');
 
 //load database
 var sqlite3 = require('sqlite3').verbose();
 var db = new sqlite3.Database('./databases/sponsorTimes.db');
+//where the more sensitive data such as IP addresses are stored
+var privateDB = new sqlite3.Database('./databases/private.db');
 
 // Create an HTTP service.
 http.createServer(app).listen(80);
@@ -21,6 +19,9 @@ http.createServer(app).listen(80);
 //  make it even harder for someone to decode the ip
 var globalSalt = "49cb0d52-1aec-4b89-85fc-fab2c53062fb";
 
+//if so, it will use the x-forwarded header instead of the ip address of the connection
+var behindProxy = true;
+
 //setup CORS correctly
 app.use(function(req, res, next) {
     res.header("Access-Control-Allow-Origin", "*");
@@ -36,12 +37,12 @@ app.get('/api/getVideoSponsorTimes', function (req, res) {
     let votes = []
     let UUIDs = [];
 
-    db.prepare("SELECT startTime, endTime, votes, UUID FROM sponsorTimes WHERE videoID = ?").all(videoID, function(err, rows) {
+    db.prepare("SELECT startTime, endTime, votes, UUID FROM sponsorTimes WHERE videoID = ? ORDER BY startTime").all(videoID, function(err, rows) {
         if (err) console.log(err);
 
         for (let i = 0; i < rows.length; i++) {
-            //check if votes are above -2
-            if (rows[i].votes < -2) {
+            //check if votes are above -1
+            if (rows[i].votes < -1) {
                 //too untrustworthy, just ignore it
                 continue;
             }
@@ -77,6 +78,10 @@ app.get('/api/getVideoSponsorTimes', function (req, res) {
     });
 });
 
+function getIP(req) {
+    return behindProxy ? req.headers['x-forwarded-for'] : req.connection.remoteAddress;
+}
+
 //add the post function
 app.get('/api/postVideoSponsorTimes', function (req, res) {
     let videoID = req.query.videoID;
@@ -84,44 +89,74 @@ app.get('/api/postVideoSponsorTimes', function (req, res) {
     let endTime = req.query.endTime;
     let userID = req.query.userID;
 
-    if (typeof videoID != 'string' || startTime == undefined || endTime == undefined || userID == undefined) {
+    //check if all correct inputs are here and the length is 1 second or more
+    if (videoID == undefined || startTime == undefined || endTime == undefined || userID == undefined
+            || Math.abs(startTime - endTime) < 1) {
         //invalid request
         res.sendStatus(400);
         return;
     }
 
-    //x-forwarded-for if this server is behind a proxy
-    let ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
-
-    //hash the ip so no one can get it from the database
-    let hashCreator = crypto.createHash('sha256');
-    let hashedIP = hashCreator.update(ip + globalSalt).digest('hex');
+    //hash the userID
+    userID = getHash(userID);
+    
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(getIP(req) + globalSalt);
 
     startTime = parseFloat(startTime);
     endTime = parseFloat(endTime);
 
-    let UUID = uuidv1();
+    if (isNaN(startTime) || isNaN(endTime)) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    if (startTime > endTime) {
+        //time can't go backwards
+        res.sendStatus(400);
+        return;
+    }
+
+    //this can just be a hash of the data
+    //it's better than generating an actual UUID like what was used before
+    //also better for duplication checking
+    let hashCreator = crypto.createHash('sha256');
+    let UUID = hashCreator.update(videoID + startTime + endTime + userID).digest('hex');
 
     //get current time
     let timeSubmitted = Date.now();
 
-    //check to see if the user has already submitted sponsors for this video
-    db.prepare("SELECT UUID FROM sponsorTimes WHERE userID = ? and videoID = ?").all([userID, videoID], function(err, rows) {
-        if (rows.length >= 4) {
-            //too many sponsors for the same video from the same user
+    let yesterday = timeSubmitted - 86400000;
+    
+    //check to see if this ip has submitted too many sponsors today
+    privateDB.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday], function(err, row) {
+        if (row.count >= 10) {
+            //too many sponsors for the same video from the same ip address
             res.sendStatus(429);
         } else {
-            //check if this info has already been submitted first
-            db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get([startTime, endTime, videoID], function(err, row) {
-                if (err) console.log(err);
-                
-                if (row == null) {
-                    //not a duplicate, execute query
-                    db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, 0, UUID, userID, hashedIP, timeSubmitted);
-
-                    res.sendStatus(200);
+            //check to see if the user has already submitted sponsors for this video
+            db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID], function(err, row) {
+                if (row.count >= 8) {
+                    //too many sponsors for the same video from the same user
+                    res.sendStatus(429);
                 } else {
-                    res.sendStatus(409);
+                    //check if this info has already been submitted first
+                    db.prepare("SELECT UUID FROM sponsorTimes WHERE startTime = ? and endTime = ? and videoID = ?").get([startTime, endTime, videoID], function(err, row) {
+                        if (err) console.log(err);
+                        
+                        if (row == null) {
+                            //not a duplicate, execute query
+                            db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, 0, UUID, userID, timeSubmitted, 0);
+
+                            //add to private db as well
+                            privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted);
+
+                            res.sendStatus(200);
+                        } else {
+                            res.sendStatus(409);
+                        }
+                    });
                 }
             });
         }
@@ -140,37 +175,244 @@ app.get('/api/voteOnSponsorTime', function (req, res) {
         return;
     }
 
-    //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
-    let incrementAmount = 0;
+    //hash the userID
+    userID = getHash(userID + UUID);
 
-    //don't use userID for now, and just add the vote
-    if (type == 1) {
-        //upvote
-        incrementAmount = 1;
-    } else if (type == 0) {
-        //downvote
-        incrementAmount = -1;
-    } else {
-        //unrecongnised type of vote
-        req.sendStatus(400);
+    //x-forwarded-for if this server is behind a proxy
+    let ip = getIP(req);
+
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(ip + globalSalt);
+
+    //check if vote has already happened
+    privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID, function(err, row) {
+        if (err) console.log(err);
+                
+        if (row != undefined && row.type == type) {
+            //they have already done this exact vote
+            res.status(405).send("Duplicate Vote");
+            return;
+        }
+
+        //-1 for downvote, 1 for upvote. Maybe more depending on reputation in the future
+        let incrementAmount = 0;
+        let oldIncrementAmount = 0;
+
+        if (type == 1) {
+            //upvote
+            incrementAmount = 1;
+        } else if (type == 0) {
+            //downvote
+            incrementAmount = -1;
+        } else {
+            //unrecongnised type of vote
+            res.sendStatus(400);
+            return;
+        }
+        if (row != undefined) {
+            if (row.type == 1) {
+                //upvote
+                oldIncrementAmount = 1;
+            } else if (row.type == 0) {
+                //downvote
+                oldIncrementAmount = -1;
+            }
+        }
+
+        //update the votes table
+        if (row != undefined) {
+            privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID);
+        } else {
+            privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type);
+        }
+
+        //update the vote count on this sponsorTime
+        //oldIncrementAmount will be zero is row is null
+        db.prepare("UPDATE sponsorTimes SET votes = votes + ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID);
+
+        //added to db
+        res.sendStatus(200);
+    });
+});
+
+//Endpoint when a sponsorTime is used up
+app.get('/api/viewedVideoSponsorTime', function (req, res) {
+    let UUID = req.query.UUID;
+
+    if (UUID == undefined) {
+        //invalid request
+        res.sendStatus(400);
         return;
     }
 
-    db.prepare("UPDATE sponsorTimes SET votes = votes + ? WHERE UUID = ?").run(incrementAmount, UUID);
+    //up the view count by one
+    db.prepare("UPDATE sponsorTimes SET views = views + 1 WHERE UUID = ?").run(UUID);
 
-    //added to db
     res.sendStatus(200);
 });
 
+//To set your username for the stats view
+app.post('/api/setUsername', function (req, res) {
+    let userID = req.query.userID;
+    let userName = req.query.username;
+
+    if (userID == undefined || userName == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    //check if username is already set
+    db.prepare("SELECT count(*) as count FROM userNames WHERE userID = ?").get(userID, function(err, row) {
+        if (err) console.log(err);
+                
+        if (row.count > 0) {
+            //already exists, update this row
+            db.prepare("UPDATE userNames SET userName = ? WHERE userID = ?").run(userName, userID);
+        } else {
+            //add to the db
+            db.prepare("INSERT INTO userNames VALUES(?, ?)").run(userID, userName);
+        }
+
+        res.sendStatus(200);
+    });
+});
+
+//get what username this user has
+app.get('/api/getUsername', function (req, res) {
+    let userID = req.query.userID;
+
+    if (userID == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    db.prepare("SELECT userName FROM userNames WHERE userID = ?").get(userID, function(err, row) {
+        if (err) console.log(err);
+        
+        if (row != null) {
+            res.send({
+                userName: row.userName
+            });
+        } else {
+            //no username yet, just send back the userID
+            res.send({
+                userName: userID
+            });
+        }
+    });
+});
+
+//Gets all the views added up for one userID
+//Useful to see how much one user has contributed
+app.get('/api/getViewsForUser', function (req, res) {
+    let userID = req.query.userID;
+
+    if (userID == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //hash the userID
+    userID = getHash(userID);
+
+    //up the view count by one
+    db.prepare("SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ?").get(userID, function(err, row) {
+        if (err) console.log(err);
+
+        if (row.viewCount != null) {
+            res.send({
+                viewCount: row.viewCount
+            });
+        } else {
+            res.sendStatus(404);
+        }
+    });
+});
+
+app.get('/api/getTopUsers', function (req, res) {
+    let sortType = req.query.sortType;
+
+    if (sortType == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //setup which sort type to use
+    let sortBy = "";
+    if (sortType == 0) {
+        sortBy = "minutesSaved";
+    } else if (sortType == 1) {
+        sortBy = "viewCount";
+    } else if (sortType == 2) {
+        sortBy = "totalSubmissions";
+    } else {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    let userNames = [];
+    let viewCounts = [];
+    let totalSubmissions = [];
+    let minutesSaved = [];
+
+    db.prepare("SELECT sponsorTimes.userID as userID, COUNT(*) as totalSubmissions, SUM(views) as viewCount, SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, userNames.userName as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID WHERE sponsorTimes.votes > -1 GROUP BY sponsorTimes.userID ORDER BY " + sortBy + " DESC LIMIT 50").all(function(err, rows) {
+        for (let i = 0; i < rows.length; i++) {
+            if (rows[i].userName != null) {
+                userNames[i] = rows[i].userName;
+            } else {
+                userNames[i] = rows[i].userID;
+            }
+
+            viewCounts[i] = rows[i].viewCount;
+            totalSubmissions[i] = rows[i].totalSubmissions;
+            minutesSaved[i] = rows[i].minutesSaved;
+        }
+
+        //send this result
+        res.send({
+            userNames: userNames,
+            viewCounts: viewCounts,
+            totalSubmissions: totalSubmissions,
+            minutesSaved: minutesSaved
+        });
+    });
+});
+
+//send out totals
+//send the total submissions, total views and total minutes saved
+app.get('/api/getTotalStats', function (req, res) {
+    db.prepare("SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes").get(function(err, row) {
+        if (row != null) {
+            //send this result
+            res.send({
+                userCount: row.userCount,
+                viewCount: row.viewCount,
+                totalSubmissions: row.totalSubmissions,
+                minutesSaved: row.minutesSaved
+            });
+        }
+    });
+});
+
 app.get('/database.db', function (req, res) {
     res.sendFile("./databases/sponsorTimes.db", { root: __dirname });
 });
 
-
 //This function will find sponsor times that are contained inside of eachother, called similar sponsor times
 //Only one similar time will be returned, randomly generated based on the sqrt of votes.
 //This allows new less voted items to still sometimes appear to give them a chance at getting votes.
-//Sponsor times with less than -2 votes are already ignored before this function is called
+//Sponsor times with less than -1 votes are already ignored before this function is called
 function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
     //list of sponsors that are contained inside eachother
     let similarSponsors = [];
@@ -327,6 +569,7 @@ function getWeightedRandomChoice(choices, weights, amountOfChoices) {
 
     //iterate and find amountOfChoices choices
     let randomNumber = Math.random();
+    
     //this array will keep adding to this variable each time one sqrt vote has been dealt with
     //this is the sum of all the sqrtVotes under this index
     let currentVoteNumber = 0;
@@ -356,4 +599,13 @@ function getWeightedRandomChoice(choices, weights, amountOfChoices) {
         finalChoices: finalChoices,
         choicesDealtWith: choicesDealtWith
     };
-}
+}
+
+function getHash(value, times=5000) {
+    for (let i = 0; i < times; i++) {
+        let hashCreator = crypto.createHash('sha256');
+        value = hashCreator.update(value).digest('hex');
+    }
+
+    return value;
+}