Merge pull request #17 from ajayyy/experimental

Changed limits and better stats

README.MD

@@ -15,3 +15,181 @@ Hopefully this project can be combined with projects like [this](https://github.
 # Client
 
 The client web browser extension is available here: https://github.com/ajayyy/SponsorBlock
+
+# API Docs
+
+Public API available at https://sponsor.ajay.app.
+
+________________________________________________________________________________
+
+`/api/getVideoSponsorTimes`
+
+**Input**:
+```
+{
+  videoID: string
+}
+```
+
+**Response**:
+```
+{
+  sponorTimes: array [float],
+  UUIDs: array [string] //The ID for this sponsor time, used to submit votes
+}
+```
+
+**Error codes**:
+
+404: Not Found
+
+__________________________________________________________________
+
+`/api/postVideoSponsorTimes`
+
+**Input**:
+```
+{
+  videoID: string,
+  startTime: float,
+  endTime: float,
+  userID: string //This should be a randomly generated UUID
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200)
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+429: Rate Limit (Too many for the same user or IP)
+
+409: Duplicate
+
+__________________________________________________________________
+
+`/api/voteOnSponsorTime`
+
+**Input**:
+```
+{
+  UUID: string, //id of the sponsor being voted on
+  userID: string,
+  type: int //0 for downvote, 1 for upvote
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200)
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+405: Duplicate
+
+__________________________________________________________________
+
+`/api/viewedVideoSponsorTime`
+
+**Input**:
+```
+{
+  UUID: string
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+__________________________________________________________________
+
+`/api/getViewsForUser`
+
+**Input**:
+```
+{
+  userID: string
+}
+```
+
+**Response**:
+```
+{
+  viewCount: int
+}
+```
+
+**Error codes**:
+
+404: Not Found
+
+__________________________________________________________________
+
+### Stats Calls
+
+`/api/getTopUsers`
+
+**Input**:
+```
+{
+  sortType: int //0 for by minutes saved, 1 for by view count, 2 for by total submissions
+}
+```
+
+**Response**:
+```
+{
+  userNames: array [string],
+  viewCounts: array [int],
+  totalSubmissions: array [int],
+  minutesSaved: array [float]
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+__________________________________________________________________
+
+`/api/getTotalStats`
+
+**Input**:
+```
+{
+  Nothing
+}
+```
+
+**Response**:
+```
+{
+  userCount: int,
+  viewCount: int,
+  totalSubmissions: int,
+  minutesSaved: float
+}
+```
+
+**Error codes**:
+
+None

index.js

@@ -1,25 +1,26 @@
 var express = require('express');
 var http = require('http');
-
 // Create a service (the app object is just a callback).
 var app = express();
 
-//uuid service
-var uuidv1 = require('uuid/v1');
-
 //hashing service
 var crypto = require('crypto');
 
 //load database
 var sqlite3 = require('sqlite3').verbose();
 var db = new sqlite3.Database('./databases/sponsorTimes.db');
+//where the more sensitive data such as IP addresses are stored
+var privateDB = new sqlite3.Database('./databases/private.db');
 
 // Create an HTTP service.
 http.createServer(app).listen(80);
 
 //global salt that is added to every ip before hashing to
 //  make it even harder for someone to decode the ip
-var globalSalt = "49cb0d52-1aec-4b89-85fc-fab2c53062fb";
+var globalSalt = "49cb0d52-1aec-4b89-85fc-fab2c53062fb"; // Should not be global
+
+//if so, it will use the x-forwarded header instead of the ip address of the connection
+var behindProxy = true;
 
 //setup CORS correctly
 app.use(function(req, res, next) {
@@ -40,8 +41,8 @@ app.get('/api/getVideoSponsorTimes', function (req, res) {
         if (err) console.log(err);
 
         for (let i = 0; i < rows.length; i++) {
-            //check if votes are above -2
-            if (rows[i].votes < -2) {
+            //check if votes are above -1
+            if (rows[i].votes < -1) {
                 //too untrustworthy, just ignore it
                 continue;
             }
@@ -77,6 +78,10 @@ app.get('/api/getVideoSponsorTimes', function (req, res) {
     });
 });
 
+function getIP(req) {
+    return behindProxy ? req.headers['x-forwarded-for'] : req.connection.remoteAddress;
+}
+
 //add the post function
 app.get('/api/postVideoSponsorTimes', function (req, res) {
     let videoID = req.query.videoID;
@@ -92,21 +97,32 @@ app.get('/api/postVideoSponsorTimes', function (req, res) {
         return;
     }
 
-    //x-forwarded-for if this server is behind a proxy
-    let ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
-
-    //hash the ip so no one can get it from the database
-    let hashedIP = ip + globalSalt;
-    //hash it 5000 times, this makes it very hard to brute force
-    for (let i = 0; i < 5000; i++) {
-        let hashCreator = crypto.createHash('sha512');
-        hashedIP = hashCreator.update(hashedIP).digest('hex');
-    }
+    //hash the userID
+    userID = getHash(userID);
+    
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(getIP(req) + globalSalt);
 
     startTime = parseFloat(startTime);
     endTime = parseFloat(endTime);
 
-    let UUID = uuidv1();
+    if (isNaN(startTime) || isNaN(endTime)) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    if (startTime > endTime) {
+        //time can't go backwards
+        res.sendStatus(400);
+        return;
+    }
+
+    //this can just be a hash of the data
+    //it's better than generating an actual UUID like what was used before
+    //also better for duplication checking
+    let hashCreator = crypto.createHash('sha256');
+    let UUID = hashCreator.update(videoID + startTime + endTime + userID).digest('hex');
 
     //get current time
     let timeSubmitted = Date.now();
@@ -114,14 +130,14 @@ app.get('/api/postVideoSponsorTimes', function (req, res) {
     let yesterday = timeSubmitted - 86400000;
     
     //check to see if this ip has submitted too many sponsors today
-    db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday], function(err, row) {
+    privateDB.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE hashedIP = ? AND videoID = ? AND timeSubmitted > ?").get([hashedIP, videoID, yesterday], function(err, row) {
         if (row.count >= 10) {
             //too many sponsors for the same video from the same ip address
             res.sendStatus(429);
         } else {
             //check to see if the user has already submitted sponsors for this video
             db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID], function(err, row) {
-                if (row.count >= 4) {
+                if (row.count >= 8) {
                     //too many sponsors for the same video from the same user
                     res.sendStatus(429);
                 } else {
@@ -131,7 +147,10 @@ app.get('/api/postVideoSponsorTimes', function (req, res) {
                         
                         if (row == null) {
                             //not a duplicate, execute query
-                            db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, 0, UUID, userID, hashedIP, timeSubmitted, 0);
+                            db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, 0, UUID, userID, timeSubmitted, 0);
+
+                            //add to private db as well
+                            privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted);
 
                             res.sendStatus(200);
                         } else {
@@ -156,8 +175,17 @@ app.get('/api/voteOnSponsorTime', function (req, res) {
         return;
     }
 
+    //hash the userID
+    userID = getHash(userID + UUID);
+
+    //x-forwarded-for if this server is behind a proxy
+    let ip = getIP(req);
+
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(ip + globalSalt);
+
     //check if vote has already happened
-    db.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID, function(err, row) {
+    privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID, function(err, row) {
         if (err) console.log(err);
                 
         if (row != undefined && row.type == type) {
@@ -193,17 +221,15 @@ app.get('/api/voteOnSponsorTime', function (req, res) {
 
         //update the votes table
         if (row != undefined) {
-            db.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID);
+            privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID);
         } else {
-            db.prepare("INSERT INTO votes VALUES(?, ?, ?)").run(userID, UUID, type);
+            privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type);
         }
 
         //update the vote count on this sponsorTime
         //oldIncrementAmount will be zero is row is null
         db.prepare("UPDATE sponsorTimes SET votes = votes + ? WHERE UUID = ?").run(incrementAmount - oldIncrementAmount, UUID);
 
-        //update the votes table
-
         //added to db
         res.sendStatus(200);
     });
@@ -236,16 +262,81 @@ app.get('/api/getViewsForUser', function (req, res) {
         return;
     }
 
+    //hash the userID
+    userID = getHash(userID);
+
     //up the view count by one
     db.prepare("SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ?").get(userID, function(err, row) {
         if (err) console.log(err);
 
-        if (row != null) {
+        if (row.viewCount != null) {
             res.send({
                 viewCount: row.viewCount
             });
         } else {
-            res.send(404);
+            res.sendStatus(404);
+        }
+    });
+});
+
+app.get('/api/getTopUsers', function (req, res) {
+    let sortType = req.query.sortType;
+
+    if (sortType == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //setup which sort type to use
+    let sortBy = "";
+    if (sortType == 0) {
+        sortBy = "minutesSaved";
+    } else if (sortType == 1) {
+        sortBy = "viewCount";
+    } else if (sortType == 2) {
+        sortBy = "totalSubmissions";
+    } else {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    let userNames = [];
+    let viewCounts = [];
+    let totalSubmissions = [];
+    let minutesSaved = [];
+
+    db.prepare("SELECT userID, COUNT(*) as totalSubmissions, SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE votes > -1 GROUP BY userID ORDER BY " + sortBy + " DESC LIMIT 50").all(function(err, rows) {
+        for (let i = 0; i < rows.length; i++) {
+            userNames[i] = rows[i].userID;
+            viewCounts[i] = rows[i].viewCount;
+            totalSubmissions[i] = rows[i].totalSubmissions;
+            minutesSaved[i] = rows[i].minutesSaved;
+        }
+
+        //send this result
+        res.send({
+            userNames: userNames,
+            viewCounts: viewCounts,
+            totalSubmissions: totalSubmissions,
+            minutesSaved: minutesSaved
+        });
+    });
+});
+
+//send out totals
+//send the total submissions, total views and total minutes saved
+app.get('/api/getTotalStats', function (req, res) {
+    db.prepare("SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes").get(function(err, row) {
+        if (row != null) {
+            //send this result
+            res.send({
+                userCount: row.userCount,
+                viewCount: row.viewCount,
+                totalSubmissions: row.totalSubmissions,
+                minutesSaved: row.minutesSaved
+            });
         }
     });
 });
@@ -254,11 +345,10 @@ app.get('/database.db', function (req, res) {
     res.sendFile("./databases/sponsorTimes.db", { root: __dirname });
 });
 
-
 //This function will find sponsor times that are contained inside of eachother, called similar sponsor times
 //Only one similar time will be returned, randomly generated based on the sqrt of votes.
 //This allows new less voted items to still sometimes appear to give them a chance at getting votes.
-//Sponsor times with less than -2 votes are already ignored before this function is called
+//Sponsor times with less than -1 votes are already ignored before this function is called
 function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
     //list of sponsors that are contained inside eachother
     let similarSponsors = [];
@@ -415,6 +505,7 @@ function getWeightedRandomChoice(choices, weights, amountOfChoices) {
 
     //iterate and find amountOfChoices choices
     let randomNumber = Math.random();
+    
     //this array will keep adding to this variable each time one sqrt vote has been dealt with
     //this is the sum of all the sqrtVotes under this index
     let currentVoteNumber = 0;
@@ -444,4 +535,13 @@ function getWeightedRandomChoice(choices, weights, amountOfChoices) {
         finalChoices: finalChoices,
         choicesDealtWith: choicesDealtWith
     };
+}
+
+function getHash(value, times=5000) {
+    for (let i = 0; i < times; i++) {
+        let hashCreator = crypto.createHash('sha256');
+        value = hashCreator.update(value).digest('hex');
+    }
+
+    return value;
 }