Merge pull request #17 from ajayyy/experimental

Changed limits and better stats

README.MD

@@ -15,3 +15,181 @@ Hopefully this project can be combined with projects like [this](https://github.
 # Client
 
 The client web browser extension is available here: https://github.com/ajayyy/SponsorBlock
+
+# API Docs
+
+Public API available at https://sponsor.ajay.app.
+
+________________________________________________________________________________
+
+`/api/getVideoSponsorTimes`
+
+**Input**:
+```
+{
+  videoID: string
+}
+```
+
+**Response**:
+```
+{
+  sponorTimes: array [float],
+  UUIDs: array [string] //The ID for this sponsor time, used to submit votes
+}
+```
+
+**Error codes**:
+
+404: Not Found
+
+__________________________________________________________________
+
+`/api/postVideoSponsorTimes`
+
+**Input**:
+```
+{
+  videoID: string,
+  startTime: float,
+  endTime: float,
+  userID: string //This should be a randomly generated UUID
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200)
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+429: Rate Limit (Too many for the same user or IP)
+
+409: Duplicate
+
+__________________________________________________________________
+
+`/api/voteOnSponsorTime`
+
+**Input**:
+```
+{
+  UUID: string, //id of the sponsor being voted on
+  userID: string,
+  type: int //0 for downvote, 1 for upvote
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200)
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+405: Duplicate
+
+__________________________________________________________________
+
+`/api/viewedVideoSponsorTime`
+
+**Input**:
+```
+{
+  UUID: string
+}
+```
+
+**Response**:
+```
+{
+  Nothing (status code 200
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+__________________________________________________________________
+
+`/api/getViewsForUser`
+
+**Input**:
+```
+{
+  userID: string
+}
+```
+
+**Response**:
+```
+{
+  viewCount: int
+}
+```
+
+**Error codes**:
+
+404: Not Found
+
+__________________________________________________________________
+
+### Stats Calls
+
+`/api/getTopUsers`
+
+**Input**:
+```
+{
+  sortType: int //0 for by minutes saved, 1 for by view count, 2 for by total submissions
+}
+```
+
+**Response**:
+```
+{
+  userNames: array [string],
+  viewCounts: array [int],
+  totalSubmissions: array [int],
+  minutesSaved: array [float]
+}
+```
+
+**Error codes**:
+
+400: Bad Request (Your inputs are wrong/impossible)
+
+__________________________________________________________________
+
+`/api/getTotalStats`
+
+**Input**:
+```
+{
+  Nothing
+}
+```
+
+**Response**:
+```
+{
+  userCount: int,
+  viewCount: int,
+  totalSubmissions: int,
+  minutesSaved: float
+}
+```
+
+**Error codes**:
+
+None

index.js

@@ -1,12 +1,8 @@
 var express = require('express');
 var http = require('http');
-
 // Create a service (the app object is just a callback).
 var app = express();
 
-//uuid service
-var uuidv1 = require('uuid/v1');
-
 //hashing service
 var crypto = require('crypto');
 
@@ -21,7 +17,10 @@ http.createServer(app).listen(80);
 
 //global salt that is added to every ip before hashing to
 //  make it even harder for someone to decode the ip
-var globalSalt = "49cb0d52-1aec-4b89-85fc-fab2c53062fb";
+var globalSalt = "49cb0d52-1aec-4b89-85fc-fab2c53062fb"; // Should not be global
+
+//if so, it will use the x-forwarded header instead of the ip address of the connection
+var behindProxy = true;
 
 //setup CORS correctly
 app.use(function(req, res, next) {
@@ -42,8 +41,8 @@ app.get('/api/getVideoSponsorTimes', function (req, res) {
         if (err) console.log(err);
 
         for (let i = 0; i < rows.length; i++) {
-            //check if votes are above -2
-            if (rows[i].votes < -2) {
+            //check if votes are above -1
+            if (rows[i].votes < -1) {
                 //too untrustworthy, just ignore it
                 continue;
             }
@@ -79,6 +78,10 @@ app.get('/api/getVideoSponsorTimes', function (req, res) {
     });
 });
 
+function getIP(req) {
+    return behindProxy ? req.headers['x-forwarded-for'] : req.connection.remoteAddress;
+}
+
 //add the post function
 app.get('/api/postVideoSponsorTimes', function (req, res) {
     let videoID = req.query.videoID;
@@ -95,18 +98,10 @@ app.get('/api/postVideoSponsorTimes', function (req, res) {
     }
 
     //hash the userID
-    userID = getHashedUserID(userID);
-
-    //x-forwarded-for if this server is behind a proxy
-    let ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
-
-    //hash the ip so no one can get it from the database
-    let hashedIP = ip + globalSalt;
-    //hash it 5000 times, this makes it very hard to brute force
-    for (let i = 0; i < 5000; i++) {
-        let hashCreator = crypto.createHash('sha256');
-        hashedIP = hashCreator.update(hashedIP).digest('hex');
-    }
+    userID = getHash(userID);
+    
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(getIP(req) + globalSalt);
 
     startTime = parseFloat(startTime);
     endTime = parseFloat(endTime);
@@ -142,7 +137,7 @@ app.get('/api/postVideoSponsorTimes', function (req, res) {
         } else {
             //check to see if the user has already submitted sponsors for this video
             db.prepare("SELECT COUNT(*) as count FROM sponsorTimes WHERE userID = ? and videoID = ?").get([userID, videoID], function(err, row) {
-                if (row.count >= 4) {
+                if (row.count >= 8) {
                     //too many sponsors for the same video from the same user
                     res.sendStatus(429);
                 } else {
@@ -181,18 +176,13 @@ app.get('/api/voteOnSponsorTime', function (req, res) {
     }
 
     //hash the userID
-    userID = getHashedUserID(userID + UUID);
+    userID = getHash(userID + UUID);
 
     //x-forwarded-for if this server is behind a proxy
-    let ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
+    let ip = getIP(req);
 
-    //hash the ip so no one can get it from the database
-    let hashedIP = ip + globalSalt;
-    //hash it 5000 times, this makes it very hard to brute force
-    for (let i = 0; i < 5000; i++) {
-        let hashCreator = crypto.createHash('sha256');
-        hashedIP = hashCreator.update(hashedIP).digest('hex');
-    }
+    //hash the ip 5000 times so no one can get it from the database
+    let hashedIP = getHash(ip + globalSalt);
 
     //check if vote has already happened
     privateDB.prepare("SELECT type FROM votes WHERE userID = ? AND UUID = ?").get(userID, UUID, function(err, row) {
@@ -273,7 +263,7 @@ app.get('/api/getViewsForUser', function (req, res) {
     }
 
     //hash the userID
-    userID = getHashedUserID(userID);
+    userID = getHash(userID);
 
     //up the view count by one
     db.prepare("SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ?").get(userID, function(err, row) {
@@ -289,26 +279,76 @@ app.get('/api/getViewsForUser', function (req, res) {
     });
 });
 
+app.get('/api/getTopUsers', function (req, res) {
+    let sortType = req.query.sortType;
+
+    if (sortType == undefined) {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    //setup which sort type to use
+    let sortBy = "";
+    if (sortType == 0) {
+        sortBy = "minutesSaved";
+    } else if (sortType == 1) {
+        sortBy = "viewCount";
+    } else if (sortType == 2) {
+        sortBy = "totalSubmissions";
+    } else {
+        //invalid request
+        res.sendStatus(400);
+        return;
+    }
+
+    let userNames = [];
+    let viewCounts = [];
+    let totalSubmissions = [];
+    let minutesSaved = [];
+
+    db.prepare("SELECT userID, COUNT(*) as totalSubmissions, SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes WHERE votes > -1 GROUP BY userID ORDER BY " + sortBy + " DESC LIMIT 50").all(function(err, rows) {
+        for (let i = 0; i < rows.length; i++) {
+            userNames[i] = rows[i].userID;
+            viewCounts[i] = rows[i].viewCount;
+            totalSubmissions[i] = rows[i].totalSubmissions;
+            minutesSaved[i] = rows[i].minutesSaved;
+        }
+
+        //send this result
+        res.send({
+            userNames: userNames,
+            viewCounts: viewCounts,
+            totalSubmissions: totalSubmissions,
+            minutesSaved: minutesSaved
+        });
+    });
+});
+
+//send out totals
+//send the total submissions, total views and total minutes saved
+app.get('/api/getTotalStats', function (req, res) {
+    db.prepare("SELECT COUNT(DISTINCT userID) as userCount, COUNT(*) as totalSubmissions, SUM(views) as viewCount, SUM((endTime - startTime) / 60 * views) as minutesSaved FROM sponsorTimes").get(function(err, row) {
+        if (row != null) {
+            //send this result
+            res.send({
+                userCount: row.userCount,
+                viewCount: row.viewCount,
+                totalSubmissions: row.totalSubmissions,
+                minutesSaved: row.minutesSaved
+            });
+        }
+    });
+});
+
 app.get('/database.db', function (req, res) {
     res.sendFile("./databases/sponsorTimes.db", { root: __dirname });
 });
 
-function getHashedUserID(userID) {
-    //hash the userID so no one can get it from the database
-    let hashedUserID = userID;
-    //hash it 5000 times, this makes it very hard to brute force
-    for (let i = 0; i < 5000; i++) {
-        let hashCreator = crypto.createHash('sha256');
-        hashedUserID = hashCreator.update(hashedUserID).digest('hex');
-    }
-
-    return hashedUserID;
-}
-
 //This function will find sponsor times that are contained inside of eachother, called similar sponsor times
 //Only one similar time will be returned, randomly generated based on the sqrt of votes.
 //This allows new less voted items to still sometimes appear to give them a chance at getting votes.
-//Sponsor times with less than -2 votes are already ignored before this function is called
+//Sponsor times with less than -1 votes are already ignored before this function is called
 function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
     //list of sponsors that are contained inside eachother
     let similarSponsors = [];
@@ -465,6 +505,7 @@ function getWeightedRandomChoice(choices, weights, amountOfChoices) {
 
     //iterate and find amountOfChoices choices
     let randomNumber = Math.random();
+    
     //this array will keep adding to this variable each time one sqrt vote has been dealt with
     //this is the sum of all the sqrtVotes under this index
     let currentVoteNumber = 0;
@@ -494,4 +535,13 @@ function getWeightedRandomChoice(choices, weights, amountOfChoices) {
         finalChoices: finalChoices,
         choicesDealtWith: choicesDealtWith
     };
+}
+
+function getHash(value, times=5000) {
+    for (let i = 0; i < times; i++) {
+        let hashCreator = crypto.createHash('sha256');
+        value = hashCreator.update(value).digest('hex');
+    }
+
+    return value;
 }