Merge pull request #91 from pdonias/rework-getSkipSegments

`getSkipSegments` rework

src/routes/getSkipSegments.js

@@ -8,175 +8,86 @@ var privateDB = databases.privateDB;
 var getHash = require('../utils/getHash.js');
 var getIP = require('../utils/getIP.js');
 
-
+//gets a weighted random choice from the choices array based on their `votes` property.
-//gets the getWeightedRandomChoice for each group in an array of groups
+//amountOfChoices specifies the maximum amount of choices to return, 1 or more.
-function getWeightedRandomChoiceForArray(choiceGroups, weights) {
-  let finalChoices = [];
-  //the indexes either chosen to be added to final indexes or chosen not to be added
-  let choicesDealtWith = [];
-  //for each choice group, what are the sums of the weights
-  let weightSums = [];
-
-  for (let i = 0; i < choiceGroups.length; i++) {
-      //find weight sums for this group
-      weightSums.push(0);
-      for (let j = 0; j < choiceGroups[i].length; j++) {
-          //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
-          if (weights[choiceGroups[i][j]] > 0) {
-              weightSums[weightSums.length - 1] += weights[choiceGroups[i][j]];
-          }
-      }
-
-      //create a random choice for this group
-      let randomChoice = getWeightedRandomChoice(choiceGroups[i], weights, 1)
-      finalChoices.push(randomChoice.finalChoices);
-
-      for (let j = 0; j < randomChoice.choicesDealtWith.length; j++) {
-          choicesDealtWith.push(randomChoice.choicesDealtWith[j])
-      }
-  }
-
-  return {
-      finalChoices: finalChoices,
-      choicesDealtWith: choicesDealtWith,
-      weightSums: weightSums
-  };
-}
-
-//gets a weighted random choice from the indexes array based on the weights.
-//amountOfChoices speicifies the amount of choices to return, 1 or more.
 //choices are unique
-function getWeightedRandomChoice(choices, weights, amountOfChoices) {
+function getWeightedRandomChoice(choices, amountOfChoices) {
-  if (amountOfChoices > choices.length) {
+  //trivial case: no need to go through the whole process
-      //not possible, since all choices must be unique
+  if (amountOfChoices >= choices.length) {
-      return null;
+    return choices;
   }
 
-  let finalChoices = [];
+  //assign a weight to each choice
-  let choicesDealtWith = [];
+  let totalWeight = 0;
-
+  choices = choices.map(choice => {
-  let sqrtWeightsList = [];
-  //the total of all the weights run through the cutom sqrt function
-  let totalSqrtWeights = 0;
-  for (let j = 0; j < choices.length; j++) {
     //multiplying by 10 makes around 13 votes the point where it the votes start not mattering as much (10 + 3)
     //The 3 makes -2 the minimum votes before being ignored completely
     //https://www.desmos.com/calculator/ljftxolg9j
     //this can be changed if this system increases in popularity.
-      let sqrtVote = Math.sqrt((weights[choices[j]] + 3) * 10);
+    const weight = Math.sqrt((choice.votes + 3) * 10);
-      sqrtWeightsList.push(sqrtVote)
+    totalWeight += weight;
-      totalSqrtWeights += sqrtVote;
 
-      //this index has now been deat with
+    return { ...choice, weight };
-      choicesDealtWith.push(choices[j]);
+  });
-  }
 
   //iterate and find amountOfChoices choices
-  let randomNumber = Math.random();
+  const chosen = [];
-  
+  while (amountOfChoices-- > 0) {
-  //this array will keep adding to this variable each time one sqrt vote has been dealt with
+    //weighted random draw of one element of choices
-  //this is the sum of all the sqrtVotes under this index
+    const randomNumber = Math.random() * totalWeight;
-  let currentVoteNumber = 0;
+    let stackWeight = choices[0].weight;
-  for (let j = 0; j < sqrtWeightsList.length; j++) {
+    let i = 0;
-      if (randomNumber > currentVoteNumber / totalSqrtWeights && randomNumber < (currentVoteNumber + sqrtWeightsList[j]) / totalSqrtWeights) {
+    while (stackWeight < randomNumber) {
-          //this one was randomly generated
+      stackWeight += choices[++i].weight;
-          finalChoices.push(choices[j]);
-          //remove that from original array, for next recursion pass if it happens
-          choices.splice(j, 1);
-          break;
     }
 
-      //add on to the count
+    //add it to the chosen ones and remove it from the choices before the next iteration
-      currentVoteNumber += sqrtWeightsList[j];
+    chosen.push(choices[i]);
+    choices.splice(i, 1);
   }
 
-  //add on the other choices as well using recursion
+  return chosen;
-  if (amountOfChoices > 1) {
-      let otherChoices = getWeightedRandomChoice(choices, weights, amountOfChoices - 1).finalChoices;
-      //add all these choices to the finalChoices array being returned
-      for (let i = 0; i < otherChoices.length; i++) {
-          finalChoices.push(otherChoices[i]);
-      }
-  }
-
-  return {
-      finalChoices: finalChoices,
-      choicesDealtWith: choicesDealtWith
-  };
 }
 
-
+//This function will find segments that are contained inside of eachother, called similar segments
-//This function will find sponsor times that are contained inside of eachother, called similar sponsor times
 //Only one similar time will be returned, randomly generated based on the sqrt of votes.
 //This allows new less voted items to still sometimes appear to give them a chance at getting votes.
-//Sponsor times with less than -1 votes are already ignored before this function is called
+//Segments with less than -1 votes are already ignored before this function is called
-function getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs) {
+function chooseSegments(segments) {
-  //create groups of sponsor times that are similar to eachother
+  //Create groups of segments that are similar to eachother
-  const groups = []
+  //Segments must be sorted by their startTime so that we can build groups chronologically:
-  sponsorTimes.forEach(([sponsorStart, sponsorEnd], i) => {
+  //1. As long as the segments' startTime fall inside the currentGroup, we keep adding them to that group
-      //find a group that overlaps with the current segment
+  //2. If a segment starts after the end of the currentGroup (> cursor), no other segment will ever fall
-      //sponsorTimes are sorted by their startTime so there should never be more than 1 similar group
+  //   inside that group (because they're sorted) so we can create a new one
-      const similarGroup = groups.find(group => group.start < sponsorEnd && sponsorStart < group.end)
+  const similarSegmentsGroups = [];
-      //add the sponsor to that group or create a new group if there aren't any
+  let currentGroup;
-      if (similarGroup === undefined) {
+  let cursor = -1; //-1 to make sure that, even if the 1st segment starts at 0, a new group is created
-          groups.push({ start: sponsorStart, end: sponsorEnd, sponsors: [i] })
+  segments.forEach(segment => {
-      } else {
+    if (segment.startTime > cursor) {
-          similarGroup.sponsors.push(i)
+      currentGroup = { segments: [], votes: 0 };
-          similarGroup.start = Math.min(similarGroup.start, sponsorStart)
+      similarSegmentsGroups.push(currentGroup);
-          similarGroup.end = Math.max(similarGroup.end, sponsorEnd)
-      }
-  })
-
-  //once all the groups have been created, get rid of the metadata and remove single-sponsor groups
-  const similarSponsorsGroups = groups.map(group => group.sponsors).filter(group => group.length > 1)
-
-  let weightedRandomIndexes = getWeightedRandomChoiceForArray(similarSponsorsGroups, votes);
-
-  let finalSponsorTimeIndexes = weightedRandomIndexes.finalChoices;
-  //the sponsor times either chosen to be added to finalSponsorTimeIndexes or chosen not to be added
-  let finalSponsorTimeIndexesDealtWith = weightedRandomIndexes.choicesDealtWith;
-
-  let voteSums = weightedRandomIndexes.weightSums;
-  //convert these into the votes
-  for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
-      //it should use the sum of votes, since anyone upvoting a similar sponsor is upvoting the existence of that sponsor.
-      votes[finalSponsorTimeIndexes[i]] = voteSums[i];
     }
 
-  //find the indexes never dealt with and add them
+    currentGroup.segments.push(segment);
-  for (let i = 0; i < sponsorTimes.length; i++) {
+    //only if it is a positive vote, otherwise it is probably just a sponsor time with slightly wrong time
-      if (!finalSponsorTimeIndexesDealtWith.includes(i)) {
+    if (segment.votes > 0) {
-          finalSponsorTimeIndexes.push(i)
+      currentGroup.votes += segment.votes;
-      }
     }
 
-  //if there are too many indexes, find the best 4
+    cursor = Math.max(cursor, segment.endTime);
-  if (finalSponsorTimeIndexes.length > 8) {
+  });
-      finalSponsorTimeIndexes = getWeightedRandomChoice(finalSponsorTimeIndexes, votes, 8).finalChoices;
-  }
 
-  //convert this to a final array to return
+  //if there are too many groups, find the best 8
-  let finalSponsorTimes = [];
+  return getWeightedRandomChoice(similarSegmentsGroups, 8).map(
-  for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
+    //randomly choose 1 good segment per group and return them
-      finalSponsorTimes.push(sponsorTimes[finalSponsorTimeIndexes[i]]);
+    group => getWeightedRandomChoice(group.segments, 1)[0]
-  }
+  );
-
-  //convert this to a final array of UUIDs as well
-  let finalUUIDs = [];
-  for (let i = 0; i < finalSponsorTimeIndexes.length; i++) {
-      finalUUIDs.push(UUIDs[finalSponsorTimeIndexes[i]]);
-  }
-
-  return {
-      sponsorTimes: finalSponsorTimes,
-      UUIDs: finalUUIDs
-  };
 }
 
 /**
  *
  * Returns what would be sent to the client.
- * Will resond with errors if required. Returns false if it errors.
+ * Will respond with errors if required. Returns false if it errors.
  *
  * @param req
  * @param res
@@ -187,8 +98,11 @@ function handleGetSegments(req, res) {
   const videoID = req.query.videoID;
   // Default to sponsor
   // If using params instead of JSON, only one category can be pulled
-    const categories = req.query.categories ? JSON.parse(req.query.categories) 
+  const categories = req.query.categories
-        : (req.query.category ? [req.query.category] : ["sponsor"]);
+    ? JSON.parse(req.query.categories)
+    : req.query.category
+    ? [req.query.category]
+    : ['sponsor'];
 
   /**
    * @type {Array<{
@@ -198,61 +112,51 @@ function handleGetSegments(req, res) {
    *              }>
    *       }
    */
-    let segments = [];
+  const segments = [];
 
-    let hashedIP = getHash(getIP(req) + config.globalSalt);
+  let userHashedIP, shadowHiddenSegments;
 
   try {
     for (const category of categories) {
-            let rows = db.prepare("SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime")
+      const categorySegments = db
-                .all(videoID, category);
+        .prepare(
-
+          'SELECT startTime, endTime, votes, UUID, shadowHidden FROM sponsorTimes WHERE videoID = ? and category = ? ORDER BY startTime'
-            let sponsorTimes = [];
+        )
-            let votes = []
+        .all(videoID, category)
-            let UUIDs = [];
+        .filter(segment => {
-    
+          if (segment.votes < -1) {
-            for (let i = 0; i < rows.length; i++) {
+            return false; //too untrustworthy, just ignore it
-                //check if votes are above -1
-                if (rows[i].votes < -1) {
-                    //too untrustworthy, just ignore it
-                    continue;
           }
 
           //check if shadowHidden
           //this means it is hidden to everyone but the original ip that submitted it
-                if (rows[i].shadowHidden == 1) {
+          if (segment.shadowHidden != 1) {
-                    //get the ip
+            return true;
-                    //await the callback
-                    let hashedIPRow = privateDB.prepare("SELECT hashedIP FROM sponsorTimes WHERE videoID = ?").all(videoID);
-    
-                    if (!hashedIPRow.some((e) => e.hashedIP === hashedIP)) {
-                        //this isn't their ip, don't send it to them
-                        continue;
-                    }
           }
 
-                sponsorTimes.push([rows[i].startTime, rows[i].endTime]);
+          if (shadowHiddenSegments === undefined) {
-                votes.push(rows[i].votes);
+            shadowHiddenSegments = privateDB
-                UUIDs.push(rows[i].UUID);
+              .prepare('SELECT hashedIP FROM sponsorTimes WHERE videoID = ?')
+              .all(videoID);
           }
 
-            organisedData = getVoteOrganisedSponsorTimes(sponsorTimes, votes, UUIDs);
+          //if this isn't their ip, don't send it to them
-            sponsorTimes = organisedData.sponsorTimes;
+          return shadowHiddenSegments.some(shadowHiddenSegment => {
-            UUIDs = organisedData.UUIDs;
+            if (userHashedIP === undefined) {
-    
+              //hash the IP only if it's strictly necessary
-            for (let i = 0; i < sponsorTimes.length; i++) {
+              userHashedIP = getHash(getIP(req) + config.globalSalt);
-                segments.push({
+            }
-                    segment: sponsorTimes[i],
+            return shadowHiddenSegment.hashedIP === userHashedIP;
-                    category: category,
+          });
-                    UUID: UUIDs[i]
         });
-            }
-        }
-    } catch(error) {
-        console.error(error);
-        res.send(500);
 
-        return false;
+      chooseSegments(categorySegments).forEach(chosenSegment => {
+        segments.push({
+          category,
+          segment: [chosenSegment.startTime, chosenSegment.endTime],
+          UUID: chosenSegment.UUID,
+        });
+      });
     }
 
     if (segments.length == 0) {
@@ -261,8 +165,13 @@ function handleGetSegments(req, res) {
     }
 
     return segments;
-}
+  } catch (error) {
+    console.error(error);
+    res.sendStatus(500);
 
+    return false;
+  }
+}
 
 module.exports = {
   handleGetSegments,
@@ -271,7 +180,7 @@ module.exports = {
 
     if (segments) {
       //send result
-            res.send(segments)
+      res.send(segments);
     }
-    }
+  },
-}
+};

src/routes/voteOnSponsorTime.js

@@ -182,7 +182,7 @@ module.exports = async function voteOnSponsorTime(req, res) {
         // Send discord message
         if (incrementAmount < 0) {
             // Get video ID
-            let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, u.userName, " +
+            let submissionInfoRow = db.prepare("SELECT s.videoID, s.userID, s.startTime, s.endTime, s.category, u.userName, " +
                 "(select count(1) from sponsorTimes where userID = s.userID) count, " +
                 "(select count(1) from sponsorTimes where userID = s.userID and votes <= -2) disregarded " +
                 "FROM sponsorTimes s left join userNames u on s.userID = u.userID where s.UUID=?"
@@ -216,6 +216,7 @@ module.exports = async function voteOnSponsorTime(req, res) {
                                       + "&t=" + (submissionInfoRow.startTime.toFixed(0) - 2),
                                     "description": "**" + row.votes + " Votes Prior | " + (row.votes + incrementAmount - oldIncrementAmount) + " Votes Now | " + row.views 
                                         + " Views**\n\n**Submission ID:** " + UUID 
+                                        + "\n**Category:** " + submissionInfoRow.category
                                         + "\n\n**Submitted by:** "+submissionInfoRow.userName+"\n " + submissionInfoRow.userID 
                                         + "\n\n**Total User Submissions:** "+submissionInfoRow.count
                                         + "\n**Ignored User Submissions:** "+submissionInfoRow.disregarded