Mirror/SponsorBlockServer
1
0
Fork 0
mirror of https://github.com/ajayyy/SponsorBlockServer.git synced 2025-12-19 05:58:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into experimental

Browse Source
This commit is contained in:
Ajay Ramachandran
2020-10-15 11:32:25 -04:00
parent 34b8119a21 ad74c2c710
commit d6ca579495
23 changed files with 1275 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/routes/deleteNoSegments.js Normal file
View File

@@ -0,0 +1,43 @@
const db = require('../databases/databases.js').db;
const getHash = require('../utils/getHash.js');
const isUserVIP = require('../utils/isUserVIP.js');
const logger = require('../utils/logger.js');
module.exports = (req, res) => {
// Collect user input data
let videoID = req.body.videoID;
let userID = req.body.userID;
let categories = req.body.categories;
// Check input data is valid
if (!videoID
|| !userID
|| !categories
|| !Array.isArray(categories)
|| categories.length === 0
) {
res.status(400).json({
message: 'Bad Format'
});
return;
}
// Check if user is VIP
userID = getHash(userID);
let userIsVIP = isUserVIP(userID);
if (!userIsVIP) {
res.status(403).json({
message: 'Must be a VIP to mark videos.'
});
return;
}
db.prepare("all", 'SELECT * FROM noSegments WHERE videoID = ?', [videoID]).filter((entry) => {
return (categories.indexOf(entry.category) !== -1);
}).forEach((entry) => {
db.prepare('run', 'DELETE FROM noSegments WHERE videoID = ? AND category = ?', [videoID, entry.category]);
});
res.status(200).json({message: 'Removed no segments entrys for video ' + videoID});
};

6
src/routes/getSkipSegmentsByHash.js
View File

@@ -20,10 +20,6 @@ module.exports = async function (req, res) {
// Get all video id's that match hash prefix
const videoIds = db.prepare('all', 'SELECT DISTINCT videoId, hashedVideoID from sponsorTimes WHERE hashedVideoID LIKE ?', [hashPrefix+'%']);
if (videoIds.length === 0) {
res.sendStatus(404);
return;
}
let segments = videoIds.map((video) => {
return {
@@ -33,5 +29,5 @@ module.exports = async function (req, res) {
};
});
res.status(200).json(segments);
res.status((segments.length === 0) ? 404 : 200).json(segments);
}

133
src/routes/getTopUsers.js
View File

@@ -1,6 +1,68 @@
var db = require('../databases/databases.js').db;
const logger = require('../utils/logger.js');
const createMemoryCache = require('../utils/createMemoryCache.js');
const config = require('../config.js');
module.exports = function getTopUsers (req, res) {
const MILLISECONDS_IN_MINUTE = 60000;
const getTopUsersWithCache = createMemoryCache(generateTopUsersStats, config.getTopUsersCacheTimeMinutes * MILLISECONDS_IN_MINUTE);
function generateTopUsersStats(sortBy, categoryStatsEnabled = false) {
return new Promise((resolve, reject) => {
const userNames = [];
const viewCounts = [];
const totalSubmissions = [];
const minutesSaved = [];
const categoryStats = categoryStatsEnabled ? [] : undefined;
let additionalFields = '';
if (categoryStatsEnabled) {
additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " +
"SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " +
"SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " +
"SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " +
"SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " +
"SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, ";
}
const rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," +
"SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " +
"SUM(votes) as userVotes, " +
additionalFields +
"IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " +
"LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " +
"WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " +
"GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " +
"ORDER BY " + sortBy + " DESC LIMIT 100", []);
for (let i = 0; i < rows.length; i++) {
userNames[i] = rows[i].userName;
viewCounts[i] = rows[i].viewCount;
totalSubmissions[i] = rows[i].totalSubmissions;
minutesSaved[i] = rows[i].minutesSaved;
if (categoryStatsEnabled) {
categoryStats[i] = [
rows[i].categorySponsor,
rows[i].categorySumIntro,
rows[i].categorySumOutro,
rows[i].categorySumInteraction,
rows[i].categorySelfpromo,
rows[i].categoryMusicOfftopic,
];
}
}
resolve({
userNames,
viewCounts,
totalSubmissions,
minutesSaved,
categoryStats
});
});
}
module.exports = async function getTopUsers (req, res) {
let sortType = req.query.sortType;
let categoryStatsEnabled = req.query.categoryStats;
@@ -9,71 +71,22 @@ module.exports = function getTopUsers (req, res) {
res.sendStatus(400);
return;
}
//setup which sort type to use
let sortBy = "";
let sortBy = '';
if (sortType == 0) {
sortBy = "minutesSaved";
sortBy = 'minutesSaved';
} else if (sortType == 1) {
sortBy = "viewCount";
sortBy = 'viewCount';
} else if (sortType == 2) {
sortBy = "totalSubmissions";
sortBy = 'totalSubmissions';
} else {
//invalid request
res.sendStatus(400);
return;
//invalid request
return res.sendStatus(400);
}
let userNames = [];
let viewCounts = [];
let totalSubmissions = [];
let minutesSaved = [];
let categoryStats = categoryStatsEnabled ? [] : undefined;
let additionalFields = '';
if (categoryStatsEnabled) {
additionalFields += "SUM(CASE WHEN category = 'sponsor' THEN 1 ELSE 0 END) as categorySponsor, " +
"SUM(CASE WHEN category = 'intro' THEN 1 ELSE 0 END) as categorySumIntro, " +
"SUM(CASE WHEN category = 'outro' THEN 1 ELSE 0 END) as categorySumOutro, " +
"SUM(CASE WHEN category = 'interaction' THEN 1 ELSE 0 END) as categorySumInteraction, " +
"SUM(CASE WHEN category = 'selfpromo' THEN 1 ELSE 0 END) as categorySelfpromo, " +
"SUM(CASE WHEN category = 'music_offtopic' THEN 1 ELSE 0 END) as categoryMusicOfftopic, ";
}
let rows = db.prepare('all', "SELECT COUNT(*) as totalSubmissions, SUM(views) as viewCount," +
"SUM((sponsorTimes.endTime - sponsorTimes.startTime) / 60 * sponsorTimes.views) as minutesSaved, " +
"SUM(votes) as userVotes, " +
additionalFields +
"IFNULL(userNames.userName, sponsorTimes.userID) as userName FROM sponsorTimes LEFT JOIN userNames ON sponsorTimes.userID=userNames.userID " +
"LEFT JOIN privateDB.shadowBannedUsers ON sponsorTimes.userID=privateDB.shadowBannedUsers.userID " +
"WHERE sponsorTimes.votes > -1 AND sponsorTimes.shadowHidden != 1 AND privateDB.shadowBannedUsers.userID IS NULL " +
"GROUP BY IFNULL(userName, sponsorTimes.userID) HAVING userVotes > 20 " +
"ORDER BY " + sortBy + " DESC LIMIT 100", []);
const stats = await getTopUsersWithCache(sortBy, categoryStatsEnabled);
for (let i = 0; i < rows.length; i++) {
userNames[i] = rows[i].userName;
viewCounts[i] = rows[i].viewCount;
totalSubmissions[i] = rows[i].totalSubmissions;
minutesSaved[i] = rows[i].minutesSaved;
if (categoryStatsEnabled) {
categoryStats[i] = [
rows[i].categorySponsor,
rows[i].categorySumIntro,
rows[i].categorySumOutro,
rows[i].categorySumInteraction,
rows[i].categorySelfpromo,
rows[i].categoryMusicOfftopic,
];
}
}
//send this result
res.send({
userNames,
viewCounts,
totalSubmissions,
minutesSaved,
categoryStats
});
}
res.send(stats);
}

82
src/routes/getUserInfo.js Normal file
View File

@@ -0,0 +1,82 @@
const db = require('../databases/databases.js').db;
const getHash = require('../utils/getHash.js');
function dbGetSubmittedSegmentSummary (userID) {
try {
let row = db.prepare("get", "SELECT SUM(((endTime - startTime) / 60) * views) as minutesSaved, count(*) as segmentCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]);
if (row.minutesSaved != null) {
return {
minutesSaved: row.minutesSaved,
segmentCount: row.segmentCount,
};
} else {
return {
minutesSaved: 0,
segmentCount: 0,
};
}
} catch (err) {
return false;
}
}
function dbGetUsername (userID) {
try {
let row = db.prepare('get', "SELECT userName FROM userNames WHERE userID = ?", [userID]);
if (row !== undefined) {
return row.userName;
} else {
//no username yet, just send back the userID
return userID;
}
} catch (err) {
return false;
}
}
function dbGetViewsForUser (userID) {
try {
let row = db.prepare('get', "SELECT SUM(views) as viewCount FROM sponsorTimes WHERE userID = ? AND votes > -2 AND shadowHidden != 1", [userID]);
//increase the view count by one
if (row.viewCount != null) {
return row.viewCount;
} else {
return 0;
}
} catch (err) {
return false;
}
}
function dbGetWarningsForUser (userID) {
try {
let rows = db.prepare('all', "SELECT * FROM warnings WHERE userID = ?", [userID]);
return rows.length;
} catch (err) {
logger.error('Couldn\'t get warnings for user ' + userID + '. returning 0') ;
return 0;
}
}
module.exports = function getUserInfo (req, res) {
let userID = req.query.userID;
if (userID == undefined) {
//invalid request
res.status(400).send('Parameters are not valid');
return;
}
//hash the userID
userID = getHash(userID);
const segmentsSummary = dbGetSubmittedSegmentSummary(userID);
res.send({
userID,
userName: dbGetUsername(userID),
minutesSaved: segmentsSummary.minutesSaved,
segmentCount: segmentsSummary.segmentCount,
viewCount: dbGetViewsForUser(userID),
warnings: dbGetWarningsForUser(userID)
});
}

101
src/routes/postSegmentShift.js Normal file
View File

@@ -0,0 +1,101 @@
const db = require('../databases/databases.js').db;
const getHash = require('../utils/getHash.js');
const isUserVIP = require('../utils/isUserVIP.js');
const logger = require('../utils/logger.js');
const ACTION_NONE = Symbol('none');
const ACTION_UPDATE = Symbol('update');
const ACTION_REMOVE = Symbol('remove');
function shiftSegment(segment, shift) {
if (segment.startTime >= segment.endTime) return {action: ACTION_NONE, segment};
if (shift.startTime >= shift.endTime) return {action: ACTION_NONE, segment};
const duration = shift.endTime - shift.startTime;
if (shift.endTime < segment.startTime) {
// Scenario #1 cut before segment
segment.startTime -= duration;
segment.endTime -= duration;
return {action: ACTION_UPDATE, segment};
}
if (shift.startTime > segment.endTime) {
// Scenario #2 cut after segment
return {action: ACTION_NONE, segment};
}
if (segment.startTime < shift.startTime && segment.endTime > shift.endTime) {
// Scenario #3 cut inside segment
segment.endTime -= duration;
return {action: ACTION_UPDATE, segment};
}
if (segment.startTime >= shift.startTime && segment.endTime > shift.endTime) {
// Scenario #4 cut overlap startTime
segment.startTime = shift.startTime;
segment.endTime -= duration;
return {action: ACTION_UPDATE, segment};
}
if (segment.startTime < shift.startTime && segment.endTime <= shift.endTime) {
// Scenario #5 cut overlap endTime
segment.endTime = shift.startTime;
return {action: ACTION_UPDATE, segment};
}
if (segment.startTime >= shift.startTime && segment.endTime <= shift.endTime) {
// Scenario #6 cut overlap startTime and endTime
return {action: ACTION_REMOVE, segment};
}
return {action: ACTION_NONE, segment};
}
module.exports = (req, res) => {
// Collect user input data
const videoID = req.body.videoID;
const startTime = req.body.startTime;
const endTime = req.body.endTime;
let userID = req.body.userID;
// Check input data is valid
if (!videoID
|| !userID
|| !startTime
|| !endTime
) {
res.status(400).json({
message: 'Bad Format'
});
return;
}
// Check if user is VIP
userID = getHash(userID);
const userIsVIP = isUserVIP(userID);
if (!userIsVIP) {
res.status(403).json({
message: 'Must be a VIP to perform this action.'
});
return;
}
try {
const segments = db.prepare('all', 'SELECT startTime, endTime, UUID FROM sponsorTimes WHERE videoID = ?', [videoID]);
const shift = {
startTime,
endTime,
};
segments.forEach(segment => {
const result = shiftSegment(segment, shift);
switch (result.action) {
case ACTION_UPDATE:
db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ? WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]);
break;
case ACTION_REMOVE:
db.prepare('run', 'UPDATE sponsorTimes SET startTime = ?, endTime = ?, votes = -2 WHERE UUID = ?', [result.segment.startTime, result.segment.endTime, result.segment.UUID]);
break;
}
});
}
catch(err) {
logger.error(err);
res.sendStatus(500);
}
res.sendStatus(200);
};

80
src/routes/postSkipSegments.js
View File

@@ -50,7 +50,7 @@ function sendWebhooks(userID, videoID, UUID, segmentInfo) {
if (config.youtubeAPIKey !== null) {
let userSubmissionCountRow = db.prepare('get', "SELECT count(*) as submissionCount FROM sponsorTimes WHERE userID = ?", [userID]);
YouTubeAPI.listVideos(videoID, "snippet", (err, data) => {
YouTubeAPI.listVideos(videoID, (err, data) => {
if (err || data.items.length === 0) {
err && logger.error(err);
return;
@@ -154,7 +154,7 @@ async function autoModerateSubmission(submission) {
// Get the video information from the youtube API
if (config.youtubeAPIKey !== null) {
let {err, data} = await new Promise((resolve, reject) => {
YouTubeAPI.listVideos(submission.videoID, "contentDetails,snippet", (err, data) => resolve({err, data}));
YouTubeAPI.listVideos(submission.videoID, (err, data) => resolve({err, data}));
});
if (err) {
@@ -269,6 +269,16 @@ module.exports = async function postSkipSegments(req, res) {
//hash the ip 5000 times so no one can get it from the database
let hashedIP = getHash(getIP(req) + config.globalSalt);
const MILLISECONDS_IN_HOUR = 3600000;
const now = Date.now();
let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?",
[userID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))]
).count;
if (warningsCount >= config.maxNumberOfActiveWarnings) {
return res.status(403).send('Submission blocked. Too many active warnings!');
}
let noSegmentList = db.prepare('all', 'SELECT category from noSegments where videoID = ?', [videoID]).map((list) => { return list.category });
@@ -396,6 +406,45 @@ module.exports = async function postSkipSegments(req, res) {
startingVotes = 10000;
}
if (config.youtubeAPIKey !== null) {
let {err, data} = await new Promise((resolve, reject) => {
YouTubeAPI.listVideos(videoID, (err, data) => resolve({err, data}));
});
//get all segments for this video and user
let allSubmittedByUser = db.prepare('all', "SELECT startTime, endTime FROM sponsorTimes WHERE userID = ? and videoID = ? and votes > -1", [userID, videoID]);
let allSegmentTimes = [];
if (allSubmittedByUser !== undefined) {
//add segments the user has previously submitted
for (const segmentInfo of allSubmittedByUser) {
allSegmentTimes.push([parseFloat(segmentInfo.startTime), parseFloat(segmentInfo.endTime)])
}
}
//add segments they are trying to add in this submission
for (let i = 0; i < segments.length; i++) {
let startTime = parseFloat(segments[i].segment[0]);
let endTime = parseFloat(segments[i].segment[1]);
allSegmentTimes.push([startTime, endTime]);
}
//merge all the times into non-overlapping arrays
const allSegmentsSorted = mergeTimeSegments(allSegmentTimes.sort(function(a, b) { return a[0]-b[0] || a[1]-b[1] }));
let videoDuration = data.items[0].contentDetails.duration;
videoDuration = isoDurations.toSeconds(isoDurations.parse(videoDuration));
if (videoDuration != 0) {
let allSegmentDuration = 0;
//sum all segment times together
allSegmentsSorted.forEach(segmentInfo => allSegmentDuration += segmentInfo[1] - segmentInfo[0]);
if (allSegmentDuration > (videoDuration/100)*80) {
// Reject submission if all segments combine are over 80% of the video
res.status(400).send("Total length of your submitted segments are over 80% of the video.");
return;
}
}
}
for (const segmentInfo of segments) {
//this can just be a hash of the data
//it's better than generating an actual UUID like what was used before
@@ -437,3 +486,30 @@ module.exports = async function postSkipSegments(req, res) {
sendWebhooks(userID, videoID, UUIDs[i], segments[i]);
}
}
// Takes an array of arrays:
// ex)
// [
// [3, 40],
// [50, 70],
// [60, 80],
// [100, 150]
// ]
// => transforms to combining overlapping segments
// [
// [3, 40],
// [50, 80],
// [100, 150]
// ]
function mergeTimeSegments(ranges) {
var result = [], last;
ranges.forEach(function (r) {
if (!last || r[0] > last[1])
result.push(last = r);
else if (r[1] > last[1])
last[1] = r[1];
});
return result;
}

24
src/routes/postWarning.js Normal file
View File

@@ -0,0 +1,24 @@
const db = require('../databases/databases.js').db;
const getHash = require('../utils/getHash.js');
const isUserVIP = require('../utils/isUserVIP.js');
const logger = require('../utils/logger.js');
module.exports = (req, res) => {
// Collect user input data
let issuerUserID = getHash(req.body.issuerUserID);
let userID = getHash(req.body.userID);
let issueTime = new Date().getTime();
// Ensure user is a VIP
if (!isUserVIP(issuerUserID)) {
logger.debug("Permission violation: User " + issuerUserID + " attempted to warn user " + userID + "."); // maybe warn?
res.status(403).json({"message": "Not a VIP"});
return;
}
db.prepare('run', 'INSERT INTO warnings (userID, issueTime, issuerUserID) VALUES (?, ?, ?)', [userID, issueTime, issuerUserID]);
res.status(200).json({
message: "Warning issued to user '" + userID + "'."
});
};

12
src/routes/voteOnSponsorTime.js
View File

@@ -49,7 +49,7 @@ function sendWebhooks(voteData) {
}
if (config.youtubeAPIKey !== null) {
YouTubeAPI.listVideos(submissionInfoRow.videoID, "snippet", (err, data) => {
YouTubeAPI.listVideos(submissionInfoRow.videoID, (err, data) => {
if (err || data.items.length === 0) {
err && logger.error(err);
return;
@@ -247,6 +247,16 @@ async function voteOnSponsorTime(req, res) {
return;
}
}
const MILLISECONDS_IN_HOUR = 3600000;
const now = Date.now();
let warningsCount = db.prepare('get', "SELECT count(1) as count FROM warnings WHERE userID = ? AND issueTime > ?",
[nonAnonUserID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))]
).count;
if (warningsCount >= config.maxNumberOfActiveWarnings) {
return res.status(403).send('Vote blocked. Too many active warnings!');
}
let voteTypeEnum = (type == 0 || type == 1) ? voteTypes.normal : voteTypes.incorrect;