From 7b413dcbb28ad0de575c1a6bdad2696e1a3f51a2 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 27 Oct 2019 17:57:53 -0400 Subject: [PATCH 1/4] Made db paths use the config --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 316d1de..7156a22 100644 --- a/index.js +++ b/index.js @@ -9,9 +9,9 @@ var crypto = require('crypto'); //load database var sqlite3 = require('sqlite3').verbose(); -var db = new sqlite3.Database('./databases/sponsorTimes.db'); +var db = new sqlite3.Database(config.db); //where the more sensitive data such as IP addresses are stored -var privateDB = new sqlite3.Database('./databases/private.db'); +var privateDB = new sqlite3.Database(config.privateDB); let config = JSON.parse(fs.readFileSync('config.json')); From e87f804070f5ff13e7d640786cde18271d376ee8 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Sun, 27 Oct 2019 18:02:12 -0400 Subject: [PATCH 2/4] Fixed config defined too late --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 7156a22..44a1b44 100644 --- a/index.js +++ b/index.js @@ -7,14 +7,14 @@ var app = express(); //hashing service var crypto = require('crypto'); +let config = JSON.parse(fs.readFileSync('config.json')); + //load database var sqlite3 = require('sqlite3').verbose(); var db = new sqlite3.Database(config.db); //where the more sensitive data such as IP addresses are stored var privateDB = new sqlite3.Database(config.privateDB); -let config = JSON.parse(fs.readFileSync('config.json')); - // Create an HTTP service. http.createServer(app).listen(config.port); From 4b47769f61a7d1d0e7b3a4643b198006eed6f7ee Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Mon, 28 Oct 2019 16:37:08 -0400 Subject: [PATCH 3/4] Added ability to enable read only mode from the config. --- config.json.example | 6 +++++- index.js | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config.json.example b/config.json.example index 9bdc16b..76eb8e0 100644 --- a/config.json.example +++ b/config.json.example @@ -2,5 +2,9 @@ "port": 80, "globalSalt": "[global salt (pepper) that is added to every ip before hashing to make it even harder for someone to decode the ip]", "adminUserID": "[the hashed id of the user who can perform admin actions]", - "behindProxy": true + "behindProxy": true, + "db": "./databases/sponsorTimes.db", + "privateDB": "./databases/private.db", + "mode": "development", + "readOnly": false } \ No newline at end of file diff --git a/index.js b/index.js index 44a1b44..2bf1333 100644 --- a/index.js +++ b/index.js @@ -9,11 +9,17 @@ var crypto = require('crypto'); let config = JSON.parse(fs.readFileSync('config.json')); -//load database var sqlite3 = require('sqlite3').verbose(); -var db = new sqlite3.Database(config.db); + +let dbMode = sqlite3.OPEN_READWRITE; +if (config.readOnly) { + dbMode = sqlite3.OPEN_READONLY; +} + +//load database +var db = new sqlite3.Database(config.db, dbMode); //where the more sensitive data such as IP addresses are stored -var privateDB = new sqlite3.Database(config.privateDB); +var privateDB = new sqlite3.Database(config.privateDB, dbMode); // Create an HTTP service. http.createServer(app).listen(config.port); From 17f7e618eccee1a6f6585f600e16863791bfc594 Mon Sep 17 00:00:00 2001 From: Ajay Ramachandran Date: Tue, 29 Oct 2019 17:29:31 -0400 Subject: [PATCH 4/4] Added all db write methods to a queue. Should resolve DB busy crash. --- index.js | 21 ++++++++++------ writeQueue.js | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 writeQueue.js diff --git a/index.js b/index.js index 2bf1333..760544a 100644 --- a/index.js +++ b/index.js @@ -4,6 +4,9 @@ var http = require('http'); // Create a service (the app object is just a callback). var app = express(); +//used to prevent database is busy errors +var writeQueue = require('./writeQueue'); + //hashing service var crypto = require('crypto'); @@ -196,7 +199,8 @@ app.get('/api/postVideoSponsorTimes', async function (req, res) { if (row == null) { //not a duplicate, execute query - db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)").run(videoID, startTime, endTime, startingVotes, UUID, userID, timeSubmitted, 0, shadowBanned, function (err) { + let preparedStatement = db.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"); + let callback = function (err) { if (err) { //a DB change probably occurred, respond as if it is a duplicate res.sendStatus(409); @@ -204,11 +208,14 @@ app.get('/api/postVideoSponsorTimes', async function (req, res) { console.log("Error when putting sponsorTime in the DB: " + videoID + ", " + startTime + ", " + "endTime" + ", " + userID); } else { //add to private db as well - privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)").run(videoID, hashedIP, timeSubmitted); + writeQueue.addToWriteQueue(new writeQueue.WriteQueue(privateDB.prepare("INSERT INTO sponsorTimes VALUES(?, ?, ?)")), [videoID, hashedIP, timeSubmitted]); res.sendStatus(200); } - }); + }; + + writeQueue.addToWriteQueue(new writeQueue.WriteQueue(preparedStatement, [videoID, startTime, endTime, startingVotes, UUID, userID, timeSubmitted, 0, shadowBanned], callback)); + } else { res.sendStatus(409); } @@ -304,7 +311,7 @@ app.get('/api/voteOnSponsorTime', function (req, res) { if (votesRow != undefined) { privateDB.prepare("UPDATE votes SET type = ? WHERE userID = ? AND UUID = ?").run(type, userID, UUID); } else { - privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)").run(UUID, userID, hashedIP, type); + writeQueue.addToWriteQueue(new writeQueue.WriteQueue(privateDB.prepare("INSERT INTO votes VALUES(?, ?, ?, ?)")), [UUID, userID, hashedIP, type]); } //update the vote count on this sponsorTime @@ -393,7 +400,7 @@ app.post('/api/setUsername', function (req, res) { db.prepare("UPDATE userNames SET userName = ? WHERE userID = ?").run(userName, userID); } else { //add to the db - db.prepare("INSERT INTO userNames VALUES(?, ?)").run(userID, userName); + writeQueue.addToWriteQueue(new writeQueue.WriteQueue(db.prepare("INSERT INTO userNames VALUES(?, ?)")), [userID, userName]); } res.sendStatus(200); @@ -473,7 +480,7 @@ app.post('/api/shadowBanUser', async function (req, res) { //add them to the shadow ban list //add it to the table - privateDB.prepare("INSERT INTO shadowBannedUsers VALUES(?)").run(userID); + writeQueue.addToWriteQueue(new writeQueue.WriteQueue(privateDB.prepare("INSERT INTO shadowBannedUsers VALUES(?)")), [userID]); //find all previous submissions and hide them db.prepare("UPDATE sponsorTimes SET shadowHidden = 1 WHERE userID = ?").run(userID); @@ -524,7 +531,7 @@ app.post('/api/addUserAsVIP', async function (req, res) { if (enabled && result.row.userCount == 0) { //add them to the vip list - db.prepare("INSERT INTO vipUsers VALUES(?)").run(userID); + writeQueue.addToWriteQueue(new writeQueue.WriteQueue(db.prepare("INSERT INTO vipUsers VALUES(?)")), [userID]); } else if (!enabled && result.row.userCount > 0) { //remove them from the shadow ban list db.prepare("DELETE FROM vipUsers WHERE userID = ?").run(userID); diff --git a/writeQueue.js b/writeQueue.js new file mode 100644 index 0000000..c7b33b9 --- /dev/null +++ b/writeQueue.js @@ -0,0 +1,69 @@ +'use strict'; + +//used to queue objects to be written to the db +class WriteQueue { + /** + * @param {Statement} statement + * @param {Array} inputs + * @param {callback} callback + */ + constructor(statement, inputs, callback) { + this.statement = statement; + this.inputs = inputs; + this.callback = callback; + } + + run() { + return new Promise((resolve, reject) => { + this.statement.run(this.inputs, (err) => this.end(err, resolve, reject)); + }); + } + + end(err, resolve, reject) { + resolve(); + + if (this.callback) { + this.callback(err); + } + } +} + +module.exports = { + WriteQueue, + addToWriteQueue +} + +/** + * Array of class write queue + * + * @typedef WriteQueue[] + */ +var dbQueue = []; + +//is a queue check currently running +var queueRunning = false; + +/** + * Adds an item to the write queue and starts the run function if needed. + * + * @param {WriteQueue} item + */ +function addToWriteQueue(item) { + dbQueue.push(item); + + if (!queueRunning) { + runThroughWriteQueue(); + } +} + +async function runThroughWriteQueue() { + queueRunning = true; + + while (dbQueue.length > 0) { + await dbQueue[0].run(); + + dbQueue.splice(0, 1); + } + + queueRunning = false; +} \ No newline at end of file