From d4695f0192b2a55e7ca846b2965f0cb28b2e1700 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 25 Jun 2021 03:33:41 -0400 Subject: [PATCH 01/10] add getUserID and tests --- src/app.ts | 4 ++ src/routes/getUserID.ts | 29 +++++++++++ test/cases/getUserID.ts | 109 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/routes/getUserID.ts create mode 100644 test/cases/getUserID.ts diff --git a/src/app.ts b/src/app.ts index 71a85bd..e02953c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -32,6 +32,7 @@ import {endpoint as getSegmentInfo} from './routes/getSegmentInfo'; import {postClearCache} from './routes/postClearCache'; import { addUnlistedVideo } from './routes/addUnlistedVideo'; import {postPurgeAllSegments} from './routes/postPurgeAllSegments'; +import {getUserID} from './routes/getUserID'; export function createServer(callback: () => void) { // Create a service (the app object is just a callback). @@ -146,6 +147,9 @@ function setupRoutes(app: Express) { app.post('/api/purgeAllSegments', postPurgeAllSegments); app.post('/api/unlistedVideo', addUnlistedVideo); + + // get userID from username + app.get('/api/userID', getUserID); if (config.postgres) { app.get('/database', (req, res) => dumpDatabase(req, res, true)); diff --git a/src/routes/getUserID.ts b/src/routes/getUserID.ts new file mode 100644 index 0000000..7e7317e --- /dev/null +++ b/src/routes/getUserID.ts @@ -0,0 +1,29 @@ +import {db} from '../databases/databases'; +import {Logger} from '../utils/logger'; +import {Request, Response} from 'express'; + +export async function getUserID(req: Request, res: Response) { + let username = req.query.username as string; + + if (username == undefined) { + //invalid request + res.sendStatus(400); + return; + } + + // add wildcard to variable + username = `%${username}%` + try { + let rows = await db.prepare('all', `SELECT "userName", "userID" FROM "userNames" WHERE "userName" LIKE ?`, [username]); + if (rows.length === 0) { + res.sendStatus(404); + return; + } else { + res.send(rows); + } + } catch (err) { + Logger.error(err); + res.sendStatus(500); + return; + } +} diff --git a/test/cases/getUserID.ts b/test/cases/getUserID.ts new file mode 100644 index 0000000..4403427 --- /dev/null +++ b/test/cases/getUserID.ts @@ -0,0 +1,109 @@ +import fetch from 'node-fetch'; +import {Done, getbaseURL} from '../utils'; +import {db} from '../../src/databases/databases'; +import {getHash} from '../../src/utils/getHash'; + +describe('getUserID', () => { + before(async () => { + const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)'; + await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_01"), 'fuzzy user 01']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_02"), 'fuzzy user 02']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_03"), 'specific user 03']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_04"), 'repeating']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_05"), 'repeating']); + }); + + it('Should be able to get a 200', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=fuzzy+user+01') + .then(async res => { + const text = await res.text() + if (res.status !== 200) done('non 200 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => done('couldn\'t call endpoint')); + }); + + it('Should be able to get a 400 (No username parameter)', (done: Done) => { + fetch(getbaseURL() + '/api/userID') + .then(res => { + if (res.status !== 400) done('non 400 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => done('couldn\'t call endpoint')); + }); + + it('Should be able to get single username', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=fuzzy+user+01') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 1) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "fuzzy user 01") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserinfo_user_01")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should be able to get multiple fuzzy user info from start', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=fuzzy+user') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 2) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "fuzzy user 01") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserinfo_user_01")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[1].userName !== "fuzzy user 02") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[1].userID !== getHash("getuserinfo_user_02")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should be able to get multiple fuzzy user info from middle', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=user') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 3) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "fuzzy user 01") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserinfo_user_01")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[1].userName !== "fuzzy user 02") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[1].userID !== getHash("getuserinfo_user_02")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[2].userName !== "specific user 03") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[2].userID !== getHash("getuserinfo_user_03")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); +}); From f29bafe89a188871a62948e7126da09d12eb99d8 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 25 Jun 2021 03:37:27 -0400 Subject: [PATCH 02/10] fiix tests --- test/cases/getUserID.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/cases/getUserID.ts b/test/cases/getUserID.ts index 4403427..cb3b8d9 100644 --- a/test/cases/getUserID.ts +++ b/test/cases/getUserID.ts @@ -6,11 +6,11 @@ import {getHash} from '../../src/utils/getHash'; describe('getUserID', () => { before(async () => { const insertUserNameQuery = 'INSERT INTO "userNames" ("userID", "userName") VALUES(?, ?)'; - await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_01"), 'fuzzy user 01']); - await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_02"), 'fuzzy user 02']); - await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_03"), 'specific user 03']); - await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_04"), 'repeating']); - await db.prepare("run", insertUserNameQuery, [getHash("getuserinfo_user_05"), 'repeating']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_01"), 'fuzzy user 01']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_02"), 'fuzzy user 02']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_03"), 'specific user 03']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_04"), 'repeating']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_05"), 'repeating']); }); it('Should be able to get a 200', (done: Done) => { @@ -43,7 +43,7 @@ describe('getUserID', () => { done('Returned incorrect number of users "' + data.length + '"'); } else if (data[0].userName !== "fuzzy user 01") { done('Returned incorrect username "' + data.userName + '"'); - } else if (data[0].userID !== getHash("getuserinfo_user_01")) { + } else if (data[0].userID !== getHash("getuserid_user_01")) { done('Returned incorrect userID "' + data.userID + '"'); } else { done(); // pass @@ -64,11 +64,11 @@ describe('getUserID', () => { done('Returned incorrect number of users "' + data.length + '"'); } else if (data[0].userName !== "fuzzy user 01") { done('Returned incorrect username "' + data.userName + '"'); - } else if (data[0].userID !== getHash("getuserinfo_user_01")) { + } else if (data[0].userID !== getHash("getuserid_user_01")) { done('Returned incorrect userID "' + data.userID + '"'); } else if (data[1].userName !== "fuzzy user 02") { done('Returned incorrect username "' + data.userName + '"'); - } else if (data[1].userID !== getHash("getuserinfo_user_02")) { + } else if (data[1].userID !== getHash("getuserid_user_02")) { done('Returned incorrect userID "' + data.userID + '"'); } else { done(); // pass @@ -89,15 +89,15 @@ describe('getUserID', () => { done('Returned incorrect number of users "' + data.length + '"'); } else if (data[0].userName !== "fuzzy user 01") { done('Returned incorrect username "' + data.userName + '"'); - } else if (data[0].userID !== getHash("getuserinfo_user_01")) { + } else if (data[0].userID !== getHash("getuserid_user_01")) { done('Returned incorrect userID "' + data.userID + '"'); } else if (data[1].userName !== "fuzzy user 02") { done('Returned incorrect username "' + data.userName + '"'); - } else if (data[1].userID !== getHash("getuserinfo_user_02")) { + } else if (data[1].userID !== getHash("getuserid_user_02")) { done('Returned incorrect userID "' + data.userID + '"'); } else if (data[2].userName !== "specific user 03") { done('Returned incorrect username "' + data.userName + '"'); - } else if (data[2].userID !== getHash("getuserinfo_user_03")) { + } else if (data[2].userID !== getHash("getuserid_user_03")) { done('Returned incorrect userID "' + data.userID + '"'); } else { done(); // pass From 09ab1dabdf25c9243cc93cc41abbb52a67c6701f Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 25 Jun 2021 11:57:27 -0400 Subject: [PATCH 03/10] set limit of 64 characters for lookup --- src/routes/getUserID.ts | 2 +- test/cases/getUserID.ts | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/routes/getUserID.ts b/src/routes/getUserID.ts index 7e7317e..5e96491 100644 --- a/src/routes/getUserID.ts +++ b/src/routes/getUserID.ts @@ -5,7 +5,7 @@ import {Request, Response} from 'express'; export async function getUserID(req: Request, res: Response) { let username = req.query.username as string; - if (username == undefined) { + if (username == undefined || username.length > 64) { //invalid request res.sendStatus(400); return; diff --git a/test/cases/getUserID.ts b/test/cases/getUserID.ts index cb3b8d9..24aabb3 100644 --- a/test/cases/getUserID.ts +++ b/test/cases/getUserID.ts @@ -11,6 +11,7 @@ describe('getUserID', () => { await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_03"), 'specific user 03']); await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_04"), 'repeating']); await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_05"), 'repeating']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_06"), getHash("getuserid_user_06")]); }); it('Should be able to get a 200', (done: Done) => { @@ -32,6 +33,25 @@ describe('getUserID', () => { .catch(err => done('couldn\'t call endpoint')); }); + it('Should be able to get a 200 (username is public id)', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username='+getHash("getuserid_user_06")) + .then(async res => { + const text = await res.text() + if (res.status !== 200) done('non 200 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => done('couldn\'t call endpoint')); + }); + + it('Should be able to get a 400 (username longer than 64 chars)', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username='+getHash("getuserid_user_06")+'0') + .then(res => { + if (res.status !== 400) done('non 400 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => done('couldn\'t call endpoint')); + }); + it('Should be able to get single username', (done: Done) => { fetch(getbaseURL() + '/api/userID?username=fuzzy+user+01') .then(async res => { From f2490beea245690b408f6667536ffc3b205ef85c Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 25 Jun 2021 14:35:51 -0400 Subject: [PATCH 04/10] put in limits and escapes --- src/routes/getUserID.ts | 14 ++++++--- test/cases/getUserID.ts | 69 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/src/routes/getUserID.ts b/src/routes/getUserID.ts index 5e96491..480280e 100644 --- a/src/routes/getUserID.ts +++ b/src/routes/getUserID.ts @@ -3,18 +3,24 @@ import {Logger} from '../utils/logger'; import {Request, Response} from 'express'; export async function getUserID(req: Request, res: Response) { - let username = req.query.username as string; + let userName = req.query.username as string; - if (username == undefined || username.length > 64) { + if (userName == undefined || userName.length > 64 || userName.length < 3) { //invalid request res.sendStatus(400); return; } + // escape [_ % \] to avoid ReDOS + userName = userName.replace('\\', '\\\\') + .replace('_', '\\_') + .replace('%', '\\%') + // add wildcard to variable - username = `%${username}%` + userName = `%${userName}%` try { - let rows = await db.prepare('all', `SELECT "userName", "userID" FROM "userNames" WHERE "userName" LIKE ?`, [username]); + let rows = await db.prepare('all', `SELECT "userName", "userID" FROM "userNames" + WHERE "userName" LIKE ? LIMIT 10`, [userName]); if (rows.length === 0) { res.sendStatus(404); return; diff --git a/test/cases/getUserID.ts b/test/cases/getUserID.ts index 24aabb3..f51f353 100644 --- a/test/cases/getUserID.ts +++ b/test/cases/getUserID.ts @@ -126,4 +126,73 @@ describe('getUserID', () => { }) .catch(err => ("couldn't call endpoint")); }); + + it('Should be able to get with public ID', (done: Done) => { + const userID = getHash("getuserid_user_06") + fetch(getbaseURL() + '/api/userID?username='+userID) + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 1) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== userID) { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== userID) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should be able to get with fuzzy public ID', (done: Done) => { + const userID = getHash("getuserid_user_06") + fetch(getbaseURL() + '/api/userID?username='+userID.substr(10,60)) + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 1) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== userID) { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== userID) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should be able to get repeating username', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=repeating') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 2) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "repeating") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_04")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[1].userName !== "repeating") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[1].userID !== getHash("getuserid_user_05")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); }); From b06a6fbb5106329799b96fefdb581529b0d20ad4 Mon Sep 17 00:00:00 2001 From: Michael C Date: Fri, 25 Jun 2021 15:57:41 -0400 Subject: [PATCH 05/10] redos prevention --- src/routes/getUserID.ts | 14 +++-- test/cases/getSegmentInfo.ts | 28 ++++----- test/cases/getUserID.ts | 108 ++++++++++++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 22 deletions(-) diff --git a/src/routes/getUserID.ts b/src/routes/getUserID.ts index 480280e..b450e47 100644 --- a/src/routes/getUserID.ts +++ b/src/routes/getUserID.ts @@ -12,15 +12,17 @@ export async function getUserID(req: Request, res: Response) { } // escape [_ % \] to avoid ReDOS - userName = userName.replace('\\', '\\\\') - .replace('_', '\\_') - .replace('%', '\\%') - + userName = userName.replace(/\\/g, '\\\\') + .replace(/_/g, '\\_') + .replace(/%/g, '\\%'); + // add wildcard to variable - userName = `%${userName}%` + userName = `%${userName}%`; + // LIMIT to reduce overhead + // ESCAPE to escape LIKE wildcards try { let rows = await db.prepare('all', `SELECT "userName", "userID" FROM "userNames" - WHERE "userName" LIKE ? LIMIT 10`, [userName]); + WHERE "userName" LIKE ? ESCAPE '\\' LIMIT 10`, [userName]); if (rows.length === 0) { res.sendStatus(404); return; diff --git a/test/cases/getSegmentInfo.ts b/test/cases/getSegmentInfo.ts index de6e1a3..58c61e2 100644 --- a/test/cases/getSegmentInfo.ts +++ b/test/cases/getSegmentInfo.ts @@ -3,20 +3,20 @@ import {db} from '../../src/databases/databases'; import {Done, getbaseURL} from '../utils'; import {getHash} from '../../src/utils/getHash'; -const ENOENTID = "0000000000000000000000000000000000000000000000000000000000000000" -const upvotedID = "a000000000000000000000000000000000000000000000000000000000000000" -const downvotedID = "b000000000000000000000000000000000000000000000000000000000000000" -const lockedupID = "c000000000000000000000000000000000000000000000000000000000000000" -const infvotesID = "d000000000000000000000000000000000000000000000000000000000000000" -const shadowhiddenID = "e000000000000000000000000000000000000000000000000000000000000000" -const lockeddownID = "f000000000000000000000000000000000000000000000000000000000000000" -const hiddenID = "1000000000000000000000000000000000000000000000000000000000000000" -const fillerID1 = "1100000000000000000000000000000000000000000000000000000000000000" -const fillerID2 = "1200000000000000000000000000000000000000000000000000000000000000" -const fillerID3 = "1300000000000000000000000000000000000000000000000000000000000000" -const fillerID4 = "1400000000000000000000000000000000000000000000000000000000000000" -const fillerID5 = "1500000000000000000000000000000000000000000000000000000000000000" -const oldID = "a0000000-0000-0000-0000-000000000000" +const ENOENTID = "0".repeat(64); +const upvotedID = "a"+"0".repeat(63); +const downvotedID = "b"+"0".repeat(63); +const lockedupID = "c"+"0".repeat(63); +const infvotesID = "d"+"0".repeat(63); +const shadowhiddenID = "e"+"0".repeat(63); +const lockeddownID = "f"+"0".repeat(63); +const hiddenID = "1"+"0".repeat(63); +const fillerID1 = "11"+"0".repeat(62); +const fillerID2 = "12"+"0".repeat(62); +const fillerID3 = "13"+"0".repeat(62); +const fillerID4 = "14"+"0".repeat(62); +const fillerID5 = "15"+"0".repeat(62); +const oldID = `${'0'.repeat(8)}-${'0000-'.repeat(3)}${'0'.repeat(12)}`; describe('getSegmentInfo', () => { before(async () => { diff --git a/test/cases/getUserID.ts b/test/cases/getUserID.ts index f51f353..21c27f8 100644 --- a/test/cases/getUserID.ts +++ b/test/cases/getUserID.ts @@ -12,6 +12,11 @@ describe('getUserID', () => { await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_04"), 'repeating']); await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_05"), 'repeating']); await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_06"), getHash("getuserid_user_06")]); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_07"), '0redos0']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_08"), '%redos%']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_09"), '_redos_']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_10"), 'redos\\%']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_11"), '\\\\\\']); }); it('Should be able to get a 200', (done: Done) => { @@ -128,7 +133,7 @@ describe('getUserID', () => { }); it('Should be able to get with public ID', (done: Done) => { - const userID = getHash("getuserid_user_06") + const userID = getHash("getuserid_user_06"); fetch(getbaseURL() + '/api/userID?username='+userID) .then(async res => { if (res.status !== 200) { @@ -150,7 +155,7 @@ describe('getUserID', () => { }); it('Should be able to get with fuzzy public ID', (done: Done) => { - const userID = getHash("getuserid_user_06") + const userID = getHash("getuserid_user_06"); fetch(getbaseURL() + '/api/userID?username='+userID.substr(10,60)) .then(async res => { if (res.status !== 200) { @@ -195,4 +200,103 @@ describe('getUserID', () => { }) .catch(err => ("couldn't call endpoint")); }); + + it('should avoid ReDOS with _', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=_redos_') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 1) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "_redos_") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_09")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('should avoid ReDOS with %', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=%redos%') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 1) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "%redos%") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_08")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('should return 404 if escaped backslashes present', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=%redos\\\\_') + .then(res => { + if (res.status !== 404) done('non 404 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('should return 404 if backslashes present', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=\\%redos\\_') + .then(res => { + if (res.status !== 404) done('non 404 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('should return user if just backslashes', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=\\\\\\') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 1) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "\\\\\\") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_11")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('should not allow usernames more than 64 characters', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username='+'0'.repeat(65)) + .then(res => { + if (res.status !== 400) done('non 400 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('should not allow usernames less than 3 characters', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=aa') + .then(res => { + if (res.status !== 400) done('non 400 (' + res.status + ')'); + else done(); // pass + }) + .catch(err => ("couldn't call endpoint")); + }); }); From 2f50d80a7525ecde94d5ece8724bb31d1ab5bceb Mon Sep 17 00:00:00 2001 From: Michael C Date: Sat, 26 Jun 2021 23:02:52 -0400 Subject: [PATCH 06/10] add explit param --- src/routes/getUserID.ts | 70 +++++++++++++++++----------- test/cases/getUserID.ts | 101 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 26 deletions(-) diff --git a/src/routes/getUserID.ts b/src/routes/getUserID.ts index b450e47..f4c9986 100644 --- a/src/routes/getUserID.ts +++ b/src/routes/getUserID.ts @@ -1,37 +1,55 @@ import {db} from '../databases/databases'; -import {Logger} from '../utils/logger'; import {Request, Response} from 'express'; +import {UserID} from '../types/user.model'; -export async function getUserID(req: Request, res: Response) { - let userName = req.query.username as string; - - if (userName == undefined || userName.length > 64 || userName.length < 3) { - //invalid request - res.sendStatus(400); - return; - } - +function getFuzzyUserID(userName: String): Promise<[{userName: String, userID: UserID }]> { // escape [_ % \] to avoid ReDOS userName = userName.replace(/\\/g, '\\\\') .replace(/_/g, '\\_') .replace(/%/g, '\\%'); - - // add wildcard to variable - userName = `%${userName}%`; - // LIMIT to reduce overhead - // ESCAPE to escape LIKE wildcards + userName = `%${userName}%`; // add wildcard to username + // LIMIT to reduce overhead | ESCAPE to escape LIKE wildcards try { - let rows = await db.prepare('all', `SELECT "userName", "userID" FROM "userNames" - WHERE "userName" LIKE ? ESCAPE '\\' LIMIT 10`, [userName]); - if (rows.length === 0) { - res.sendStatus(404); - return; - } else { - res.send(rows); - } + return db.prepare('all', `SELECT "userName", "userID" FROM "userNames" WHERE "userName" + LIKE ? ESCAPE '\\' LIMIT 10`, [userName]) } catch (err) { - Logger.error(err); - res.sendStatus(500); - return; + return null; + } +} + +function getExactUserID(userName: String): Promise<[{userName: String, userID: UserID }]> { + try { + return db.prepare('all', `SELECT "userName", "userID" from "userNames" WHERE "userName" = ? LIMIT 10`, [userName]); + } catch (err) { + return null; + } +} + +export async function getUserID(req: Request, res: Response) { + let userName = req.query.username as string; + const exactSearch = req.query.exact + ? req.query.exact == "true" + : false as Boolean; + + // if not exact and length is 1, also skip + if (userName == undefined || userName.length > 64 || + (!exactSearch && userName.length < 3)) { + // invalid request + res.sendStatus(400); + return false; + } + const results = exactSearch + ? await getExactUserID(userName) + : await getFuzzyUserID(userName); + + if (results === undefined || results === null) { + res.sendStatus(500); + return false; + } else if (results.length as number === 0) { + res.sendStatus(404); + return false; + } else { + res.send(results); + return false; } } diff --git a/test/cases/getUserID.ts b/test/cases/getUserID.ts index 21c27f8..7fe5d70 100644 --- a/test/cases/getUserID.ts +++ b/test/cases/getUserID.ts @@ -17,6 +17,7 @@ describe('getUserID', () => { await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_09"), '_redos_']); await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_10"), 'redos\\%']); await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_11"), '\\\\\\']); + await db.prepare("run", insertUserNameQuery, [getHash("getuserid_user_12"), 'a']); }); it('Should be able to get a 200', (done: Done) => { @@ -201,6 +202,31 @@ describe('getUserID', () => { .catch(err => ("couldn't call endpoint")); }); + it('Should be able to get repeating fuzzy username', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=peat') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 2) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "repeating") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_04")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[1].userName !== "repeating") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[1].userID !== getHash("getuserid_user_05")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + it('should avoid ReDOS with _', (done: Done) => { fetch(getbaseURL() + '/api/userID?username=_redos_') .then(async res => { @@ -299,4 +325,79 @@ describe('getUserID', () => { }) .catch(err => ("couldn't call endpoint")); }); + + it('should allow exact match', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=a&exact=true') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 1) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "a") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_12")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should be able to get repeating username with exact username', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=repeating&exact=true') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 2) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "repeating") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_04")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[1].userName !== "repeating") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[1].userID !== getHash("getuserid_user_05")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); + + it('Should not get exact unless explicitly set to true', (done: Done) => { + fetch(getbaseURL() + '/api/userID?username=user&exact=1') + .then(async res => { + if (res.status !== 200) { + done("non 200"); + } else { + const data = await res.json(); + if (data.length !== 3) { + done('Returned incorrect number of users "' + data.length + '"'); + } else if (data[0].userName !== "fuzzy user 01") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[0].userID !== getHash("getuserid_user_01")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[1].userName !== "fuzzy user 02") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[1].userID !== getHash("getuserid_user_02")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else if (data[2].userName !== "specific user 03") { + done('Returned incorrect username "' + data.userName + '"'); + } else if (data[2].userID !== getHash("getuserid_user_03")) { + done('Returned incorrect userID "' + data.userID + '"'); + } else { + done(); // pass + } + } + }) + .catch(err => ("couldn't call endpoint")); + }); }); From f5bb221ecdcaa40948df6a12ce94c793b5d099ac Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Sun, 27 Jun 2021 11:45:42 +0700 Subject: [PATCH 07/10] Add log when update username --- databases/_upgrade_private_2.sql | 13 ++++++++ src/routes/setUsername.ts | 17 ++++++++-- test/cases/setUsername.ts | 57 +++++++++++++++++++++++++++----- 3 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 databases/_upgrade_private_2.sql diff --git a/databases/_upgrade_private_2.sql b/databases/_upgrade_private_2.sql new file mode 100644 index 0000000..1a70e67 --- /dev/null +++ b/databases/_upgrade_private_2.sql @@ -0,0 +1,13 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "userNameLogs" ( + "userID" TEXT NOT NULL, + "newUserName" TEXT NOT NULL, + "oldUserName" TEXT NOT NULL, + "updatedByAdmin" BOOLEAN NOT NULL, + "updatedAt" INTEGER NOT NULL +); + +UPDATE "config" SET value = 2 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/src/routes/setUsername.ts b/src/routes/setUsername.ts index fdd99dc..2a0e75b 100644 --- a/src/routes/setUsername.ts +++ b/src/routes/setUsername.ts @@ -1,9 +1,16 @@ import {config} from '../config'; import {Logger} from '../utils/logger'; -import {db} from '../databases/databases'; +import {db, privateDB} from '../databases/databases'; import {getHash} from '../utils/getHash'; import {Request, Response} from 'express'; +async function logUserNameChange(userID: string, newUserName: string, oldUserName: string, updatedByAdmin: boolean): Promise { + return privateDB.prepare('run', + `INSERT INTO "userNameLogs"("userID", "newUserName", "oldUserName", "updatedByAdmin", "updatedAt") VALUES(?, ?, ?, ?, ?)`, + [userID, newUserName, oldUserName, + updatedByAdmin, new Date().getTime()] + ); +} + export async function setUsername(req: Request, res: Response) { let userID = req.query.userID as string; let userName = req.query.username as string; @@ -55,16 +62,20 @@ export async function setUsername(req: Request, res: Response) { try { //check if username is already set - let row = await db.prepare('get', `SELECT count(*) as count FROM "userNames" WHERE "userID" = ?`, [userID]); + let row = await db.prepare('get', `SELECT userName FROM "userNames" WHERE "userID" = ? LIMIT 1`, [userID]); + let oldUserName = ''; - if (row.count > 0) { + if (row.userName && row.userName.length !== 0) { //already exists, update this row + oldUserName = row.userName; await db.prepare('run', `UPDATE "userNames" SET "userName" = ? WHERE "userID" = ?`, [userName, userID]); } else { //add to the db await db.prepare('run', `INSERT INTO "userNames"("userID", "userName") VALUES(?, ?)`, [userID, userName]); } + await logUserNameChange(userID, userName, oldUserName, adminUserIDInput !== undefined); + res.sendStatus(200); } catch (err) { Logger.error(err); diff --git a/test/cases/setUsername.ts b/test/cases/setUsername.ts index ae3a8f5..d82c121 100644 --- a/test/cases/setUsername.ts +++ b/test/cases/setUsername.ts @@ -1,6 +1,6 @@ import fetch from 'node-fetch'; import { Done, getbaseURL } from '../utils'; -import { db } from '../../src/databases/databases'; +import { db, privateDB } from '../../src/databases/databases'; import { getHash } from '../../src/utils/getHash'; const adminPrivateUserID = 'testUserId'; @@ -21,6 +21,7 @@ const username07 = 'Username 07'; async function addUsername(userID: string, userName: string, locked = 0) { await db.prepare('run', 'INSERT INTO "userNames" ("userID", "userName", "locked") VALUES(?, ?, ?)', [userID, userName, locked]); + await addLogUserNameChange(userID, userName); } async function getUsername(userID: string) { @@ -31,6 +32,40 @@ async function getUsername(userID: string) { return row.userName; } +async function addLogUserNameChange(userID: string, newUserName: string, oldUserName: string = '') { + privateDB.prepare('run', + `INSERT INTO "userNameLogs"("userID", "newUserName", "oldUserName", "updatedAt", "updatedByAdmin") VALUES(?, ?, ?, ?, ?)`, + [getHash(userID), newUserName, oldUserName, new Date().getTime(), + true] + ); +} + +async function getLastLogUserNameChange(userID: string) { + return privateDB.prepare('get', `SELECT * FROM "userNameLogs" WHERE "userID" = ? ORDER BY "updatedAt" DESC LIMIT 1`, [getHash(userID)]); +} + +function wellFormatUserName(userName: string) { + return userName.replace(/[\u0000-\u001F\u007F-\u009F]/g, ''); +} + +async function testUserNameChangelog(userID: string, newUserName: string, oldUserName: string, byAdmin: boolean, done: Done) { + + const log = await getLastLogUserNameChange(userID); + + if (newUserName !== log.newUserName) { + return done(`UserID '${userID}' incorrect log on newUserName: ${newUserName} !== ${log.newUserName}`); + } + + if (oldUserName !== log.oldUserName) { + return done(`UserID '${userID}' incorrect log on oldUserName: ${oldUserName} !== ${log.oldUserName}`); + } + + if (+byAdmin !== log.updatedByAdmin) { + return done(`UserID '${userID}' incorrect log on updatedByAdmin: ${byAdmin} !== ${log.updatedByAdmin}`); + } + + return done(); +} + describe('setUsername', () => { before(async () => { await addUsername(getHash(user01PrivateUserID), username01, 0); @@ -46,9 +81,11 @@ describe('setUsername', () => { fetch(`${getbaseURL()}/api/setUsername?userID=${user01PrivateUserID}&username=Changed%20Username`, { method: 'POST', }) - .then(res => { + .then(async res => { if (res.status !== 200) done(`Status code was ${res.status}`); - else done(); // pass + else { + testUserNameChangelog(user01PrivateUserID, decodeURIComponent('Changed%20Username'), username01, false, done); + } }) .catch(err => done(`couldn't call endpoint`)); }); @@ -113,7 +150,9 @@ describe('setUsername', () => { .then(async res => { const username = await getUsername(getHash(user03PrivateUserID)); if (username !== newUsername) done(`Username did not change`); - else done(); + else { + testUserNameChangelog(user03PrivateUserID, newUsername, username03, false, done); + } }) .catch(err => done(`couldn't call endpoint`)); }); @@ -139,7 +178,7 @@ describe('setUsername', () => { .then(async res => { const username = await getUsername(getHash(user05PrivateUserID)); if (username === newUsername) done(`Username contains unicode control characters`); - else done(); + else testUserNameChangelog(user05PrivateUserID, wellFormatUserName(newUsername), username05, false, done); }) .catch(err => done(`couldn't call endpoint`)); }); @@ -164,7 +203,7 @@ describe('setUsername', () => { .then(async res => { const username = await getUsername(getHash(user06PrivateUserID)); if (username !== newUsername) done(`Failed to change username from '${username06}' to '${newUsername}'`); - else done(); + else testUserNameChangelog(user06PrivateUserID, newUsername, username06, true, done); }) .catch(err => done(`couldn't call endpoint`)); }); @@ -175,9 +214,9 @@ describe('setUsername', () => { method: 'POST', }) .then(async res => { - const username = await getUsername(getHash(user06PrivateUserID)); - if (username !== newUsername) done(`Failed to change username from '${username06}' to '${newUsername}'`); - else done(); + const username = await getUsername(getHash(user07PrivateUserID)); + if (username !== newUsername) done(`Failed to change username from '${username07}' to '${newUsername}'`); + else testUserNameChangelog(user07PrivateUserID, newUsername, username07, true, done); }) .catch(err => done(`couldn't call endpoint`)); }); From f3542b74029631f45a9737e816a8b7f3b7bc3d87 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Sun, 27 Jun 2021 11:51:04 +0700 Subject: [PATCH 08/10] Fix column name for postgre --- src/routes/setUsername.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/setUsername.ts b/src/routes/setUsername.ts index 2a0e75b..dbcdd91 100644 --- a/src/routes/setUsername.ts +++ b/src/routes/setUsername.ts @@ -62,7 +62,7 @@ export async function setUsername(req: Request, res: Response) { try { //check if username is already set - let row = await db.prepare('get', `SELECT userName FROM "userNames" WHERE "userID" = ? LIMIT 1`, [userID]); + let row = await db.prepare('get', `SELECT "userName" FROM "userNames" WHERE "userID" = ? LIMIT 1`, [userID]); let oldUserName = ''; if (row.userName && row.userName.length !== 0) { From 41ba37c04eb46885ee8b8bdf26af9b78e24b3103 Mon Sep 17 00:00:00 2001 From: Haidang666 Date: Sun, 27 Jun 2021 11:57:39 +0700 Subject: [PATCH 09/10] Fix boolean comparison, Add log table to schema visual --- DatabaseSchema.md | 11 +++++++++++ test/cases/setUsername.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/DatabaseSchema.md b/DatabaseSchema.md index 32fdc6e..16aa6f8 100644 --- a/DatabaseSchema.md +++ b/DatabaseSchema.md @@ -3,6 +3,7 @@ [vipUsers](###vipUsers) [sponsorTimes](###sponsorTimes) [userNames](###userNames) +[userNameLogs](###userNameLogs) [categoryVotes](###categoryVotes) [lockCategories](###lockCategories) [warnings](###warnings) @@ -61,6 +62,16 @@ | -- | :--: | | userNames_userID | userID | +### userNameLogs + +| Name | Type | | +| -- | :--: | -- | +| userID | TEXT | not null | +| newUserName | TEXT | not null | +| oldUserName | TEXT | not null | +| updatedByAdmin | BOOLEAN | not null | +| updatedAt | INTEGER | not null | + ### categoryVotes | Name | Type | | diff --git a/test/cases/setUsername.ts b/test/cases/setUsername.ts index d82c121..286cee8 100644 --- a/test/cases/setUsername.ts +++ b/test/cases/setUsername.ts @@ -59,7 +59,7 @@ async function testUserNameChangelog(userID: string, newUserName: string, oldUse return done(`UserID '${userID}' incorrect log on oldUserName: ${oldUserName} !== ${log.oldUserName}`); } - if (+byAdmin !== log.updatedByAdmin) { + if (byAdmin !== Boolean(log.updatedByAdmin)) { return done(`UserID '${userID}' incorrect log on updatedByAdmin: ${byAdmin} !== ${log.updatedByAdmin}`); } From 33a45ce0a2447241bbfb898d2545d32c92fc493c Mon Sep 17 00:00:00 2001 From: Michael C Date: Sun, 27 Jun 2021 01:05:06 -0400 Subject: [PATCH 10/10] fix TS declarations --- src/routes/getUserID.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/getUserID.ts b/src/routes/getUserID.ts index f4c9986..6ecc1b7 100644 --- a/src/routes/getUserID.ts +++ b/src/routes/getUserID.ts @@ -2,7 +2,7 @@ import {db} from '../databases/databases'; import {Request, Response} from 'express'; import {UserID} from '../types/user.model'; -function getFuzzyUserID(userName: String): Promise<[{userName: String, userID: UserID }]> { +function getFuzzyUserID(userName: String): Promise<{userName: String, userID: UserID }[]> { // escape [_ % \] to avoid ReDOS userName = userName.replace(/\\/g, '\\\\') .replace(/_/g, '\\_') @@ -17,7 +17,7 @@ function getFuzzyUserID(userName: String): Promise<[{userName: String, userID: U } } -function getExactUserID(userName: String): Promise<[{userName: String, userID: UserID }]> { +function getExactUserID(userName: String): Promise<{userName: String, userID: UserID }[]> { try { return db.prepare('all', `SELECT "userName", "userID" from "userNames" WHERE "userName" = ? LIMIT 10`, [userName]); } catch (err) { @@ -45,7 +45,7 @@ export async function getUserID(req: Request, res: Response) { if (results === undefined || results === null) { res.sendStatus(500); return false; - } else if (results.length as number === 0) { + } else if (results.length === 0) { res.sendStatus(404); return false; } else {