From 51b46f11c731ca96193ca5777324d86a6e6819da Mon Sep 17 00:00:00 2001 From: Tarek Rached Date: Mon, 26 Oct 2020 16:05:13 -0700 Subject: [PATCH 1/4] search folders in batches to avoid 413 entity too large from drive api --- server/search.js | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/server/search.js b/server/search.js index 73b4fa4e..53dc9fc4 100644 --- a/server/search.js +++ b/server/search.js @@ -6,22 +6,34 @@ const list = require('./list') const log = require('./logger') const driveId = process.env.DRIVE_ID +const MAX_FOLDERS_TO_SEARCH = 100 // got 413 entity too large at 129, this gives us some headroom exports.run = async (query, driveType = 'team') => { const authClient = await getAuth() - let folderIds + let allFolderIds const drive = google.drive({version: 'v3', auth: authClient}) if (driveType === 'folder') { - folderIds = await getAllFolders({drive}) + allFolderIds = await getAllFolders({drive}) } + log.debug(`searching ${allFolderIds.length} folders in chunks of ${MAX_FOLDERS_TO_SEARCH}`) - const files = await fullSearch({drive, query, folderIds, driveType}) - .catch((err) => { - log.error(`Error when searching for ${query}, ${err}`) - throw err - }) + const files = [] + + while (allFolderIds.length > 0) { + const folderIds = allFolderIds.splice(0, MAX_FOLDERS_TO_SEARCH) + + const theseFiles = await fullSearch({drive, query, folderIds, driveType}) + .catch((err) => { + log.error(`Error when searching for ${query}, ${err}`) + throw err + }) + + files.push(...theseFiles) + } + + log.debug(`got ${files.length} results`) const fileMetas = files .map((file) => { return list.getMeta(file.id) || {} }) From b8f4d76ac7b082f762c85cfd7998e98d59f38b7b Mon Sep 17 00:00:00 2001 From: Tarek Rached Date: Mon, 26 Oct 2020 16:27:05 -0700 Subject: [PATCH 2/4] search batches of folders concurrently --- server/search.js | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/server/search.js b/server/search.js index 53dc9fc4..d487c879 100644 --- a/server/search.js +++ b/server/search.js @@ -19,27 +19,28 @@ exports.run = async (query, driveType = 'team') => { } log.debug(`searching ${allFolderIds.length} folders in chunks of ${MAX_FOLDERS_TO_SEARCH}`) - const files = [] + const folderIdBatches = [] while (allFolderIds.length > 0) { - const folderIds = allFolderIds.splice(0, MAX_FOLDERS_TO_SEARCH) - - const theseFiles = await fullSearch({drive, query, folderIds, driveType}) - .catch((err) => { - log.error(`Error when searching for ${query}, ${err}`) - throw err - }) - - files.push(...theseFiles) + folderIdBatches.push(allFolderIds.splice(0, MAX_FOLDERS_TO_SEARCH)) } - log.debug(`got ${files.length} results`) - - const fileMetas = files - .map((file) => { return list.getMeta(file.id) || {} }) - .filter(({path, tags}) => (path || '').split('/')[1] !== 'trash' && !(tags || []).includes('hidden')) - - return fileMetas + try { + const files = (await Promise.all( + folderIdBatches.map((folderIds) => + fullSearch({drive, query, folderIds, driveType}) + ) + )).flat() + + const fileMetas = files + .map((file) => { return list.getMeta(file.id) || {} }) + .filter(({path, tags}) => (path || '').split('/')[1] !== 'trash' && !(tags || []).includes('hidden')) + + return fileMetas + } catch (err) { + log.error(`Error when searching for ${query}, ${err}`) + throw err + } } async function fullSearch({drive, query, folderIds, results = [], nextPageToken: pageToken, driveType}) { From a5b0ac30f496f215580971ad4969bee3bac62038 Mon Sep 17 00:00:00 2001 From: Tarek Rached Date: Mon, 26 Oct 2020 17:18:56 -0700 Subject: [PATCH 3/4] handle case of no folder ids found --- server/search.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/server/search.js b/server/search.js index d487c879..a1c3e119 100644 --- a/server/search.js +++ b/server/search.js @@ -10,19 +10,22 @@ const MAX_FOLDERS_TO_SEARCH = 100 // got 413 entity too large at 129, this gives exports.run = async (query, driveType = 'team') => { const authClient = await getAuth() - let allFolderIds - const drive = google.drive({version: 'v3', auth: authClient}) + let folderIdBatches = [] + if (driveType === 'folder') { - allFolderIds = await getAllFolders({drive}) - } - log.debug(`searching ${allFolderIds.length} folders in chunks of ${MAX_FOLDERS_TO_SEARCH}`) + let allFolderIds = await getAllFolders({drive}) + while (allFolderIds.length > 0) { + folderIdBatches.push(allFolderIds.splice(0, MAX_FOLDERS_TO_SEARCH)) + } + + log.debug(`searching ${allFolderIds.length} folders in chunks of ${MAX_FOLDERS_TO_SEARCH}`) - const folderIdBatches = [] + } - while (allFolderIds.length > 0) { - folderIdBatches.push(allFolderIds.splice(0, MAX_FOLDERS_TO_SEARCH)) + if (folderIdBatches.length === 0) { + folderIdBatches.push([]) } try { From 3ff854a94d5014b11305b64740836d71cc3525ad Mon Sep 17 00:00:00 2001 From: Tarek Rached Date: Tue, 27 Oct 2020 10:13:56 -0700 Subject: [PATCH 4/4] replace Array.flat() with .reduce() bc node 10 doesn't have .flat() --- server/search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/search.js b/server/search.js index a1c3e119..25b17138 100644 --- a/server/search.js +++ b/server/search.js @@ -33,7 +33,7 @@ exports.run = async (query, driveType = 'team') => { folderIdBatches.map((folderIds) => fullSearch({drive, query, folderIds, driveType}) ) - )).flat() + )).reduce((files, fileBatch) => files.concat(fileBatch), []) const fileMetas = files .map((file) => { return list.getMeta(file.id) || {} })