diff --git a/bin/commands/runs.js b/bin/commands/runs.js index df02954a..2f58a904 100644 --- a/bin/commands/runs.js +++ b/bin/commands/runs.js @@ -279,7 +279,11 @@ module.exports = function run(args, rawArgs) { let test_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.fileName)); let npm_zip_size = utils.fetchZipSize(path.join(process.cwd(), config.packageFileName)); - let node_modules_size = await utils.fetchFolderSize(path.join(process.cwd(), "node_modules")); + // Perf: node_modules size is instrumentation-only, so don't block the upload on + // walking the tree — start the walk here and await it only after the upload has + // completed (in the common case it resolves while the upload is in flight, adding + // zero wall-clock; fetchFolderSize never rejects, so the floating promise is safe). + let nodeModulesSizePromise = utils.fetchFolderSize(path.join(process.cwd(), "node_modules")); if (Constants.turboScaleObj.enabled) { // Note: Calculating md5 here for turboscale force-upload so that we don't need to re-calculate at hub @@ -306,6 +310,9 @@ module.exports = function run(args, rawArgs) { markBlockEnd('zip.zipUpload'); markBlockEnd('zip'); + // Walk was started before the upload; usually already resolved by now. + let node_modules_size = await nodeModulesSizePromise; + if (process.env.BROWSERSTACK_TEST_ACCESSIBILITY === 'true') { supportFileCleanup(); } diff --git a/bin/helpers/archiver.js b/bin/helpers/archiver.js index fb3b3155..a30c072e 100644 --- a/bin/helpers/archiver.js +++ b/bin/helpers/archiver.js @@ -58,7 +58,12 @@ const archiveSpecs = (runSettings, filePath, excludeFiles, md5data) => { let ignoreFiles = utils.getFilesToIgnore(runSettings, excludeFiles); logger.debug(`Patterns ignored during zip ${ignoreFiles}`); - archive.glob(`**/*.+(${Constants.allowedFileTypes.join("|")})`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles, dot:true }); + // Perf: `ignore` filters entries only AFTER the walk visits them, so the + // globber still descends into node_modules/.git/etc. `skip` prunes those directories + // from the traversal entirely — on large monorepos this cuts zip creation from + // minutes to seconds without changing the archive contents. + let skipDirectories = utils.getDirectorySkipPatterns(ignoreFiles); + archive.glob(`**/*.+(${Constants.allowedFileTypes.join("|")})`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles, skip: skipDirectories, dot:true }); let packageJSON = {}; diff --git a/bin/helpers/checkUploaded.js b/bin/helpers/checkUploaded.js index dfc29241..9578d127 100644 --- a/bin/helpers/checkUploaded.js +++ b/bin/helpers/checkUploaded.js @@ -25,6 +25,9 @@ const checkSpecsMd5 = (runSettings, args, instrumentBlocks) => { let options = { cwd: cypressFolderPath, ignore: ignoreFiles, + // Perf: prune ignored directories from the md5 walk instead of + // filtering entries after descending into them (see utils.getDirectorySkipPatterns). + skip: utils.getDirectorySkipPatterns(ignoreFiles), pattern: `**/*.+(${Constants.allowedFileTypes.join("|")})` }; hashHelper.hashWrapper(options, instrumentBlocks).then(function (data) { diff --git a/bin/helpers/utils.js b/bin/helpers/utils.js index 3ec867b9..2d5fe8b0 100644 --- a/bin/helpers/utils.js +++ b/bin/helpers/utils.js @@ -1103,6 +1103,20 @@ exports.getFilesToIgnore = (runSettings, excludeFiles, logging = true) => { return ignoreFiles; } +// Perf: derive directory-pruning patterns from the ignore list. +// readdir-glob (used both by archiver.glob for tests.zip and by hashUtil for the +// spec md5 check) applies `ignore` per-entry AFTER walking, so it still descends +// into node_modules/.git/dist etc. On large monorepos that walk alone takes +// minutes ("Creating tests.zip" stalls). Its `skip` option instead prevents +// DESCENDING into matching directories. Any ignore pattern ending in '/**' is +// safe to reuse as a skip: if a directory matches '/**' then every descendant +// also matches it, so pruning cannot change which files end up included. +exports.getDirectorySkipPatterns = (ignoreFiles) => { + return (ignoreFiles || []).filter((pattern) => + typeof pattern === 'string' && pattern.endsWith('/**') + ); +} + exports.getNumberOfSpecFiles = (bsConfig, args, cypressConfig, turboScaleSession=false) => { let defaultSpecFolder let testFolderPath @@ -1722,13 +1736,20 @@ exports.fetchZipSize = (fileName) => { } } -const getDirectorySize = async function(dir) { +const getDirectorySize = async function(dir, deadline) { try{ + // Perf: this telemetry-only walk recursively stats every file (it is + // pointed at node_modules in runs.js). On large monorepos it blocked the pipeline + // between archiving and uploading for tens of seconds. Stop descending once the + // deadline passes — folder size is best-effort telemetry, never worth stalling a run. + if (deadline && Date.now() > deadline) { + return 0; + } const subdirs = (await readdir(dir)); const files = await Promise.all(subdirs.map(async (subdir) => { const res = path.resolve(dir, subdir); const s = (await stat(res)); - return s.isDirectory() ? getDirectorySize(res) : (s.size); + return s.isDirectory() ? getDirectorySize(res, deadline) : (s.size); })); return files.reduce((a, f) => a+f, 0); }catch(e){ @@ -1738,10 +1759,15 @@ const getDirectorySize = async function(dir) { } }; -exports.fetchFolderSize = async (dir) => { +exports.fetchFolderSize = async (dir, timeoutMs = 5000) => { try { if(fs.existsSync(dir)){ - return (await getDirectorySize(dir) / 1024 / 1024); + const deadline = Date.now() + timeoutMs; + const size = (await getDirectorySize(dir, deadline) / 1024 / 1024); + if (Date.now() > deadline) { + logger.debug(`Folder size calculation for ${dir} exceeded ${timeoutMs}ms; reporting partial size.`); + } + return size; } return 0; } catch (error) { diff --git a/test/unit/bin/helpers/utils.js b/test/unit/bin/helpers/utils.js index 1d24ced6..9360fb90 100644 --- a/test/unit/bin/helpers/utils.js +++ b/test/unit/bin/helpers/utils.js @@ -770,6 +770,34 @@ describe('utils', () => { }); }); + // Perf: directory-shaped ignore patterns are reused as readdir-glob `skip` + // patterns so the zip/md5 walks prune excluded trees instead of descending into them. + describe('getDirectorySkipPatterns', () => { + it('keeps only patterns ending in /** (safe to prune)', () => { + const ignore = [ + '**/node_modules/**', + 'node_modules/**', + 'dist/**', + 'apps/admin-portal/**', + 'package.json', // file pattern — must NOT be used for pruning + '.env', + '**/README.md', + ]; + chai.expect(utils.getDirectorySkipPatterns(ignore)).to.be.eql([ + '**/node_modules/**', + 'node_modules/**', + 'dist/**', + 'apps/admin-portal/**', + ]); + }); + + it('handles empty/undefined input and non-string entries', () => { + chai.expect(utils.getDirectorySkipPatterns(undefined)).to.be.eql([]); + chai.expect(utils.getDirectorySkipPatterns([])).to.be.eql([]); + chai.expect(utils.getDirectorySkipPatterns([null, 42, 'x/**'])).to.be.eql(['x/**']); + }); + }); + describe('setTestEnvs', () => { it('set env only from args', () => { let argsEnv = 'env3=value3, env4=value4';