perf(SDK-6463): prune excluded dirs from zip/md5 walks — minutes-long 'Creating tests.zip' stall on monorepos

bin/helpers/archiver.js

@@ -58,7 +58,12 @@ const archiveSpecs = (runSettings, filePath, excludeFiles, md5data) => {
 
     let ignoreFiles = utils.getFilesToIgnore(runSettings, excludeFiles);
     logger.debug(`Patterns ignored during zip ${ignoreFiles}`);
-    archive.glob(`**/*.+(${Constants.allowedFileTypes.join("|")})`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles, dot:true });
+    // Perf: `ignore` filters entries only AFTER the walk visits them, so the
+    // globber still descends into node_modules/.git/etc. `skip` prunes those directories
+    // from the traversal entirely — on large monorepos this cuts zip creation from
+    // minutes to seconds without changing the archive contents.
+    let skipDirectories = utils.getDirectorySkipPatterns(ignoreFiles);
+    archive.glob(`**/*.+(${Constants.allowedFileTypes.join("|")})`, { cwd: cypressFolderPath, matchBase: true, ignore: ignoreFiles, skip: skipDirectories, dot:true });
 
     let packageJSON = {};
 

bin/helpers/checkUploaded.js

@@ -25,6 +25,9 @@ const checkSpecsMd5 = (runSettings, args, instrumentBlocks) => {
     let options = {
       cwd: cypressFolderPath,
       ignore: ignoreFiles,
+      // Perf: prune ignored directories from the md5 walk instead of
+      // filtering entries after descending into them (see utils.getDirectorySkipPatterns).
+      skip: utils.getDirectorySkipPatterns(ignoreFiles),
       pattern: `**/*.+(${Constants.allowedFileTypes.join("|")})`
     };
     hashHelper.hashWrapper(options, instrumentBlocks).then(function (data) {

bin/helpers/utils.js

@@ -1103,6 +1103,20 @@ exports.getFilesToIgnore = (runSettings, excludeFiles, logging = true) => {
   return ignoreFiles;
 }
 
+// Perf: derive directory-pruning patterns from the ignore list.
+// readdir-glob (used both by archiver.glob for tests.zip and by hashUtil for the
+// spec md5 check) applies `ignore` per-entry AFTER walking, so it still descends
+// into node_modules/.git/dist etc. On large monorepos that walk alone takes
+// minutes ("Creating tests.zip" stalls). Its `skip` option instead prevents
+// DESCENDING into matching directories. Any ignore pattern ending in '/**' is
+// safe to reuse as a skip: if a directory matches '<x>/**' then every descendant
+// also matches it, so pruning cannot change which files end up included.
+exports.getDirectorySkipPatterns = (ignoreFiles) => {
+  return (ignoreFiles || []).filter((pattern) =>
+    typeof pattern === 'string' && pattern.endsWith('/**')
+  );
+}
+
 exports.getNumberOfSpecFiles = (bsConfig, args, cypressConfig, turboScaleSession=false) => {
   let defaultSpecFolder
   let testFolderPath
@@ -1722,13 +1736,20 @@ exports.fetchZipSize = (fileName) => {
   }
 }
 
-const getDirectorySize = async function(dir) {
+const getDirectorySize = async function(dir, deadline) {
   try{
+    // Perf: this telemetry-only walk recursively stats every file (it is
+    // pointed at node_modules in runs.js). On large monorepos it blocked the pipeline
+    // between archiving and uploading for tens of seconds. Stop descending once the
+    // deadline passes — folder size is best-effort telemetry, never worth stalling a run.
+    if (deadline && Date.now() > deadline) {
+      return 0;
+    }
     const subdirs = (await readdir(dir));
     const files = await Promise.all(subdirs.map(async (subdir) => {
       const res = path.resolve(dir, subdir);
       const s = (await stat(res));
-      return s.isDirectory() ? getDirectorySize(res) : (s.size);
+      return s.isDirectory() ? getDirectorySize(res, deadline) : (s.size);
     }));
     return files.reduce((a, f) => a+f, 0);
   }catch(e){
@@ -1738,10 +1759,15 @@ const getDirectorySize = async function(dir) {
   }
 };
 
-exports.fetchFolderSize = async (dir) => {
+exports.fetchFolderSize = async (dir, timeoutMs = 5000) => {
   try {
     if(fs.existsSync(dir)){
-      return (await getDirectorySize(dir) / 1024 / 1024);
+      const deadline = Date.now() + timeoutMs;
+      const size = (await getDirectorySize(dir, deadline) / 1024 / 1024);
+      if (Date.now() > deadline) {
+        logger.debug(`Folder size calculation for ${dir} exceeded ${timeoutMs}ms; reporting partial size.`);
+      }
+      return size;
     }
     return 0;
   } catch (error) {

test/unit/bin/helpers/utils.js

@@ -770,6 +770,34 @@ describe('utils', () => {
     });
   });
 
+  // Perf: directory-shaped ignore patterns are reused as readdir-glob `skip`
+  // patterns so the zip/md5 walks prune excluded trees instead of descending into them.
+  describe('getDirectorySkipPatterns', () => {
+    it('keeps only patterns ending in /** (safe to prune)', () => {
+      const ignore = [
+        '**/node_modules/**',
+        'node_modules/**',
+        'dist/**',
+        'apps/admin-portal/**',
+        'package.json',        // file pattern — must NOT be used for pruning
+        '.env',
+        '**/README.md',
+      ];
+      chai.expect(utils.getDirectorySkipPatterns(ignore)).to.be.eql([
+        '**/node_modules/**',
+        'node_modules/**',
+        'dist/**',
+        'apps/admin-portal/**',
+      ]);
+    });
+
+    it('handles empty/undefined input and non-string entries', () => {
+      chai.expect(utils.getDirectorySkipPatterns(undefined)).to.be.eql([]);
+      chai.expect(utils.getDirectorySkipPatterns([])).to.be.eql([]);
+      chai.expect(utils.getDirectorySkipPatterns([null, 42, 'x/**'])).to.be.eql(['x/**']);
+    });
+  });
+
   describe('setTestEnvs', () => {
     it('set env only from args', () => {
       let argsEnv = 'env3=value3, env4=value4';