title: NodeJS: scanning a directory tree is slow as hell
body: <p>I have a NW.js app that simply (recursively) scans a directory tree and get the stats for each file/directory. It also performs a MD5 for files.</p>
  
  <p>I have 29k files, 850 folders, all for 120GB data.</p>
  
  <p>After almost 7 minutes, my code only scanned 4080 files over the 29k files.</p>
  
  <p>How is it possible that it is so slow?? Is there something I can do to improve performance? Otherwise, Node would be useless to me...</p>
  
  <p>What is surprising, is that it took "only" 7 seconds to scan 1k files. Why is it 60 times longer to scan only 4 times as much files?</p>
  
  <p>When I check the processes, I can see that Node moves a lot in RAM usage: from 20MB to 400MB (it fluctuates both ways). But the CPU usage is stuck at 1%.</p>
  
  <p>It is weird, because I don't think I am allocating so much RAM. Actually, I don't allocate anything! Please see my code below.</p>
  
  <pre><code>if (process.argv.length &lt; 3)
      process.exit();
  
  
  var fs = require('fs');
  var md5 = require('md5');
  var md5File = require('md5-file');
  
  var iTotal = 0;
  var iNbFiles = 0;
  var iNbFolders = 0;
  
  var iBegin = Date.now();
  
  var App =
  {
      scan: function(path)
      {
          var items = fs.readdirSync(path);
          var i, item, stats, fullPath, isFolder, fileMD5;
          var len = items.length;
          var md5Hash = md5(path);
  
          for (i = 0; i &lt; len; i++)
          {
              item = items[i];
              fullPath = path + '/' + item;
              stats = fs.statSync(fullPath);
              if (stats.isSymbolicLink())
                  continue;
  
              isFolder = stats.isDirectory();
              if (!isFolder)
              {
                  fileMD5 = md5File(fullPath);
                  iNbFiles++;
              }
              else
              {
                  fileMD5 = null;
                  iNbFolders++;
              }
  
              iTotal++;
              process.send({_type: 'item', name: item, path: path, path_md5: md5Hash, full_path: fullPath, file_md5: fileMD5, stats: stats, is_folder: isFolder});
              if (isFolder)
                  App.scan(path + '/' + item);
          }
  
          process.send({_type: 'temp', total: iTotal, files: iNbFiles, folders: iNbFolders, elapsed: (Date.now() - iBegin)});
      }
  };
  
  App.scan(process.argv[2]);
  
  // Send the final and definitive value of "total"
  process.send({_type: 'total', total: iTotal, files: iNbFiles, folders: iNbFolders});
  
  process.exit();
  </code></pre>