How to use ContinuationToken with recursive folder iterator

If you’re trying to recursively iterate on a folder and want to use continuation tokens (as is probably required for large folders), you’ll need a data structure that can store multiple sets of continuation tokens. Both for files and folders, but also for each folder in the current hierarchy.

The simplest data structure would be an array of objects.

Here is a solution that gives you the template for creating a function that can recursively process files and store continuation tokens so it can resume if it times out.

1. Simply modify MAX_RUNNING_TIME_MS to your desired value (now it’s set to 1 minute).

   You don’t want to set it more than ~4.9 minutes as the script could timeout before then and not store its current state.

2. Update the processFile method to do whatever you want on files.
3. Finally, call processRootFolder() and pass it a Folder. It’ll be smart enough to know how to resume processing the folder.

Sure there is room for improvement (e.g. it simply checks the folder name to see if it’s a resume vs. a restart) but this will most likely be sufficient for 95% of people that need to iterate recursively on a folder with continuation tokens.

function processRootFolder(rootFolder) {
  var MAX_RUNNING_TIME_MS = 1 * 60 * 1000;
  var RECURSIVE_ITERATOR_KEY = "RECURSIVE_ITERATOR_KEY";

  var startTime = (new Date()).getTime();

  // [{folderName: String, fileIteratorContinuationToken: String?, folderIteratorContinuationToken: String}]
  var recursiveIterator = JSON.parse(PropertiesService.getDocumentProperties().getProperty(RECURSIVE_ITERATOR_KEY));
  if (recursiveIterator !== null) {
    // verify that it's actually for the same folder
    if (rootFolder.getName() !== recursiveIterator[0].folderName) {
      console.warn("Looks like this is a new folder. Clearing out the old iterator.");
      recursiveIterator = null;
    } else {
      console.info("Resuming session.");
    }
  }
  if (recursiveIterator === null) {
    console.info("Starting new session.");
    recursiveIterator = [];
    recursiveIterator.push(makeIterationFromFolder(rootFolder));
  }

  while (recursiveIterator.length > 0) {
    recursiveIterator = nextIteration(recursiveIterator, startTime);

    var currTime = (new Date()).getTime();
    var elapsedTimeInMS = currTime - startTime;
    var timeLimitExceeded = elapsedTimeInMS >= MAX_RUNNING_TIME_MS;
    if (timeLimitExceeded) {
      PropertiesService.getDocumentProperties().setProperty(RECURSIVE_ITERATOR_KEY, JSON.stringify(recursiveIterator));
      console.info("Stopping loop after '%d' milliseconds. Please continue running.", elapsedTimeInMS);
      return;
    }
  }

  console.info("Done running");
  PropertiesService.getDocumentProperties().deleteProperty(RECURSIVE_ITERATOR_KEY);
}

// process the next file or folder
function nextIteration(recursiveIterator) {
  var currentIteration = recursiveIterator[recursiveIterator.length-1];
  if (currentIteration.fileIteratorContinuationToken !== null) {
    var fileIterator = DriveApp.continueFileIterator(currentIteration.fileIteratorContinuationToken);
    if (fileIterator.hasNext()) {
      // process the next file
      var path = recursiveIterator.map(function(iteration) { return iteration.folderName; }).join("https://stackoverflow.com/");
      processFile(fileIterator.next(), path);
      currentIteration.fileIteratorContinuationToken = fileIterator.getContinuationToken();
      recursiveIterator[recursiveIterator.length-1] = currentIteration;
      return recursiveIterator;
    } else {
      // done processing files
      currentIteration.fileIteratorContinuationToken = null;
      recursiveIterator[recursiveIterator.length-1] = currentIteration;
      return recursiveIterator;
    }
  }

  if (currentIteration.folderIteratorContinuationToken !== null) {
    var folderIterator = DriveApp.continueFolderIterator(currentIteration.folderIteratorContinuationToken);
    if (folderIterator.hasNext()) {
      // process the next folder
      var folder = folderIterator.next();
      recursiveIterator[recursiveIterator.length-1].folderIteratorContinuationToken = folderIterator.getContinuationToken();
      recursiveIterator.push(makeIterationFromFolder(folder));
      return recursiveIterator;
    } else {
      // done processing subfolders
      recursiveIterator.pop();
      return recursiveIterator;
    }
  }

  throw "should never get here";
}

function makeIterationFromFolder(folder) {
  return {
    folderName: folder.getName(), 
    fileIteratorContinuationToken: folder.getFiles().getContinuationToken(),
    folderIteratorContinuationToken: folder.getFolders().getContinuationToken()
  };
}

function processFile(file, path) {
  console.log(path + "https://stackoverflow.com/" + file.getName());
}