text:
//-*- Mode: C++ -*-
// $Id$

#ifndef ALIHLTCTPDATA_H
#define ALIHLTCTPDATA_H
/

#include "TNamed.h"
#include "TClonesArray.h"
#include "TArrayL64.h"
#include "AliHLTLogging.h"
#include "AliHLTDataTypes.h"
#include "AliHLTReadoutList.h"
#include <string>

/**
 * @class AliHLTCTPData
 * This is a container for the CTP trigger classes, the mapping to the bit
 * field, and counters.
 *
 * The class is initialized from CTP_TRIGGER_CLASSES part of the ECS parameters.
 * and stores internally a list of trigger classes holding the information on bit
 * position, class name and involved detectors. The general format og the parameter
 * is as follows:
 * <pre>
 * [bit position]:[Trigger class identifier string]:[detector-id-nr]-[detector-id-nr]-...,[bit position]:.....
 * </pre>
 *
 * The list of participating detectors is converted into an AliHLTReadoutList
 * object named after the trigger class name, and can be used as mask for the
 * readout list generated by a component.
 *
 * The object is also stored as part of the HLTGlobalTriggerDecision
 * @ingroup alihlt_trigger
 */
class AliHLTCTPData: public TNamed, public AliHLTLogging
{
 public:
  /// default constructor
  AliHLTCTPData();
  /// standard constructor including initialization from CTP_TRIGGER_CLASS
  AliHLTCTPData(const char* parameter);
  /// copy constructor
  AliHLTCTPData(const AliHLTCTPData&);
  ///assignment operator
  AliHLTCTPData& operator=(const AliHLTCTPData&);
  /// destructor
  virtual ~AliHLTCTPData();

  /// Add counters
  AliHLTCTPData& operator += (const AliHLTCTPData&);
  /// Add counters
  AliHLTCTPData operator + (const AliHLTCTPData&) const;

  /// Subtract counters
  AliHLTCTPData& operator -= (const AliHLTCTPData&);
  /// Subtract counters
  AliHLTCTPData operator - (const AliHLTCTPData&) const;

  /**
   * Init the class ids and mapping from the CTP_TRIGGER_CLASS parameter.
   * The general format of the parameter is as follows:
   */
  int InitCTPTriggerClasses(const char* ctpString);

  /// etract the active trigger mask from the trigger data
  static AliHLTTriggerMask_t ActiveTriggers(const AliHLTComponentTriggerData& trigData);

  /**
   * Evaluate an expression of trigger class ids with respect to the trigger mask.
   */
  bool EvaluateCTPTriggerClass(const char* expression, const AliHLTComponentTriggerData& trigData) const;

  /**
   * Evaluate an expression of trigger class ids with respect to the trigger mask.
   */
  bool EvaluateCTPTriggerClass(const char* expression, AliHLTTriggerMask_t triggerMask) const;

  /**
   * Evaluate an expression of trigger class ids with respect to the current trigger mask.
   */
  bool EvaluateCTPTriggerClass(const char* expression) const {
    return EvaluateCTPTriggerClass(expression, fTriggers);
  }

  /**
   * Reset all counters
   */
  void ResetCounters();

  /**
   * Get index of a trigger class in the tigger pattern
   */
  int Index(const char* name) const;

  /**
   * Check state of a trigger class.
   * If the class name is not part of the current trigger setup (i.e. ECS parameter
   * does not contain a trigger definition for this class name) the function
   * returns -1
   * @return -1 class name not initialized,
   *          0 trigger not active
   *          1 trigger active
   */
  int CheckTrigger(const char* name) const;

  /** glob match name in trigger mask **/
  int MatchTriggerGlob(const char* glob) const;

  /** regex match in trigger mask **/
  int MatchTriggerRE(const char* re) const;

  /**
   * Increment counter for CTP trigger classes
   * @param classIds  comma separated list of class ids
   */
  void Increment(const char* classIds);

  /**
   * Increment counter for CTP trigger classes
   * @param triggerPattern  corresponds to the 100bit trigger mask in the CDH
   */
  void Increment(AliHLTTriggerMask_t triggerPattern);

  /**
   * Increment counter for a CTP trigger class
   * @param classIdx  index of the class in the 100bit trigger mask
   */
  void Increment(int classIdx);

  /**
   * Increment counters according to the trigger data struct.
   * First extract trigger pattern from the CDH and then
   * increment from the trigger pattern.
   */
  int Increment(AliHLTComponentTriggerData& trigData);

  /**
   * Return a readout list for the active trigger classes.
   * The list is an 'OR' of the active trugger classes.
   */
  AliHLTReadoutList ReadoutList(const AliHLTComponentTriggerData& trigData) const;

  /**
   * Return a readout list for the active trigger classes.
   * The list is an 'OR' of the active trugger classes.
   */
  AliHLTReadoutList ReadoutList(AliHLTTriggerMask_t triggerMask) const;

  /**
   * Return a readout list for the active trigger classes.
   * The list is an 'OR' of the active trugger classes.
   */
  AliHLTReadoutList ReadoutList() const {
    return ReadoutList(fTriggers);
  }

  /**
   * Inherited from TObject, this prints the contents of the trigger decision.
   */
  virtual void Print(Option_t* option = "") const;

  /**
   * this is for compatibility with ESD, fills 2 words with 50 bit trigger masks
  */
  void      GetTriggerMaskAll(ULong64_t& low,ULong64_t& high) const;

  /**
   * fills a string with the names of all active trigger classes
   * returns number of classes
   */
  int GetFiredTriggerClasses(std::string& string) const;

  AliHLTTriggerMask_t   Mask() const { return fMask; }
  AliHLTTriggerMask_t   Triggers() const { return fTriggers; }
  void                  SetTriggers(AliHLTTriggerMask_t triggers) { fTriggers=triggers; }
  void                  SetTriggers(AliHLTComponentTriggerData trigData) {SetTriggers(ActiveTriggers(trigData));}
  const TArrayL64&      Counters() const { return fCounters; }
  AliHLTUInt64_t        Counter(int index) const;
  AliHLTUInt64_t        Counter(const char* classId) const;
  const char*           Name(int index) const;

 protected:
 private:
  /// Helper
  std::string TriggerMaskToString(AliHLTTriggerMask_t mask) const;

  /**
   * Add counters.
   * Base methods for operators.
   * @param src    instance to add
   * @param factor +1/-1 for addition/subtraction
   * @skipped      target to get the numner of not matching class names
   */
  int Add(const AliHLTCTPData& src, int factor, int &skipped);

  /**
   * Read the map of trigger class indexes
   * The map is used in the generation of the TFormula from the trigger
   * expression in order to handle similar class names correctly.
   * For names like 'base', 'baseA', 'baseB' the class with the longer name
   * has to be replaced first.
   */
  int ReadMap(vector<unsigned> &map) const;

  int ReadMap() {return ReadMap(fMap);}

  /** simple char-by-char glob string matching
   * e.g. ***FOO matches BARFOO **/
  Bool_t Globncmp(const char* triggerName, const char* glob, int triggerNameSize, int globSize ) const;

  AliHLTTriggerMask_t fMask;      /// mask of initialized trigger classes
  AliHLTTriggerMask_t fTriggers;  /// current trigger
  TClonesArray        fClassIds;  /// array of trigger class ids
  TArrayL64           fCounters;  /// trigger class counters
  vector<unsigned>    fMap;     //! index map for trigger expression evaluation

  ClassDef(AliHLTCTPData, 3)
};

#endif


text:
"""Wrapper for running the test under heapchecker and analyzing the output."""

import datetime
import logging
import os
import re

import common
import path_utils
import suppressions

class HeapcheckWrapper(object):
  TMP_FILE = 'heapcheck.log'
  SANITY_TEST_SUPPRESSION = "Heapcheck sanity test"
  LEAK_REPORT_RE = re.compile(
     'Leak of ([0-9]*) bytes in ([0-9]*) objects allocated from:')
  STACK_LINE_RE = re.compile('\s*@\s*(?:0x)?[0-9a-fA-F]+\s*([^\n]*)')
  BORING_CALLERS = common.BoringCallers(mangled=False, use_re_wildcards=True)

  def __init__(self, supp_files):
    self._mode = 'strict'
    self._timeout = 1200
    self._nocleanup_on_exit = False
    self._suppressions = []
    for fname in supp_files:
      self._suppressions.extend(suppressions.ReadSuppressionsFromFile(fname))
    if os.path.exists(self.TMP_FILE):
      os.remove(self.TMP_FILE)

  def PutEnvAndLog(self, env_name, env_value):
    """Sets the env var |env_name| to |env_value| and writes to logging.info.
    """
    os.putenv(env_name, env_value)
    logging.info('export %s=%s', env_name, env_value)

  def Execute(self):
    """Executes the app to be tested."""
    logging.info('starting execution...')
    proc = ['sh', path_utils.ScriptDir() + '/heapcheck_std.sh']
    proc += self._args
    self.PutEnvAndLog('G_SLICE', 'always-malloc')
    self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
    self.PutEnvAndLog('NSS_DISABLE_UNLOAD', '1')
    self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
    self.PutEnvAndLog('HEAPCHECK', self._mode)
    self.PutEnvAndLog('HEAP_CHECK_MAX_LEAKS', '-1')
    self.PutEnvAndLog('KEEP_SHADOW_STACKS', '1')
    self.PutEnvAndLog('PPROF_PATH',
        path_utils.ScriptDir() +
        '/../../third_party/tcmalloc/chromium/src/pprof')
    self.PutEnvAndLog('LD_LIBRARY_PATH',
                      '/usr/lib/debug/:/usr/lib32/debug/')

    common.RunSubprocess(proc, self._timeout)

    # Always return true, even if running the subprocess failed. We depend on
    # Analyze to determine if the run was valid.
    return True

  def Analyze(self, log_lines, check_sanity=False):
    """Analyzes the app's output and applies suppressions to the reports.

    Analyze() searches the logs for leak reports and tries to apply
    suppressions to them. Unsuppressed reports and other log messages are
    dumped as is.

    If |check_sanity| is True, the list of suppressed reports is searched for a
    report starting with SANITY_TEST_SUPPRESSION. If there isn't one, Analyze
    returns 2 regardless of the unsuppressed reports.

    Args:
      log_lines:      An iterator over the app's log lines.
      check_sanity:   A flag that determines whether we should check the tool's
                      sanity.
    Returns:
      2, if the sanity check fails,
      1, if unsuppressed reports remain in the output and the sanity check
      passes,
      0, if all the errors are suppressed and the sanity check passes.
    """
    return_code = 0
    # leak signature: [number of bytes, number of objects]
    cur_leak_signature = None
    cur_stack = []
    cur_report = []
    reported_hashes = {}
    # Statistics grouped by suppression description:
    # [hit count, bytes, objects].
    used_suppressions = {}
    for line in log_lines:
      line = line.rstrip()  # remove the trailing \n
      match = self.STACK_LINE_RE.match(line)
      if match:
        cur_stack.append(match.groups()[0])
        cur_report.append(line)
        continue
      else:
        if cur_stack:
          # Try to find the suppression that applies to the current leak stack.
          description = ''
          for supp in self._suppressions:
            if supp.Match(cur_stack):
              cur_stack = []
              description = supp.description
              break
          if cur_stack:
            if not cur_leak_signature:
              print 'Missing leak signature for the following stack: '
              for frame in cur_stack:
                print '   ' + frame
              print 'Aborting...'
              return 3

            # Drop boring callers from the stack to get less redundant info
            # and fewer unique reports.
            found_boring = False
            for i in range(1, len(cur_stack)):
              for j in self.BORING_CALLERS:
                if re.match(j, cur_stack[i]):
                  cur_stack = cur_stack[:i]
                  cur_report = cur_report[:i]
                  found_boring = True
                  break
              if found_boring:
                break

            error_hash = hash("".join(cur_stack)) & 0xffffffffffffffff
            if error_hash not in reported_hashes:
              reported_hashes[error_hash] = 1
              # Print the report and set the return code to 1.
              print ('Leak of %d bytes in %d objects allocated from:'
                     % tuple(cur_leak_signature))
              print '\n'.join(cur_report)
              return_code = 1
              # Generate the suppression iff the stack contains more than one
              # frame (otherwise it's likely to be broken)
              if len(cur_stack) > 1 or found_boring:
                print '\nSuppression (error hash=#%016X#):\n{'  % (error_hash)
                print '   <insert_a_suppression_name_here>'
                print '   Heapcheck:Leak'
                for frame in cur_stack:
                  print '   fun:' + frame
                print '}\n\n'
              else:
                print ('This stack may be broken due to omitted frame pointers.'
                       ' It is not recommended to suppress it.\n')
          else:
            # Update the suppressions histogram.
            if description in used_suppressions:
              hits, bytes, objects = used_suppressions[description]
              hits += 1
              bytes += cur_leak_signature[0]
              objects += cur_leak_signature[1]
              used_suppressions[description] = [hits, bytes, objects]
            else:
              used_suppressions[description] = [1] + cur_leak_signature
        cur_stack = []
        cur_report = []
        cur_leak_signature = None
        match = self.LEAK_REPORT_RE.match(line)
        if match:
          cur_leak_signature = map(int, match.groups())
        else:
          print line
    # Print the list of suppressions used.
    is_sane = False
    if used_suppressions:
      print
      print '-----------------------------------------------------'
      print 'Suppressions used:'
      print '   count    bytes  objects name'
      histo = {}
      for description in used_suppressions:
        if description.startswith(HeapcheckWrapper.SANITY_TEST_SUPPRESSION):
          is_sane = True
        hits, bytes, objects = used_suppressions[description]
        line = '%8d %8d %8d %s' % (hits, bytes, objects, description)
        if hits in histo:
          histo[hits].append(line)
        else:
          histo[hits] = [line]
      keys = histo.keys()
      keys.sort()
      for count in keys:
        for line in histo[count]:
          print line
      print '-----------------------------------------------------'
    if check_sanity and not is_sane:
      logging.error("Sanity check failed")
      return 2
    else:
      return return_code

  def RunTestsAndAnalyze(self, check_sanity):
    self.Execute()
    log_file = file(self.TMP_FILE, 'r')
    ret = self.Analyze(log_file, check_sanity)
    log_file.close()
    return ret

  def Main(self, args, check_sanity=False):
    self._args = args
    start = datetime.datetime.now()
    retcode = -1
    retcode = self.RunTestsAndAnalyze(check_sanity)
    end = datetime.datetime.now()
    seconds = (end - start).seconds
    hours = seconds / 3600
    seconds %= 3600
    minutes = seconds / 60
    seconds %= 60
    logging.info('elapsed time: %02d:%02d:%02d', hours, minutes, seconds)
    logging.info('For more information on the Heapcheck bot see '
                 'http://dev.chromium.org/developers/how-tos/'
                 'using-the-heap-leak-checker')
    return retcode

def RunTool(args, supp_files, module):
  tool = HeapcheckWrapper(supp_files)
  MODULES_TO_SANITY_CHECK = ["base"]
  check_sanity = module in MODULES_TO_SANITY_CHECK
  return tool.Main(args[1:], check_sanity)

redpajama_set_name: RedPajamaGithub


text:
<?php

namespace Sonata\FormatterBundle\Extension;

class ControlFlowExtension extends BaseExtension
{
    /**
     * Returns an array of available filters
     *
     * @return array
     */
    public function getAllowedFilters()
    {
        return array(
            'escape'
        );
    }

    /**
     * Returns an array of available tags
     *
     * @return array
     */
    public function getAllowedTags()
    {
        return array(
            'if',
            'for'
        );
    }

    /**
     * Returns an array of available functions
     *
     * @return array
     */
    public function getAllowedFunctions()
    {
        return array();
    }

    /**
     * Returns the name of the extension.
     *
     * @return string The extension name
     */
    public function getName()
    {
        return 'sonata_formatter_extension_flow';
    }
}


text:
package org.sagebionetworks.bridge.udd.synapse;

import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;

import java.io.File;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import au.com.bytecode.opencsv.CSVReader;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.io.CharStreams;
import org.joda.time.LocalDate;
import org.mockito.ArgumentCaptor;
import org.sagebionetworks.client.exceptions.SynapseException;
import org.sagebionetworks.repo.model.file.BulkFileDownloadResponse;
import org.sagebionetworks.repo.model.file.FileDownloadSummary;
import org.testng.annotations.Test;

import org.sagebionetworks.bridge.file.InMemoryFileHelper;
import org.sagebionetworks.bridge.schema.UploadSchema;
import org.sagebionetworks.bridge.schema.UploadSchemaKey;
import org.sagebionetworks.bridge.udd.exceptions.AsyncTaskExecutionException;

@SuppressWarnings({ "rawtypes", "unchecked" })
public class SynapseDownloadFromTableTaskTest {
    private static final UploadSchemaKey TEST_SCHEMA_KEY = new UploadSchemaKey.Builder().withAppId("test-study")
            .withSchemaId("test-schema").withRevision(42).build();

    // The default test schema should include at least 2 file handle IDs and a mix of file handles and
    // non-file-handles.
    private static final UploadSchema DEFAULT_TEST_SCHEMA = new UploadSchema.Builder().withKey(TEST_SCHEMA_KEY)
            .addField("foo", "INT").addField("bar", "ATTACHMENT_BLOB").addField("baz", "ATTACHMENT_JSON_BLOB").build();

    private InMemoryFileHelper inMemoryFileHelper;
    private ArgumentCaptor<String> synapseQueryCaptor;
    private ArgumentCaptor<Set> synapseFileHandleIdSetCaptor;
    private SynapseDownloadFromTableTask task;
    private File tmpDir;

    @Test
    public void csvHasNoUserRows() throws Exception {
        // setup
        String csvContent = "\"recordId\",\"healthCode\",\"foo\",\"bar\",\"baz\"";
        setupTestWithArgs(DEFAULT_TEST_SCHEMA, csvContent, null, null);

        // execute and validate
        SynapseDownloadFromTableResult result = task.call();
        assertNull(result.getCsvFile());
        assertNull(result.getBulkDownloadFile());
        postValidation(result);
    }

    @Test
    public void csvHasNoHealthCode() throws Exception {
        // setup
        String csvContent = "\"recordId\",\"foo\",\"bar\",\"baz\"\n" +
                "\"record-1\",\"37\",\"file-handle-1\",\"file-handle-2\"";
        setupTestWithArgs(DEFAULT_TEST_SCHEMA, csvContent, null, null);

        // execute
        Exception thrownEx = null;
        try {
            task.call();
            fail("expected exception");
        } catch (IllegalStateException ex) {
            thrownEx = ex;
        }
        assertTrue(thrownEx.getMessage().contains("healthCodeColumnIndex"));

        // validate
        postValidation(null);
    }

    @Test
    public void schemaHasNoFileHandles() throws Exception {
        // setup
        UploadSchema schema = new UploadSchema.Builder().withKey(TEST_SCHEMA_KEY).addField("asdf", "INT").build();
        String csvContent = "\"recordId\",\"healthCode\",\"asdf\"\n" +
                "\"record-1\",\"test-health-code\",\"7\"";
        setupTestWithArgs(schema, csvContent, null, null);

        // execute and validate
        SynapseDownloadFromTableResult result = task.call();
        assertNull(result.getBulkDownloadFile());
        List<String[]> parsedCsv = parseCsv(result.getCsvFile());
        assertEquals(parsedCsv.size(), 2);

        // header
        assertEquals(parsedCsv.get(0).length, 3);
        assertEquals(parsedCsv.get(0)[0], "recordId");
        assertEquals(parsedCsv.get(0)[1], "healthCode");
        assertEquals(parsedCsv.get(0)[2], "asdf");

        // row 1
        assertEquals(parsedCsv.get(1).length, 3);
        assertEquals(parsedCsv.get(1)[0], "record-1");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(1)[1]));
        assertEquals(parsedCsv.get(1)[2], "7");

        postValidation(result);
    }

    @Test
    public void csvHasNoFileHandles() throws Exception {
        // setup
        String csvContent = "\"recordId\",\"healthCode\",\"foo\",\"bar\",\"baz\"\n" +
                "\"record-1\",\"test-health-code\",\"13\",,";
        setupTestWithArgs(DEFAULT_TEST_SCHEMA, csvContent, null, null);

        // execute and validate
        SynapseDownloadFromTableResult result = task.call();
        assertNull(result.getBulkDownloadFile());
        List<String[]> parsedCsv = parseCsv(result.getCsvFile());
        assertEquals(parsedCsv.size(), 2);

        // header
        assertEquals(parsedCsv.get(0).length, 5);
        assertEquals(parsedCsv.get(0)[0], "recordId");
        assertEquals(parsedCsv.get(0)[1], "healthCode");
        assertEquals(parsedCsv.get(0)[2], "foo");
        assertEquals(parsedCsv.get(0)[3], "bar");
        assertEquals(parsedCsv.get(0)[4], "baz");

        // row 1
        assertEquals(parsedCsv.get(1).length, 5);
        assertEquals(parsedCsv.get(1)[0], "record-1");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(1)[1]));
        assertEquals(parsedCsv.get(1)[2], "13");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(1)[3]));
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(1)[4]));

        postValidation(result);
    }

    @Test
    public void csvAndBulkDownload() throws Exception {
        // For full branch coverage, we need the following cases:
        // * row with no file handles
        // * row with 1 file handle
        // * row with 2 file handles
        // * file summary with null file handle id
        // * file summary with failure message
        // * file summary with unknown error
        // * missing file summary (unknown error)
        // * extraneous file summary

        // setup
        String csvContent = "\"recordId\",\"healthCode\",\"foo\",\"bar\",\"baz\"\n" +
                "\"record-1\",\"test-health-code\",\"4\",,\n" +
                "\"record-2\",\"test-health-code\",\"8\",\"file-handle-2a\",\n" +
                "\"record-3\",\"test-health-code\",\"15\",\"file-handle-3a\",\"file-handle-3b\"\n" +
                "\"record-4\",\"test-health-code\",\"16\",\"service-error-file-handle\",\"unknown-error-file-handle\"\n" +
                "\"record-5\",\"test-health-code\",\"23\",\"missing-file-handle\",";

        List<FileDownloadSummary> fileSummaryList = new ArrayList<>();
        {
            FileDownloadSummary fileSummary = new FileDownloadSummary();
            fileSummary.setFileHandleId("file-handle-2a");
            fileSummary.setZipEntryName("zip-entry-2a");
            fileSummaryList.add(fileSummary);
        }
        {
            FileDownloadSummary fileSummary = new FileDownloadSummary();
            fileSummary.setFileHandleId("file-handle-3a");
            fileSummary.setZipEntryName("zip-entry-3a");
            fileSummaryList.add(fileSummary);
        }
        {
            FileDownloadSummary fileSummary = new FileDownloadSummary();
            fileSummary.setFileHandleId("file-handle-3b");
            fileSummary.setZipEntryName("zip-entry-3b");
            fileSummaryList.add(fileSummary);
        }
        {
            FileDownloadSummary fileSummary = new FileDownloadSummary();
            fileSummary.setFailureMessage("You should never see this message");
            fileSummaryList.add(fileSummary);
        }
        {
            FileDownloadSummary fileSummary = new FileDownloadSummary();
            fileSummary.setFileHandleId("service-error-file-handle");
            fileSummary.setFailureMessage("service error");
            fileSummaryList.add(fileSummary);
        }
        {
            FileDownloadSummary fileSummary = new FileDownloadSummary();
            fileSummary.setFileHandleId("unknown-error-file-handle");
            fileSummaryList.add(fileSummary);
        }
        {
            FileDownloadSummary fileSummary = new FileDownloadSummary();
            fileSummary.setFileHandleId("extraneous-file-handle");
            fileSummary.setZipEntryName("extraneous-zip-entry");
            fileSummaryList.add(fileSummary);
        }

        setupTestWithArgs(DEFAULT_TEST_SCHEMA, csvContent, null, fileSummaryList);

        // execute
        SynapseDownloadFromTableResult result = task.call();

        // validate CSV
        List<String[]> parsedCsv = parseCsv(result.getCsvFile());
        assertEquals(parsedCsv.size(), 6);

        // header
        assertEquals(parsedCsv.get(0).length, 5);
        assertEquals(parsedCsv.get(0)[0], "recordId");
        assertEquals(parsedCsv.get(0)[1], "healthCode");
        assertEquals(parsedCsv.get(0)[2], "foo");
        assertEquals(parsedCsv.get(0)[3], "bar");
        assertEquals(parsedCsv.get(0)[4], "baz");

        // row 1
        assertEquals(parsedCsv.get(1).length, 5);
        assertEquals(parsedCsv.get(1)[0], "record-1");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(1)[1]));
        assertEquals(parsedCsv.get(1)[2], "4");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(1)[3]));
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(1)[4]));

        // row 2
        assertEquals(parsedCsv.get(2).length, 5);
        assertEquals(parsedCsv.get(2)[0], "record-2");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(2)[1]));
        assertEquals(parsedCsv.get(2)[2], "8");
        assertEquals(parsedCsv.get(2)[3], "zip-entry-2a");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(2)[4]));

        // row 3
        assertEquals(parsedCsv.get(3).length, 5);
        assertEquals(parsedCsv.get(3)[0], "record-3");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(3)[1]));
        assertEquals(parsedCsv.get(3)[2], "15");
        assertEquals(parsedCsv.get(3)[3], "zip-entry-3a");
        assertEquals(parsedCsv.get(3)[4], "zip-entry-3b");

        // row 4 - For the unknown error, instead of string matching, just verify that the error message exists and
        // that it's not the same as the file handle ID.
        assertEquals(parsedCsv.get(4).length, 5);
        assertEquals(parsedCsv.get(4)[0], "record-4");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(4)[1]));
        assertEquals(parsedCsv.get(4)[2], "16");
        assertEquals(parsedCsv.get(4)[3], "service error");
        assertNotEquals(parsedCsv.get(4)[4], "unknown-error-file-handle");
        assertFalse(Strings.isNullOrEmpty(parsedCsv.get(4)[4]));

        // row 5 - For the missing file handle, similar idea as the unknown error.
        assertEquals(parsedCsv.get(5).length, 5);
        assertEquals(parsedCsv.get(5)[0], "record-5");
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(5)[1]));
        assertEquals(parsedCsv.get(5)[2], "23");
        assertNotEquals(parsedCsv.get(5)[3], "missing-file-handle");
        assertFalse(Strings.isNullOrEmpty(parsedCsv.get(5)[3]));
        assertTrue(Strings.isNullOrEmpty(parsedCsv.get(5)[4]));

        // validate bulk download file - It's just "dummy zip content"
        try (Reader bulkDownloadFileReader = inMemoryFileHelper.getReader(result.getBulkDownloadFile())) {
            assertEquals(CharStreams.toString(bulkDownloadFileReader), "dummy zip content");
        }

        // validate the file handles we sent to Synapse for the bulk download
        Set<String> fileHandleIdSet = synapseFileHandleIdSetCaptor.getValue();
        assertEquals(fileHandleIdSet.size(), 6);
        assertTrue(fileHandleIdSet.contains("file-handle-2a"));
        assertTrue(fileHandleIdSet.contains("file-handle-3a"));
        assertTrue(fileHandleIdSet.contains("file-handle-3b"));
        assertTrue(fileHandleIdSet.contains("service-error-file-handle"));
        assertTrue(fileHandleIdSet.contains("unknown-error-file-handle"));
        assertTrue(fileHandleIdSet.contains("missing-file-handle"));

        postValidation(result);
    }

    @Test
    public void firstErrorCase() throws Exception {
        // Test getting an error on the first step (download CSV). This allows us to test that cleanup works even when
        // almost everything is null.

        // setup
        setupTestWithArgs(DEFAULT_TEST_SCHEMA, null, new TestSynapseException(), null);

        // execute
        Exception thrownEx = null;
        try {
            task.call();
            fail("expected exception");
        } catch (AsyncTaskExecutionException ex) {
            thrownEx = ex;
        }
        assertNotNull(thrownEx);
        postValidation(null);
    }

    @Test
    public void lastErrorCase() throws Exception {
        // Test getting an error on the last step (editing CSV). This allows us to test full cleanup, including the
        // bulk download file. Since the editCsv() step has no mock dependencies, we'll use a spy to get it to throw an
        // exception.

        // setup
        String csvContent = "\"recordId\",\"healthCode\",\"foo\",\"bar\",\"baz\"\n" +
                "\"record-1\",\"test-health-code\",\"1337\",\"test-file-handle\",";

        FileDownloadSummary fileSummary = new FileDownloadSummary();
        fileSummary.setFileHandleId("test-file-handle");
        fileSummary.setZipEntryName("test-zip-entry");

        setupTestWithArgs(DEFAULT_TEST_SCHEMA, csvContent, null, ImmutableList.of(fileSummary));

        task = spy(task);
        doThrow(new AsyncTaskExecutionException()).when(task).editCsv();

        // execute
        Exception thrownEx = null;
        try {
            task.call();
            fail("expected exception");
        } catch (AsyncTaskExecutionException ex) {
            thrownEx = ex;
        }
        assertNotNull(thrownEx);

        // validate the file handles we sent to Synapse for the bulk download
        Set<String> fileHandleIdSet = synapseFileHandleIdSetCaptor.getValue();
        assertEquals(fileHandleIdSet.size(), 1);
        assertTrue(fileHandleIdSet.contains("test-file-handle"));

        postValidation(null);
    }

    private void setupTestWithArgs(UploadSchema schema, String csvContent, SynapseException csvException,
            List<FileDownloadSummary> fileSummaryList) throws Exception {
        // mock file helper and temp dir
        inMemoryFileHelper = new InMemoryFileHelper();
        tmpDir = inMemoryFileHelper.createTempDir();

        // set up params and task
        SynapseDownloadFromTableParameters params = new SynapseDownloadFromTableParameters.Builder()
                .withSynapseTableId("test-table-id").withHealthCode("test-health-code")
                .withStartDate(LocalDate.parse("2015-03-09")).withEndDate(LocalDate.parse("2015-09-16"))
                .withTempDir(tmpDir).withSchema(schema).build();
        task = new SynapseDownloadFromTableTask(params);
        task.setFileHelper(inMemoryFileHelper);

        // mock Synapse CSV content
        SynapseHelper mockSynapseHelper = mock(SynapseHelper.class);
        synapseQueryCaptor = ArgumentCaptor.forClass(String.class);
        when(mockSynapseHelper.generateFileHandleFromTableQuery(synapseQueryCaptor.capture(), eq("test-table-id")))
                .thenReturn("query-csv-file-handle-id");
        doAnswer(invocation -> {
            if (csvException != null) {
                throw csvException;
            }

            File targetFile = invocation.getArgumentAt(1, File.class);
            try (Writer targetFileWriter = inMemoryFileHelper.getWriter(targetFile)) {
                targetFileWriter.write(csvContent);
            }

            // Needed because Answer declares a return type, even if it's null.
            return null;
        }).when(mockSynapseHelper).downloadFileHandle(eq("query-csv-file-handle-id"), any(File.class));

        if (fileSummaryList != null) {
            // mock Synapse bulk download
            BulkFileDownloadResponse bulkDownloadResponse = new BulkFileDownloadResponse();
            bulkDownloadResponse.setResultZipFileHandleId("bulk-download-file-handle-id");
            bulkDownloadResponse.setFileSummary(fileSummaryList);

            synapseFileHandleIdSetCaptor = ArgumentCaptor.forClass(Set.class);
            when(mockSynapseHelper.generateBulkDownloadFileHandle(eq("test-table-id"),
                    synapseFileHandleIdSetCaptor.capture())).thenReturn(bulkDownloadResponse);

            doAnswer(invocation -> {
                File targetFile = invocation.getArgumentAt(1, File.class);
                try (Writer targetFileWriter = inMemoryFileHelper.getWriter(targetFile)) {
                    targetFileWriter.write("dummy zip content");
                }

                // Needed because Answer declares a return type, even if it's null.
                return null;
            }).when(mockSynapseHelper).downloadFileHandle(eq("bulk-download-file-handle-id"), any(File.class));
        }

        task.setSynapseHelper(mockSynapseHelper);
    }

    // Direct string matching means we tightly couple to the CSV writer implementation. Instead, re-parse the file as a
    // CSV and check the values are what we expect.
    private List<String[]> parseCsv(File csvFile) throws Exception {
        try (CSVReader csvFileReader = new CSVReader(inMemoryFileHelper.getReader(csvFile))) {
            return csvFileReader.readAll();
        }
    }

    private void postValidation(SynapseDownloadFromTableResult result) throws Exception {
        // SynapseDownloadFromTableTask should only leave behind the files it returned and the temp dir. Clean these
        // files up (which is what the packager would do) and then verify that the mock file system is now empty.
        if (result != null) {
            if (result.getCsvFile() != null) {
                inMemoryFileHelper.deleteFile(result.getCsvFile());
            }

            if (result.getBulkDownloadFile() != null) {
                inMemoryFileHelper.deleteFile(result.getBulkDownloadFile());
            }
        }

        inMemoryFileHelper.deleteDir(tmpDir);
        assertTrue(inMemoryFileHelper.isEmpty());

        // Validate the Synapse query contains the expected values. Don't string match the entire string. Just validate
        // that table ID, health code, start date, and end date were used.
        String query = synapseQueryCaptor.getValue();
        assertTrue(query.contains("test-table-id"));
        assertTrue(query.contains("test-health-code"));
        assertTrue(query.contains("2015-03-09"));
        assertTrue(query.contains("2015-09-16"));
    }
}