gt:                 if (t.getStatus() == TaskStatus.FAULTY) {
context: /*
   * ProActive Parallel Suite(TM):
   * The Open Source library for parallel and distributed
   * Workflows & Scheduling, Orchestration, Cloud Automation
   * and Big Data Analysis on Enterprise Grids & Clouds.
   *
   * Copyright (c) 2007 - 2017 ActiveEon
   * Contact: contact@activeeon.com
   *
   * This library is free software: you can redistribute it and/or
   * modify it under the terms of the GNU Affero General Public License
   * as published by the Free Software Foundation: version 3 of
   * the License.
   *
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU Affero General Public License for more details.
   *
   * You should have received a copy of the GNU Affero General Public License
   * along with this program. If not, see <http://www.gnu.org/licenses/>.
   *
   * If needed, contact us to obtain a release under GPL Version 2 or 3
   * or a different license than the AGPL.
   */
  package functionaltests.utils;
  
  import static org.junit.Assert.assertFalse;
  import static org.junit.Assert.fail;
  
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Calendar;
  import java.util.HashMap;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.concurrent.TimeUnit;
  
  import org.junit.Assert;
  import org.objectweb.proactive.api.PAFuture;
  import org.objectweb.proactive.core.ProActiveTimeoutException;
  import org.objectweb.proactive.core.config.CentralPAPropertyRepository;
  import org.objectweb.proactive.core.node.Node;
  import org.objectweb.proactive.core.node.NodeException;
  import org.objectweb.proactive.core.node.NodeFactory;
  import org.objectweb.proactive.core.process.JVMProcessImpl;
  import org.objectweb.proactive.extensions.pnp.PNPConfig;
  import org.ow2.proactive.db.SortOrder;
  import org.ow2.proactive.db.SortParameter;
  import org.ow2.proactive.resourcemanager.common.NodeState;
  import org.ow2.proactive.resourcemanager.common.event.RMEventType;
  import org.ow2.proactive.resourcemanager.common.event.RMNodeEvent;
  import org.ow2.proactive.resourcemanager.frontend.ResourceManager;
  import org.ow2.proactive.resourcemanager.utils.RMNodeStarter;
  import org.ow2.proactive.scheduler.common.JobFilterCriteria;
  import org.ow2.proactive.scheduler.common.JobSortParameter;
  import org.ow2.proactive.scheduler.common.Scheduler;
  import org.ow2.proactive.scheduler.common.SchedulerAuthenticationInterface;
  import org.ow2.proactive.scheduler.common.SchedulerEvent;
  import org.ow2.proactive.scheduler.common.exception.UnknownJobException;
  import org.ow2.proactive.scheduler.common.job.*;
  import org.ow2.proactive.scheduler.common.job.factories.JobFactory;
  import org.ow2.proactive.scheduler.common.task.TaskInfo;
  import org.ow2.proactive.scheduler.common.task.TaskResult;
  import org.ow2.proactive.scheduler.common.task.TaskState;
  import org.ow2.proactive.scheduler.common.task.TaskStatus;
  
  import com.google.common.collect.ImmutableList;
  
  import functionaltests.monitor.JobEventMonitor;
  import functionaltests.monitor.RMMonitorsHandler;
  import functionaltests.monitor.SchedulerMonitorsHandler;
  
  
  /**
   * Static helpers that provide main operations for Scheduler functional test.
   *
   * The Scheduler instance is static, in order to keep it running across tests if possible.
   * If a Scheduler is explicitly started with a configuration different than the running one
   * the Scheduler will kill and started with this new configuration.
   *
   * For waitForEvent**() methods, it acts as Producer-consumer mechanism ;
   * a Scheduler produce events that are memorized,
   * and waiting methods waitForEvent**() are consumers of these events.
   * It means that an event asked to be waited for by a call to waitForEvent**() methods, is removed
   * after its occurrence. On the contrary, an event is kept till a waitForEvent**() for this event
   * has been called.
   *
   * waitForTerminatedJob() method doesn't act as other waitForEvent**() Methods.
   * This method deduce a job finished from current Scheduler's job states and received event.
   * This method can also be used for testing for job submission with killing and restarting
   * Scheduler.
   *
   * @author ProActive team
   * @since ProActive Scheduling 1.0
   */
  public class SchedulerTHelper {
  
      private static TestScheduler scheduler = new TestScheduler();
  
      public static final String extraNS = "extra";
  
      static {
          Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
              @Override
              public void run() {
                  try {
                      scheduler.kill();
                  } catch (Exception e) {
                      e.printStackTrace();
                  }
              }
          }));
      }
  
      // can be changed by starting the Scheduler manually
      private SchedulerTestConfiguration currentTestConfiguration = SchedulerTestConfiguration.defaultConfiguration();
  
      /**
       * Creates a test scheduler
       * @param restart if true then a new scheduler will be recreated
       * @throws Exception
       */
      public SchedulerTHelper(boolean restart) throws Exception {
          this(restart, false);
      }
  
      /**
       * Creates a test scheduler
       *
       * @param restart if true then a new scheduler will be recreated
       * @param emptyRM if true then a the scheduler must have zero node
       * @throws Exception
       */
      public SchedulerTHelper(boolean restart, boolean emptyRM) throws Exception {
          if (emptyRM) {
              startScheduler(restart, SchedulerTestConfiguration.emptyResourceManager());
          } else {
              startScheduler(restart, SchedulerTestConfiguration.defaultConfiguration());
          }
      }
  
      /**
       * Creates a test scheduler with a given configuration
       *
       * @param restart       if true then a new scheduler will be recreated
       * @param configuration configuration to use
       * @throws Exception
       */
      public SchedulerTHelper(boolean restart, SchedulerTestConfiguration configuration) throws Exception {
          startScheduler(restart, configuration);
      }
  
      /**
       * Creates a test scheduler with a given configuration
       *
       * @param configuration the Scheduler configuration file to use (default is
       *                      functionalTSchedulerProperties.ini)
       * 			null to use the default one.
       * @throws Exception if an error occurs.
       */
      public SchedulerTHelper(boolean restart, String configuration) throws Exception {
          startScheduler(restart, SchedulerTestConfiguration.customSchedulerConfig(configuration));
      }
  
      /**
       * Creates a test scheduler with a given configuration and empty RM
       *
       * @param configuration the Scheduler configuration file to use (default is
       *                      functionalTSchedulerProperties.ini)
       *                      null to use the default one.
       * @throws Exception if an error occurs.
       */
      public SchedulerTHelper(boolean restart, boolean emptyRM, String configuration) throws Exception {
          if (emptyRM) {
              startScheduler(restart, SchedulerTestConfiguration.emptyRMandCustomSchedulerConfig(configuration));
          } else {
              startScheduler(restart, SchedulerTestConfiguration.customSchedulerConfig(configuration));
          }
      }
  
      public SchedulerTHelper(boolean deployLocalNodes, String schedPropertiesFilePath, String rmPropertiesFilePath,
              String rmUrl, boolean cleanTMP) throws Exception {
          if (cleanTMP) {
              startScheduler(deployLocalNodes, schedPropertiesFilePath, rmPropertiesFilePath, rmUrl);
          } else {
              startSchedulerWithoutCleaningTMP(deployLocalNodes, schedPropertiesFilePath, rmPropertiesFilePath, rmUrl);
          }
      }
  
      /**
       * Creates a test scheduler with a given number of nodes,
       * and given scheduler and resource manager configurations
       * @param localnodes number of nodes to create
       * @param schedPropertiesFilePath configuration file for the scheduler
       * @param rmPropertiesFilePath configuration file for the rm
       * @param rmUrl url of the resource manager
       * @throws Exception
       */
      public SchedulerTHelper(boolean localnodes, String schedPropertiesFilePath, String rmPropertiesFilePath,
              String rmUrl) throws Exception {
          startScheduler(localnodes, schedPropertiesFilePath, rmPropertiesFilePath, rmUrl);
      }
  
      private void startScheduler(String configuration) throws Exception {
          startScheduler(true, SchedulerTestConfiguration.customSchedulerConfig(configuration));
      }
  
      private void startScheduler(boolean localnodes, String schedPropertiesFilePath, String rmPropertiesFilePath,
              String rmUrl) throws Exception {
          SchedulerTestConfiguration configuration = new SchedulerTestConfiguration(schedPropertiesFilePath,
                                                                                    rmPropertiesFilePath,
                                                                                    localnodes,
                                                                                    TestScheduler.PNP_PORT,
                                                                                    rmUrl);
          startScheduler(true, configuration);
      }
  
      private void startSchedulerWithoutCleaningTMP(boolean localnodes, String schedPropertiesFilePath,
              String rmPropertiesFilePath, String rmUrl) throws Exception {
          SchedulerTestConfiguration configuration = new SchedulerTestConfiguration(schedPropertiesFilePath,
                                                                                    rmPropertiesFilePath,
                                                                                    localnodes,
                                                                                    TestScheduler.PNP_PORT,
                                                                                    rmUrl);
          startSchedulerWithoutCleaningTMP(true, configuration);
      }
  
      private void startScheduler(boolean restart, SchedulerTestConfiguration configuration) throws Exception {
          if (restart || !scheduler.isStartedWithSameConfiguration(configuration)) {
              log("Kill previous Scheduler and alive connexions");
              killScheduler();
              log("Starting Scheduler");
              scheduler.start(configuration, true);
              RMTestUser.getInstance().connect(TestUsers.DEMO, scheduler.getRMUrl());
          }
          currentTestConfiguration = configuration;
      }
  
      private void startSchedulerWithoutCleaningTMP(boolean restart, SchedulerTestConfiguration configuration)
              throws Exception {
          if (restart || !scheduler.isStartedWithSameConfiguration(configuration)) {
              log("Kill previous Scheduler and alive connexions");
              killScheduler();
              log("Starting Scheduler");
              scheduler.start(configuration, false);
              RMTestUser.getInstance().connect(TestUsers.DEMO, scheduler.getRMUrl());
          }
          currentTestConfiguration = configuration;
      }
  
      public boolean isStarted() {
          return scheduler.isStarted();
      }
  
      /**
       * Kill the forked Scheduler if exists.
       */
      public void killScheduler() throws Exception {
          SchedulerTestUser.getInstance().schedulerIsRestarted();
          RMTestUser.getInstance().disconnectFromRM();
          log("Killing scheduler process");
          scheduler.kill();
      }
  
      /**
       * Restart the scheduler using a forked JVM and all children Nodes.
       * User or administrator interface is not reconnected automatically.
       *
       * @param configuration the Scheduler configuration file to use (default is
       *                      functionalTSchedulerProperties.ini)
       * 			null to use the default one.
       * @throws Exception
       */
      public void killSchedulerAndNodesAndRestart(String configuration) throws Exception {
          killScheduler();
          startScheduler(configuration);
      }
  
      /**
       * Log a String on console.
       */
      public static void log(String s) {
          System.out.println("------------------------------" +
                             String.format("[%1$tH:%1$tM:%1$tS,%1$tL] ", Calendar.getInstance().getTime()) + s);
      }
  
      public static void log(Exception e) {
          e.printStackTrace();
      }
  
      /**
       * Return Scheduler authentication interface. Start Scheduler with test
       * configuration file, if scheduler is not yet started.
       * @throws Exception
       */
      public SchedulerAuthenticationInterface getSchedulerAuth() throws Exception {
          return scheduler.getAuth();
      }
  
      /**
       * Return Scheduler's interface. Start Scheduler if needed,
       * connect as administrator if needed (if not yet connected as user).
       *
       * WARNING : if there was a previous connection as User, this connection is shut down.
       * And so some event can be missed by event receiver, between disconnection and reconnection
       * (only one connection to Scheduler per body is possible).
       *
       * @return scheduler interface
       * @throws Exception if an error occurs.
       */
      public Scheduler getSchedulerInterface() throws Exception {
          getResourceManager(); // get ready to receive RM events as well
          return getSchedulerInterface(TestUsers.DEMO);
      }
  
      public Scheduler getSchedulerInterface(UserType userType) throws Exception {
          switch (userType) {
              case ADMIN:
                  return getSchedulerInterface(TestUsers.ADMIN);
              case USER:
                  return getSchedulerInterface(TestUsers.DEMO);
          }
          return null;
      }
  
      /**
       * Return Scheduler's interface. Start Scheduler if needed,
       * connect as administrator if needed (if not yet connected as user).
       *
       * WARNING : if there was a previous connection as User, this connection is shut down.
       * And so some event can be missed by event receiver, between disconnection and reconnection
       * (only one connection to Scheduler per body is possible).
       *
       * @param user Type of user
       * @return scheduler interface
       * @throws Exception if an error occurs.
       */
      public Scheduler getSchedulerInterface(TestUsers user) throws Exception {
  
          if (!SchedulerTestUser.getInstance().is(user)) { // changing user on the fly
              SchedulerTestUser.getInstance().disconnectFromScheduler();
              SchedulerTestUser.getInstance().connect(user, scheduler.getUrl());
          }
  
          if (!SchedulerTestUser.getInstance().isConnected()) {
              SchedulerTestUser.getInstance().connect(user, scheduler.getUrl());
          }
  
          return SchedulerTestUser.getInstance().getScheduler();
      }
  
      /**
       * Return Scheduler's interface. Start Scheduler if needed,
       * connect as administrator if needed (if not yet connected as user).
       *
       * WARNING : if there was a previous connection as User, this connection is shut down.
       * And so some event can be missed by event receiver, between disconnection and reconnection
       * (only one connection to Scheduler per body is possible).
       *
       * @param username username
       * @param password password
       * @return scheduler interface
       * @throws Exception if an error occurs.
       */
      public Scheduler getSchedulerInterface(String username, String password, byte[] key) throws Exception {
  
          if (!SchedulerTestUser.getInstance().is(username, password)) { // changing user on the fly
              SchedulerTestUser.getInstance().disconnectFromScheduler();
              SchedulerTestUser.getInstance().connect(username, password, key, scheduler.getUrl());
          }
  
          if (!SchedulerTestUser.getInstance().isConnected()) {
              SchedulerTestUser.getInstance().connect(username, password, key, scheduler.getUrl());
          }
  
          return SchedulerTestUser.getInstance().getScheduler();
      }
  
      /**
       * Submit a job, and return immediately.
       * Connect as user if needed (if not yet connected as user).
       * @param jobToSubmit job object to schedule.
       * @return JobId the job's identifier corresponding to submission.
       * @throws Exception if an error occurs at job submission.
       */
      public JobId submitJob(Job jobToSubmit) throws Exception {
          Scheduler userInt = getSchedulerInterface();
          return userInt.submit(jobToSubmit);
      }
  
      public JobId submitJob(String jobDescPath) throws Exception {
          Job jobToSubmit = JobFactory.getFactory().createJob(jobDescPath);
          Scheduler userInt = getSchedulerInterface();
          return userInt.submit(jobToSubmit);
      }
  
      /**
       * Submits a job with a list of variables
       *
       * @param jobDescPath
       * @param variables
       * @return
       * @throws Exception
       */
      public JobId submitJob(String jobDescPath, Map<String, String> variables) throws Exception {
          Job jobToSubmit = JobFactory.getFactory().createJob(jobDescPath, variables, null);
          Scheduler userInt = getSchedulerInterface();
          return userInt.submit(jobToSubmit);
      }
  
      public JobId submitJobWithGI(String jobDescPath, Map<String, String> generalInformation) throws Exception {
          Job jobToSubmit = JobFactory.getFactory().createJob(jobDescPath);
          jobToSubmit.setGenericInformation(generalInformation);
          Scheduler userInt = getSchedulerInterface();
          return userInt.submit(jobToSubmit);
      }
  
      /**
       * Kills a job
       * @return success or failure at killing the job
       * @throws Exception
       */
      public boolean killJob(String jobId) throws Exception {
          Scheduler userInt = getSchedulerInterface();
          return userInt.killJob(jobId);
      }
  
      /**
       * Kills a tzsk
       *
       * @return success or failure at killing the task
       * @throws Exception
       */
      public boolean killTask(String jobId, String taskName) throws Exception {
          Scheduler userInt = getSchedulerInterface();
          return userInt.killTask(jobId, taskName);
      }
  
      /**
       * Remove a job from Scheduler database.
       * connect as user if needed (if not yet connected as user).
       * @param id of the job to remove from database.
       * @throws Exception if an error occurs at job removal.
       */
      public void removeJob(JobId id) throws Exception {
          Scheduler userInt = getSchedulerInterface();
          userInt.removeJob(id);
      }
  
      public void restartAllInErrorTasks(String jobId) throws Exception {
          Scheduler userInt = getSchedulerInterface();
          userInt.restartAllInErrorTasks(jobId);
      }
  
      /**
       * Creates and submit a job from an XML job descriptor, and check, with assertions,
       * event related to this job submission :
       * 1/ job submitted event
       * 2/ job passing from pending to running (with state set to running).
       * 3/ job passing from running to finished (with state set to finished).
       * 4/ every task finished without error
       *
       * Then returns.
       *
       * This is the simplest events sequence of a job submission. If you need to test
       * specific events or task states (failures, rescheduling etc, you must not use this
       * helper and check events sequence with waitForEvent**() functions.
       *
       * @param jobDescPath path to an XML job descriptor to submit
       * @return JobId, the job's identifier.
       * @throws Exception if an error occurs at job creation/submission, or during
       * verification of events sequence.
       */
      public JobId testJobSubmission(String jobDescPath) throws Exception {
          Job jobToTest = JobFactory.getFactory().createJob(jobDescPath);
          return testJobSubmission(jobToTest, false);
      }
  
      /**
       * Creates and submit a job from an XML job descriptor, and check, with assertions,
       * event related to this job submission :
       * 1/ job submitted event
       * 2/ job passing from pending to running (with state set to running).
       * 3/ job passing from running to finished (with state set to finished).
       * 4/ every task finished without error
       * <p>
       * Then returns.
       * <p>
       * This is the simplest events sequence of a job submission. If you need to test
       * specific events or task states (failures, rescheduling etc, you must not use this
       * helper and check events sequence with waitForEvent**() functions.
       *
       * @param jobDescPath   path to an XML job descriptor to submit
       * @param acceptSkipped if true then skipped task will not fail the test
       * @return JobId, the job's identifier.
       * @throws Exception if an error occurs at job creation/submission, or during
       *                   verification of events sequence.
       */
      public JobId testJobSubmission(String jobDescPath, boolean acceptSkipped) throws Exception {
          Job jobToTest = JobFactory.getFactory().createJob(jobDescPath);
          return testJobSubmission(jobToTest, acceptSkipped);
      }
  
      /**
       * Creates and submit a job from an XML job descriptor, and check, with assertions,
       * event related to this job submission :
       * 1/ job submitted event
       * 2/ job passing from pending to running (with state set to running).
       * 3/ job passing from running to finished (with state set to finished).
       * 4/ every task finished without error
       * <p>
       * Then returns.
       * <p>
       * This is the simplest events sequence of a job submission. If you need to test
       * specific events or task states (failures, rescheduling etc, you must not use this
       * helper and check events sequence with waitForEvent**() functions.
       *
       * @param jobToSubmit job object to schedule.
       * @return JobId, the job's identifier.
       * @throws Exception if an error occurs at job submission, or during
       *                   verification of events sequence.
       */
      public JobId testJobSubmission(Job jobToSubmit) throws Exception {
          return testJobSubmission(jobToSubmit, false);
      }
  
      /**
       * Creates and submit a job from an XML job descriptor, and check, with assertions,
       * event related to this job submission :
       * 1/ job submitted event
       * 2/ job passing from pending to running (with state set to running).
       * 3/ job passing from running to finished (with state set to finished).
       * 4/ every task finished without error
       *
       * Then returns.
       *
       * This is the simplest events sequence of a job submission. If you need to test
       * specific events or task states (failures, rescheduling etc, you must not use this
       * helper and check events sequence with waitForEvent**() functions.
       *
       * @param jobToSubmit job object to schedule.
       * @param acceptSkipped if true then skipped task will not fail the test
       * @return JobId, the job's identifier.
       * @throws Exception if an error occurs at job submission, or during
       * verification of events sequence.
       */
      public JobId testJobSubmission(Job jobToSubmit, boolean acceptSkipped) throws Exception {
          return testJobSubmission(jobToSubmit, acceptSkipped, true);
      }
  
      /**
       * Creates and submit a job from an XML job descriptor, and check, with assertions,
       * event related to this job submission :
       * 1/ job submitted event
       * 2/ job passing from pending to running (with state set to running).
       * 3/ job passing from running to finished (with state set to finished).
       * 4/ every task finished with or without error (configurable)
       * <p>
       * Then returns.
       * <p>
       * This is the simplest events sequence of a job submission. If you need to test
       * specific events or task states (failures, rescheduling etc, you must not use this
       * helper and check events sequence with waitForEvent**() functions.
       *
       * @param jobToSubmit     job object to schedule.
       * @param acceptSkipped   if true then skipped task will not fail the test
       * @param failIfTaskError if true then the test will fail if a task was in error
       * @return JobId, the job's identifier.
       * @throws Exception if an error occurs at job submission, or during
       *                   verification of events sequence.
       */
      public JobId testJobSubmission(Job jobToSubmit, boolean acceptSkipped, boolean failIfTaskError) throws Exception {
          return testJobSubmission(getSchedulerInterface(), jobToSubmit, acceptSkipped, failIfTaskError);
      }
  
      /**
       * Creates and submit a job from an XML job descriptor, and check, with assertions,
       * event related to this job submission :
       * 1/ job submitted event
       * 2/ job passing from pending to running (with state set to running).
       * 3/ job passing from running to finished (with state set to finished).
       * 4/ every task finished with or without error (configurable)
       * <p>
       * Then returns.
       * <p>
       * This is the simplest events sequence of a job submission. If you need to test
       * specific events or task states (failures, rescheduling etc, you must not use this
       * helper and check events sequence with waitForEvent**() functions.
       *
       * @param userInt         scheduler interface
       * @param jobToSubmit     job object to schedule.
       * @param acceptSkipped   if true then skipped task will not fail the test
       * @param failIfTaskError if true then the test will fail if a task was in error
       * @return JobId, the job's identifier.
       * @throws Exception if an error occurs at job submission, or during
       *                   verification of events sequence.
       */
      public JobId testJobSubmission(Scheduler userInt, Job jobToSubmit, boolean acceptSkipped, boolean failIfTaskError)
              throws Exception {
  
          JobId id = userInt.submit(jobToSubmit);
  
          log("Job submitted, id " + id.toString());
  
          log("Waiting for jobSubmitted");
          JobState receivedState = waitForEventJobSubmitted(id);
          Assert.assertEquals(id, receivedState.getId());
  
          log("Waiting for job running");
          JobInfo jInfo = waitForEventJobRunning(id);
  
          Assert.assertEquals(jInfo.getJobId(), id);
          Assert.assertEquals("Job " + jInfo.getJobId(), JobStatus.RUNNING, jInfo.getStatus());
  
          log("Waiting for job finished");
          jInfo = waitForEventJobFinished(userInt, id);
          Assert.assertEquals("Job " + jInfo.getJobId(), JobStatus.FINISHED, jInfo.getStatus());
          log("Job finished");
  
          boolean taskError = false;
          String message = "";
  
          if (jobToSubmit instanceof TaskFlowJob) {
  
              JobState jobState = userInt.getJobState(id);
  
              for (TaskState t : jobState.getTasks()) {
                  log("Looking at the result of task : " + t.getName());
  