text:
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;

import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;

import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import org.apache.hadoop.yarn.util.ConverterUtils;

import com.google.common.annotations.VisibleForTesting;

public class ContainerLaunch implements Callable<Integer> {

  private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);

  public static final String CONTAINER_SCRIPT =
    Shell.appendScriptExtension("launch_container");
  public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";

  private static final String PID_FILE_NAME_FMT = "%s.pid";
  private static final String EXIT_CODE_FILE_SUFFIX = ".exitcode";

  protected final Dispatcher dispatcher;
  protected final ContainerExecutor exec;
  private final Application app;
  protected final Container container;
  private final Configuration conf;
  private final Context context;
  private final ContainerManagerImpl containerManager;

  protected AtomicBoolean shouldLaunchContainer = new AtomicBoolean(false);
  protected AtomicBoolean completed = new AtomicBoolean(false);

  private long sleepDelayBeforeSigKill = 250;
  private long maxKillWaitTime = 2000;

  protected Path pidFilePath = null;

  private final LocalDirsHandlerService dirsHandler;

  public ContainerLaunch(Context context, Configuration configuration,
      Dispatcher dispatcher, ContainerExecutor exec, Application app,
      Container container, LocalDirsHandlerService dirsHandler,
      ContainerManagerImpl containerManager) {
    this.context = context;
    this.conf = configuration;
    this.app = app;
    this.exec = exec;
    this.container = container;
    this.dispatcher = dispatcher;
    this.dirsHandler = dirsHandler;
    this.containerManager = containerManager;
    this.sleepDelayBeforeSigKill =
        conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
            YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
    this.maxKillWaitTime =
        conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
            YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
  }

  @VisibleForTesting
  public static String expandEnvironment(String var,
      Path containerLogDir) {
    var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
      containerLogDir.toString());
    var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
      File.pathSeparator);

    // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
    // as %VAR% and on Linux replaced as "$VAR"
    if (Shell.WINDOWS) {
      var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
    } else {
      var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
      var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
    }
    return var;
  }

  @Override
  @SuppressWarnings("unchecked") // dispatcher not typed
  public Integer call() {
    final ContainerLaunchContext launchContext = container.getLaunchContext();
    Map<Path,List<String>> localResources = null;
    ContainerId containerID = container.getContainerId();
    String containerIdStr = ConverterUtils.toString(containerID);
    final List<String> command = launchContext.getCommands();
    int ret = -1;

    // CONTAINER_KILLED_ON_REQUEST should not be missed if the container
    // is already at KILLING
    if (container.getContainerState() == ContainerState.KILLING) {
      dispatcher.getEventHandler().handle(
          new ContainerExitEvent(containerID,
              ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
              Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
                  ExitCode.TERMINATED.getExitCode(),
              "Container terminated before launch."));
      return 0;
    }

    Path containerLogDir;
    try {
      localResources = container.getLocalizedResources();
      if (localResources == null) {
        throw RPCUtil.getRemoteException(
            "Unable to get local resources when Container " + containerID +
            " is at " + container.getContainerState());
      }

      final String user = container.getUser();
      // /////////////////////////// Variable expansion
      // Before the container script gets written out.
      List<String> newCmds = new ArrayList<String>(command.size());
      String appIdStr = app.getAppId().toString();
      String relativeContainerLogDir = ContainerLaunch
          .getRelativeContainerLogDir(appIdStr, containerIdStr);
      containerLogDir =
          dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
      for (String str : command) {
        // TODO: Should we instead work via symlinks without this grammar?
        newCmds.add(expandEnvironment(str, containerLogDir));
      }
      launchContext.setCommands(newCmds);

      Map<String, String> environment = launchContext.getEnvironment();
      // Make a copy of env to iterate & do variable expansion
      for (Entry<String, String> entry : environment.entrySet()) {
        String value = entry.getValue();
        value = expandEnvironment(value, containerLogDir);
        entry.setValue(value);
      }
      // /////////////////////////// End of variable expansion

      FileContext lfs = FileContext.getLocalFSFileContext();

      Path nmPrivateContainerScriptPath =
          dirsHandler.getLocalPathForWrite(
              getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
                  + CONTAINER_SCRIPT);
      Path nmPrivateTokensPath =
          dirsHandler.getLocalPathForWrite(
              getContainerPrivateDir(appIdStr, containerIdStr)
                  + Path.SEPARATOR
                  + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                      containerIdStr));
      Path nmPrivateClasspathJarDir =
          dirsHandler.getLocalPathForWrite(
              getContainerPrivateDir(appIdStr, containerIdStr));
      DataOutputStream containerScriptOutStream = null;
      DataOutputStream tokensOutStream = null;

      // Select the working directory for the container
      Path containerWorkDir =
          dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE
              + Path.SEPARATOR + user + Path.SEPARATOR
              + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
              + Path.SEPARATOR + containerIdStr,
              LocalDirAllocator.SIZE_UNKNOWN, false);

      String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);

      // pid file should be in nm private dir so that it is not
      // accessible by users
      pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
      List<String> localDirs = dirsHandler.getLocalDirs();
      List<String> logDirs = dirsHandler.getLogDirs();

      List<String> containerLogDirs = new ArrayList<String>();
      for( String logDir : logDirs) {
        containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
      }

      if (!dirsHandler.areDisksHealthy()) {
        ret = ContainerExitStatus.DISKS_FAILED;
        throw new IOException("Most of the disks failed. "
            + dirsHandler.getDisksHealthReport(false));
      }

      List<String> containerLocalDirs = new ArrayList<>(localDirs.size());
      try {
        // /////////// Write out the container-script in the nmPrivate space.
        List<Path> appDirs = new ArrayList<Path>(localDirs.size());
        for (String localDir : localDirs) {
          Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
          Path userdir = new Path(usersdir, user);
          Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
          appDirs.add(new Path(appsdir, appIdStr));

          String containerLocalDir = localDir + Path.SEPARATOR +
              ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
              + Path.SEPARATOR
              + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr
              + Path.SEPARATOR;

          containerLocalDirs.add(containerLocalDir);
        }
        containerScriptOutStream =
          lfs.create(nmPrivateContainerScriptPath,
              EnumSet.of(CREATE, OVERWRITE));

        // Set the token location too.
        environment.put(
            ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
            new Path(containerWorkDir,
                FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
        // Sanitize the container's environment
        sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
          localResources, nmPrivateClasspathJarDir);

        // Write out the environment
        exec.writeLaunchEnv(containerScriptOutStream, environment,
          localResources, launchContext.getCommands(),
            new Path(containerLogDirs.get(0)));

        // /////////// End of writing out container-script

        // /////////// Write out the container-tokens in the nmPrivate space.
        tokensOutStream =
            lfs.create(nmPrivateTokensPath, EnumSet.of(CREATE, OVERWRITE));
        Credentials creds = container.getCredentials();
        creds.writeTokenStorageToStream(tokensOutStream);
        // /////////// End of writing out container-tokens
      } finally {
        IOUtils.cleanup(LOG, containerScriptOutStream, tokensOutStream);
      }

      // LaunchContainer is a blocking call. We are here almost means the
      // container is launched, so send out the event.
      dispatcher.getEventHandler().handle(new ContainerEvent(
            containerID,
            ContainerEventType.CONTAINER_LAUNCHED));
      context.getNMStateStore().storeContainerLaunched(containerID);

      // Check if the container is signalled to be killed.
      if (!shouldLaunchContainer.compareAndSet(false, true)) {
        LOG.info("Container " + containerIdStr + " not launched as "
            + "cleanup already called");
        ret = ExitCode.TERMINATED.getExitCode();
      }
      else {
        exec.activateContainer(containerID, pidFilePath);
        ret = exec.launchContainer(new ContainerStartContext.Builder()
            .setContainer(container)
            .setLocalizedResources(localResources)
            .setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath)
            .setNmPrivateTokensPath(nmPrivateTokensPath)
            .setUser(user)
            .setAppId(appIdStr)
            .setContainerWorkDir(containerWorkDir)
            .setLocalDirs(localDirs)
            .setLogDirs(logDirs)
            .setContainerLocalDirs(containerLocalDirs)
            .setContainerLogDirs(containerLogDirs)
            .build());
      }
    } catch (Throwable e) {
      LOG.warn("Failed to launch container.", e);
      dispatcher.getEventHandler().handle(new ContainerExitEvent(
          containerID, ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
          e.getMessage()));
      return ret;
    } finally {
      completed.set(true);
      exec.deactivateContainer(containerID);
      try {
        context.getNMStateStore().storeContainerCompleted(containerID, ret);
      } catch (IOException e) {
        LOG.error("Unable to set exit code for container " + containerID);
      }
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Container " + containerIdStr + " completed with exit code "
                + ret);
    }

    StringBuilder diagnosticInfo =
        new StringBuilder("Container exited with a non-zero exit code ");
    diagnosticInfo.append(ret);
    diagnosticInfo.append(". ");
    if (ret == ExitCode.FORCE_KILLED.getExitCode()
        || ret == ExitCode.TERMINATED.getExitCode()) {
      // If the process was killed, Send container_cleanedup_after_kill and
      // just break out of this method.
      dispatcher.getEventHandler().handle(
            new ContainerExitEvent(containerID,
                ContainerEventType.CONTAINER_KILLED_ON_REQUEST, ret,
                diagnosticInfo.toString()));
      return ret;
    }

    if (ret != 0) {
      handleContainerExitWithFailure(containerID, ret, containerLogDir,
          diagnosticInfo);
      return ret;
    }

    LOG.info("Container " + containerIdStr + " succeeded ");
    dispatcher.getEventHandler().handle(
        new ContainerEvent(containerID,
            ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS));
    return 0;
  }

  /**
   * Tries to tail and fetch TAIL_SIZE_IN_BYTES of data from the error log.
   * ErrorLog filename is not fixed and depends upon app, hence file name
   * pattern is used.
   * @param containerID
   * @param ret
   * @param containerLogDir
   * @param diagnosticInfo
   */
  @SuppressWarnings("unchecked")
  private void handleContainerExitWithFailure(ContainerId containerID, int ret,
      Path containerLogDir, StringBuilder diagnosticInfo) {
    LOG.warn(diagnosticInfo);

    String errorFileNamePattern =
        conf.get(YarnConfiguration.NM_CONTAINER_STDERR_PATTERN,
            YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_PATTERN);
    FSDataInputStream errorFileIS = null;
    try {
      FileSystem fileSystem = FileSystem.getLocal(conf).getRaw();
      FileStatus[] errorFileStatuses = fileSystem
          .globStatus(new Path(containerLogDir, errorFileNamePattern));
      if (errorFileStatuses != null && errorFileStatuses.length != 0) {
        long tailSizeInBytes =
            conf.getLong(YarnConfiguration.NM_CONTAINER_STDERR_BYTES,
                YarnConfiguration.DEFAULT_NM_CONTAINER_STDERR_BYTES);
        Path errorFile = errorFileStatuses[0].getPath();
        long fileSize = errorFileStatuses[0].getLen();

        // if more than one file matches the stderr pattern, take the latest
        // modified file, and also append the file names in the diagnosticInfo
        if (errorFileStatuses.length > 1) {
          String[] errorFileNames = new String[errorFileStatuses.length];
          long latestModifiedTime = errorFileStatuses[0].getModificationTime();
          errorFileNames[0] = errorFileStatuses[0].getPath().getName();
          for (int i = 1; i < errorFileStatuses.length; i++) {
            errorFileNames[i] = errorFileStatuses[i].getPath().getName();
            if (errorFileStatuses[i]
                .getModificationTime() > latestModifiedTime) {
              latestModifiedTime = errorFileStatuses[i].getModificationTime();
              errorFile = errorFileStatuses[i].getPath();
              fileSize = errorFileStatuses[i].getLen();
            }
          }
          diagnosticInfo.append("Error files: ")
              .append(StringUtils.join(", ", errorFileNames)).append(".\n");
        }

        long startPosition =
            (fileSize < tailSizeInBytes) ? 0 : fileSize - tailSizeInBytes;
        int bufferSize =
            (int) ((fileSize < tailSizeInBytes) ? fileSize : tailSizeInBytes);
        byte[] tailBuffer = new byte[bufferSize];
        errorFileIS = fileSystem.open(errorFile);
        errorFileIS.readFully(startPosition, tailBuffer);

        diagnosticInfo.append("Last ").append(tailSizeInBytes)
            .append(" bytes of ").append(errorFile.getName()).append(" :\n")
            .append(new String(tailBuffer, StandardCharsets.UTF_8));
      }
    } catch (IOException e) {
      LOG.error("Failed to get tail of the container's error log file", e);
    } finally {
      IOUtils.cleanup(LOG, errorFileIS);
    }

    this.dispatcher.getEventHandler()
        .handle(new ContainerExitEvent(containerID,
            ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
            diagnosticInfo.toString()));
  }

  protected String getPidFileSubpath(String appIdStr, String containerIdStr) {
    return getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR
        + String.format(ContainerLaunch.PID_FILE_NAME_FMT, containerIdStr);
  }

  /**
   * Cleanup the container.
   * Cancels the launch if launch has not started yet or signals
   * the executor to not execute the process if not already done so.
   * Also, sends a SIGTERM followed by a SIGKILL to the process if
   * the process id is available.
   * @throws IOException
   */
  @SuppressWarnings("unchecked") // dispatcher not typed
  public void cleanupContainer() throws IOException {
    ContainerId containerId = container.getContainerId();
    String containerIdStr = ConverterUtils.toString(containerId);
    LOG.info("Cleaning up container " + containerIdStr);

    try {
      context.getNMStateStore().storeContainerKilled(containerId);
    } catch (IOException e) {
      LOG.error("Unable to mark container " + containerId
          + " killed in store", e);
    }

    // launch flag will be set to true if process already launched
    boolean alreadyLaunched = !shouldLaunchContainer.compareAndSet(false, true);
    if (!alreadyLaunched) {
      LOG.info("Container " + containerIdStr + " not launched."
          + " No cleanup needed to be done");
      return;
    }

    LOG.debug("Marking container " + containerIdStr + " as inactive");
    // this should ensure that if the container process has not launched
    // by this time, it will never be launched
    exec.deactivateContainer(containerId);

    if (LOG.isDebugEnabled()) {
      LOG.debug("Getting pid for container " + containerIdStr + " to kill"
          + " from pid file "
          + (pidFilePath != null ? pidFilePath.toString() : "null"));
    }

    // however the container process may have already started
    try {

      // get process id from pid file if available
      // else if shell is still active, get it from the shell
      String processId = null;
      if (pidFilePath != null) {
        processId = getContainerPid(pidFilePath);
      }

      // kill process
      if (processId != null) {
        String user = container.getUser();
        LOG.debug("Sending signal to pid " + processId
            + " as user " + user
            + " for container " + containerIdStr);

        final Signal signal = sleepDelayBeforeSigKill > 0
          ? Signal.TERM
          : Signal.KILL;

        boolean result = exec.signalContainer(
            new ContainerSignalContext.Builder()
                .setContainer(container)
                .setUser(user)
                .setPid(processId)
                .setSignal(signal)
                .build());

        LOG.debug("Sent signal " + signal + " to pid " + processId
          + " as user " + user
          + " for container " + containerIdStr
          + ", result=" + (result? "success" : "failed"));

        if (sleepDelayBeforeSigKill > 0) {
          new DelayedProcessKiller(container, user,
              processId, sleepDelayBeforeSigKill, Signal.KILL, exec).start();
        }
      }
    } catch (Exception e) {
      String message =
          "Exception when trying to cleanup container " + containerIdStr
              + ": " + StringUtils.stringifyException(e);
      LOG.warn(message);
      dispatcher.getEventHandler().handle(
        new ContainerDiagnosticsUpdateEvent(containerId, message));
    } finally {
      // cleanup pid file if present
      if (pidFilePath != null) {
        FileContext lfs = FileContext.getLocalFSFileContext();
        lfs.delete(pidFilePath, false);
        lfs.delete(pidFilePath.suffix(EXIT_CODE_FILE_SUFFIX), false);
      }
    }
  }

  /**
   * Send a signal to the container.
   *
   *
   * @throws IOException
   */
  @SuppressWarnings("unchecked") // dispatcher not typed
  public void signalContainer(SignalContainerCommand command)
      throws IOException {
    ContainerId containerId =
        container.getContainerTokenIdentifier().getContainerID();
    String containerIdStr = ConverterUtils.toString(containerId);
    String user = container.getUser();
    Signal signal = translateCommandToSignal(command);
    if (signal.equals(Signal.NULL)) {
      LOG.info("ignore signal command " + command);
      return;
    }

    LOG.info("Sending signal " + command + " to container " + containerIdStr);

    boolean alreadyLaunched = !shouldLaunchContainer.compareAndSet(false, true);
    if (!alreadyLaunched) {
      LOG.info("Container " + containerIdStr + " not launched."
          + " Not sending the signal");
      return;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Getting pid for container " + containerIdStr
          + " to send signal to from pid file "
          + (pidFilePath != null ? pidFilePath.toString() : "null"));
    }

    try {
      // get process id from pid file if available
      // else if shell is still active, get it from the shell
      String processId = null;
      if (pidFilePath != null) {
        processId = getContainerPid(pidFilePath);
      }

      if (processId != null) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Sending signal to pid " + processId
              + " as user " + user
              + " for container " + containerIdStr);
        }

        boolean result = exec.signalContainer(
            new ContainerSignalContext.Builder()
                .setContainer(container)
                .setUser(user)
                .setPid(processId)
                .setSignal(signal)
                .build());

        String diagnostics = "Sent signal " + command
            + " (" + signal + ") to pid " + processId
            + " as user " + user
            + " for container " + containerIdStr
            + ", result=" + (result ? "success" : "failed");
        LOG.info(diagnostics);

        dispatcher.getEventHandler().handle(
            new ContainerDiagnosticsUpdateEvent(containerId, diagnostics));
      }
    } catch (Exception e) {
      String message =
          "Exception when sending signal to container " + containerIdStr
              + ": " + StringUtils.stringifyException(e);
      LOG.warn(message);
    }
  }

  @VisibleForTesting
  public static Signal translateCommandToSignal(
      SignalContainerCommand command) {
    Signal signal = Signal.NULL;
    switch (command) {
      case OUTPUT_THREAD_DUMP:
        // TODO for windows support.
        signal = Shell.WINDOWS ? Signal.NULL: Signal.QUIT;
        break;
      case GRACEFUL_SHUTDOWN:
        signal = Signal.TERM;
        break;
      case FORCEFUL_SHUTDOWN:
        signal = Signal.KILL;
        break;
    }
    return signal;
  }

  /**
   * Loop through for a time-bounded interval waiting to
   * read the process id from a file generated by a running process.
   * @param pidFilePath File from which to read the process id
   * @return Process ID
   * @throws Exception
   */
  private String getContainerPid(Path pidFilePath) throws Exception {
    String containerIdStr =
        ConverterUtils.toString(container.getContainerId());
    String processId = null;
    LOG.debug("Accessing pid for container " + containerIdStr
        + " from pid file " + pidFilePath);
    int sleepCounter = 0;
    final int sleepInterval = 100;

    // loop waiting for pid file to show up
    // until our timer expires in which case we admit defeat
    while (true) {
      processId = ProcessIdFileReader.getProcessId(pidFilePath);
      if (processId != null) {
        LOG.debug("Got pid " + processId + " for container "
            + containerIdStr);
        break;
      }
      else if ((sleepCounter*sleepInterval) > maxKillWaitTime) {
        LOG.info("Could not get pid for " + containerIdStr
        		+ ". Waited for " + maxKillWaitTime + " ms.");
        break;
      }
      else {
        ++sleepCounter;
        Thread.sleep(sleepInterval);
      }
    }
    return processId;
  }

  public static String getRelativeContainerLogDir(String appIdStr,
      String containerIdStr) {
    return appIdStr + Path.SEPARATOR + containerIdStr;
  }

  private String getContainerPrivateDir(String appIdStr, String containerIdStr) {
    return getAppPrivateDir(appIdStr) + Path.SEPARATOR + containerIdStr
        + Path.SEPARATOR;
  }

  private String getAppPrivateDir(String appIdStr) {
    return ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
        + appIdStr;
  }

  Context getContext() {
    return context;
  }

  public static abstract class ShellScriptBuilder {
    public static ShellScriptBuilder create() {
      return Shell.WINDOWS ? new WindowsShellScriptBuilder() :
        new UnixShellScriptBuilder();
    }

    private static final String LINE_SEPARATOR =
        System.getProperty("line.separator");
    private final StringBuilder sb = new StringBuilder();

    public abstract void command(List<String> command) throws IOException;

    public abstract void whitelistedEnv(String key, String value) throws IOException;

    public abstract void env(String key, String value) throws IOException;

    public final void symlink(Path src, Path dst) throws IOException {
      if (!src.isAbsolute()) {
        throw new IOException("Source must be absolute");
      }
      if (dst.isAbsolute()) {
        throw new IOException("Destination must be relative");
      }
      if (dst.toUri().getPath().indexOf('/') != -1) {
        mkdir(dst.getParent());
      }
      link(src, dst);
    }

    /**
     * Method to copy files that are useful for debugging container failures.
     * This method will be called by ContainerExecutor when setting up the
     * container launch script. The method should take care to make sure files
     * are read-able by the yarn user if the files are to undergo
     * log-aggregation.
     * @param src path to the source file
     * @param dst path to the destination file - should be absolute
     * @throws IOException
     */
    public abstract void copyDebugInformation(Path src, Path dst)
        throws IOException;

    /**
     * Method to dump debug information to the a target file. This method will
     * be called by ContainerExecutor when setting up the container launch
     * script.
     * @param output the file to which debug information is to be written
     * @throws IOException
     */
    public abstract void listDebugInformation(Path output) throws IOException;

    @Override
    public String toString() {
      return sb.toString();
    }

    public final void write(PrintStream out) throws IOException {
      out.append(sb);
    }

    protected final void line(String... command) {
      for (String s : command) {
        sb.append(s);
      }
      sb.append(LINE_SEPARATOR);
    }

    protected abstract void link(Path src, Path dst) throws IOException;

    protected abstract void mkdir(Path path) throws IOException;
  }

  private static final class UnixShellScriptBuilder extends ShellScriptBuilder {

    private void errorCheck() {
      line("hadoop_shell_errorcode=$?");
      line("if [ $hadoop_shell_errorcode -ne 0 ]");
      line("then");
      line("  exit $hadoop_shell_errorcode");
      line("fi");
    }

    public UnixShellScriptBuilder(){
      line("#!/bin/bash");
      line();
    }

    @Override
    public void command(List<String> command) {
      line("exec /bin/bash -c \"", StringUtils.join(" ", command), "\"");
      errorCheck();
    }

    @Override
    public void whitelistedEnv(String key, String value) {
      line("export ", key, "=${", key, ":-", "\"", value, "\"}");
    }

    @Override
    public void env(String key, String value) {
      line("export ", key, "=\"", value, "\"");
    }

    @Override
    protected void link(Path src, Path dst) throws IOException {
      line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\"");
      errorCheck();
    }

    @Override
    protected void mkdir(Path path) {
      line("mkdir -p ", path.toString());
      errorCheck();
    }

    @Override
    public void copyDebugInformation(Path src, Path dest) throws IOException {
      line("# Creating copy of launch script");
      line("cp \"", src.toUri().getPath(), "\" \"", dest.toUri().getPath(),
          "\"");
      // set permissions to 640 because we need to be able to run
      // log aggregation in secure mode as well
      if(dest.isAbsolute()) {
        line("chmod 640 \"", dest.toUri().getPath(), "\"");
      }
    }

    @Override
    public void listDebugInformation(Path output) throws  IOException {
      line("# Determining directory contents");
      line("echo \"ls -l:\" 1>\"", output.toString(), "\"");
      line("ls -l 1>>\"", output.toString(), "\"");

      // don't run error check because if there are loops
      // find will exit with an error causing container launch to fail
      // find will follow symlinks outside the work dir if such sylimks exist
      // (like public/app local resources)
      line("echo \"find -L . -maxdepth 5 -ls:\" 1>>\"", output.toString(),
          "\"");
      line("find -L . -maxdepth 5 -ls 1>>\"", output.toString(), "\"");
      line("echo \"broken symlinks(find -L . -maxdepth 5 -type l -ls):\" 1>>\"",
          output.toString(), "\"");
      line("find -L . -maxdepth 5 -type l -ls 1>>\"", output.toString(), "\"");
    }
  }

  private static final class WindowsShellScriptBuilder
      extends ShellScriptBuilder {

    private void errorCheck() {
      line("@if %errorlevel% neq 0 exit /b %errorlevel%");
    }

    private void lineWithLenCheck(String... commands) throws IOException {
      Shell.checkWindowsCommandLineLength(commands);
      line(commands);
    }

    public WindowsShellScriptBuilder() {
      line("@setlocal");
      line();
    }

    @Override
    public void command(List<String> command) throws IOException {
      lineWithLenCheck("@call ", StringUtils.join(" ", command));
      errorCheck();
    }

    @Override
    public void whitelistedEnv(String key, String value) throws IOException {
      lineWithLenCheck("@set ", key, "=", value);
      errorCheck();
    }

    @Override
    public void env(String key, String value) throws IOException {
      lineWithLenCheck("@set ", key, "=", value);
      errorCheck();
    }

    @Override
    protected void link(Path src, Path dst) throws IOException {
      File srcFile = new File(src.toUri().getPath());
      String srcFileStr = srcFile.getPath();
      String dstFileStr = new File(dst.toString()).getPath();
      lineWithLenCheck(String.format("@%s symlink \"%s\" \"%s\"",
          Shell.getWinUtilsPath(), dstFileStr, srcFileStr));
      errorCheck();
    }

    @Override
    protected void mkdir(Path path) throws IOException {
      lineWithLenCheck(String.format("@if not exist \"%s\" mkdir \"%s\"",
          path.toString(), path.toString()));
      errorCheck();
    }

    @Override
    public void copyDebugInformation(Path src, Path dest)
        throws IOException {
      // no need to worry about permissions - in secure mode
      // WindowsSecureContainerExecutor will set permissions
      // to allow NM to read the file
      line("rem Creating copy of launch script");
      lineWithLenCheck(String.format("copy \"%s\" \"%s\"", src.toString(),
          dest.toString()));
    }

    @Override
    public void listDebugInformation(Path output) throws IOException {
      line("rem Determining directory contents");
      lineWithLenCheck(
          String.format("@echo \"dir:\" > \"%s\"", output.toString()));
      lineWithLenCheck(String.format("dir >> \"%s\"", output.toString()));
    }
  }

  private static void putEnvIfNotNull(
      Map<String, String> environment, String variable, String value) {
    if (value != null) {
      environment.put(variable, value);
    }
  }

  private static void putEnvIfAbsent(
      Map<String, String> environment, String variable) {
    if (environment.get(variable) == null) {
      putEnvIfNotNull(environment, variable, System.getenv(variable));
    }
  }

  public void sanitizeEnv(Map<String, String> environment, Path pwd,
      List<Path> appDirs, List<String> containerLogDirs,
      Map<Path, List<String>> resources,
      Path nmPrivateClasspathJarDir) throws IOException {
    /**
     * Non-modifiable environment variables
     */

    environment.put(Environment.CONTAINER_ID.name(), container
        .getContainerId().toString());

    environment.put(Environment.NM_PORT.name(),
      String.valueOf(this.context.getNodeId().getPort()));

    environment.put(Environment.NM_HOST.name(), this.context.getNodeId()
      .getHost());

    environment.put(Environment.NM_HTTP_PORT.name(),
      String.valueOf(this.context.getHttpPort()));

    environment.put(Environment.LOCAL_DIRS.name(),
        StringUtils.join(",", appDirs));

    environment.put(Environment.LOG_DIRS.name(),
      StringUtils.join(",", containerLogDirs));

    environment.put(Environment.USER.name(), container.getUser());

    environment.put(Environment.LOGNAME.name(), container.getUser());

    environment.put(Environment.HOME.name(),
        conf.get(
            YarnConfiguration.NM_USER_HOME_DIR,
            YarnConfiguration.DEFAULT_NM_USER_HOME_DIR
            )
        );

    environment.put(Environment.PWD.name(), pwd.toString());

    putEnvIfNotNull(environment,
        Environment.HADOOP_CONF_DIR.name(),
        System.getenv(Environment.HADOOP_CONF_DIR.name())
        );

    if (!Shell.WINDOWS) {
      environment.put("JVM_PID", "$$");
    }

    /**
     * Modifiable environment variables
     */

    // allow containers to override these variables
    String[] whitelist = conf.get(YarnConfiguration.NM_ENV_WHITELIST, YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");

    for(String whitelistEnvVariable : whitelist) {
      putEnvIfAbsent(environment, whitelistEnvVariable.trim());
    }

    // variables here will be forced in, even if the container has specified them.
    Apps.setEnvFromInputString(environment, conf.get(
      YarnConfiguration.NM_ADMIN_USER_ENV,
      YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV), File.pathSeparator);

    // TODO: Remove Windows check and use this approach on all platforms after
    // additional testing.  See YARN-358.
    if (Shell.WINDOWS) {

      String inputClassPath = environment.get(Environment.CLASSPATH.name());

      if (inputClassPath != null && !inputClassPath.isEmpty()) {

        //On non-windows, localized resources
        //from distcache are available via the classpath as they were placed
        //there but on windows they are not available when the classpath
        //jar is created and so they "are lost" and have to be explicitly
        //added to the classpath instead.  This also means that their position
        //is lost relative to other non-distcache classpath entries which will
        //break things like mapreduce.job.user.classpath.first.  An environment
        //variable can be set to indicate that distcache entries should come
        //first

        boolean preferLocalizedJars = Boolean.parseBoolean(
          environment.get(Environment.CLASSPATH_PREPEND_DISTCACHE.name())
          );

        boolean needsSeparator = false;
        StringBuilder newClassPath = new StringBuilder();
        if (!preferLocalizedJars) {
          newClassPath.append(inputClassPath);
          needsSeparator = true;
        }

        // Localized resources do not exist at the desired paths yet, because the
        // container launch script has not run to create symlinks yet.  This
        // means that FileUtil.createJarWithClassPath can't automatically expand
        // wildcards to separate classpath entries for each file in the manifest.
        // To resolve this, append classpath entries explicitly for each
        // resource.
        for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
          boolean targetIsDirectory = new File(entry.getKey().toUri().getPath())
            .isDirectory();

          for (String linkName : entry.getValue()) {
            // Append resource.
            if (needsSeparator) {
              newClassPath.append(File.pathSeparator);
            } else {
              needsSeparator = true;
            }
            newClassPath.append(pwd.toString())
              .append(Path.SEPARATOR).append(linkName);

            // FileUtil.createJarWithClassPath must use File.toURI to convert
            // each file to a URI to write into the manifest's classpath.  For
            // directories, the classpath must have a trailing '/', but
            // File.toURI only appends the trailing '/' if it is a directory that
            // already exists.  To resolve this, add the classpath entries with
            // explicit trailing '/' here for any localized resource that targets
            // a directory.  Then, FileUtil.createJarWithClassPath will guarantee
            // that the resulting entry in the manifest's classpath will have a
            // trailing '/', and thus refer to a directory instead of a file.
            if (targetIsDirectory) {
              newClassPath.append(Path.SEPARATOR);
            }
          }
        }
        if (preferLocalizedJars) {
          if (needsSeparator) {
            newClassPath.append(File.pathSeparator);
          }
          newClassPath.append(inputClassPath);
        }

        // When the container launches, it takes the parent process's environment
        // and then adds/overwrites with the entries from the container launch
        // context.  Do the same thing here for correct substitution of
        // environment variables in the classpath jar manifest.
        Map<String, String> mergedEnv = new HashMap<String, String>(
          System.getenv());
        mergedEnv.putAll(environment);

        // this is hacky and temporary - it's to preserve the windows secure
        // behavior but enable non-secure windows to properly build the class
        // path for access to job.jar/lib/xyz and friends (see YARN-2803)
        Path jarDir;
        if (exec instanceof WindowsSecureContainerExecutor) {
          jarDir = nmPrivateClasspathJarDir;
        } else {
          jarDir = pwd;
        }
        String[] jarCp = FileUtil.createJarWithClassPath(
          newClassPath.toString(), jarDir, pwd, mergedEnv);
        // In a secure cluster the classpath jar must be localized to grant access
        Path localizedClassPathJar = exec.localizeClasspathJar(
            new Path(jarCp[0]), pwd, container.getUser());
        String replacementClassPath = localizedClassPathJar.toString() + jarCp[1];
        environment.put(Environment.CLASSPATH.name(), replacementClassPath);
      }
    }
    // put AuxiliaryService data to environment
    for (Map.Entry<String, ByteBuffer> meta : containerManager
        .getAuxServiceMetaData().entrySet()) {
      AuxiliaryServiceHelper.setServiceDataIntoEnv(
          meta.getKey(), meta.getValue(), environment);
    }
  }

  public static String getExitCodeFile(String pidFile) {
    return pidFile + EXIT_CODE_FILE_SUFFIX;
  }
}


text:
from django.conf import settings
from django.conf.urls import include, url
from django.conf.urls.static import static
from django.contrib import admin
from django.core.urlresolvers import reverse_lazy
from django.views.generic import RedirectView

from block.models import Page
from block.views import (
    PageDesignView,
    PageTemplateView,
)

from .views import (
    ExampleView,
    SettingsView,
    TitleCreateView,
    TitlePublishView,
    TitleRemoveView,
    TitleUpdateView,
)

admin.autodiscover()

urlpatterns = [
    # '/' send to home
    url(regex=r'^$',
        view=PageTemplateView.as_view(),
        kwargs=dict(page=Page.HOME),
        name='project.home'
        ),
    # admin, login
    url(regex=r'^admin/',
        view=include(admin.site.urls)
        ),
    url(regex=r'^block/',
        view=include('block.urls.block')
        ),
    url(regex=r'^settings/$',
        view=SettingsView.as_view(),
        name='project.settings'
        ),
    url(regex=r'^wizard/',
        view=include('block.urls.wizard')
        ),
    url(regex=r'^',
        view=include('login.urls')
        ),
    # home page when logged in
    url(r'^home/user/$',
        view=RedirectView.as_view(url=reverse_lazy('project.home'), permanent=False),
        name='project.dash'
        ),
    # custom page - see https://www.pkimber.net/open/app-block.html
    url(regex=r'^calendar/information/$',
        view=ExampleView.as_view(),
        kwargs=dict(page=Page.CUSTOM, menu='calendar-information'),
        name='calendar.information'
        ),
    # block page design
    url(regex=r'^(?P<page>[-\w\d]+)/design/$',
        view=PageDesignView.as_view(),
        name='project.page.design'
        ),
    url(regex=r'^(?P<page>[-\w\d]+)/(?P<menu>[-\w\d]+)/design/$',
        view=PageDesignView.as_view(),
        name='project.page.design'
        ),
    # block page view
    url(regex=r'^(?P<page>[-\w\d]+)/$',
        view=PageTemplateView.as_view(),
        name='project.page'
        ),
    url(regex=r'^(?P<page>[-\w\d]+)/(?P<menu>[-\w\d]+)/$',
        view=PageTemplateView.as_view(),
        name='project.page'
        ),
    # title create, publish, update and remove
    url(regex=r'^title/create/(?P<page>[-\w\d]+)/(?P<section>[-\w\d]+)/$',
        view=TitleCreateView.as_view(),
        name='example.title.create'
        ),
    url(regex=r'^title/create/(?P<page>[-\w\d]+)/(?P<menu>[-\w\d]+)/(?P<section>[-\w\d]+)/$',
        view=TitleCreateView.as_view(),
        name='example.title.create'
        ),
    url(regex=r'^title/(?P<pk>\d+)/publish/$',
        view=TitlePublishView.as_view(),
        name='example.title.publish'
        ),
    url(regex=r'^title/(?P<pk>\d+)/update/$',
        view=TitleUpdateView.as_view(),
        name='example.title.update'
        ),
    url(regex=r'^title/(?P<pk>\d+)/remove/$',
        view=TitleRemoveView.as_view(),
        name='example.title.remove'
        ),
]

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
#   ^ helper function to return a URL pattern for serving files in debug mode.
# https://docs.djangoproject.com/en/1.5/howto/static-files/#serving-files-uploaded-by-a-user

if settings.ALLOW_DEBUG_TOOLBAR:
    import debug_toolbar
    urlpatterns += [url(r'^__debug__/', include(debug_toolbar.urls)), ]

redpajama_set_name: RedPajamaGithub


text:
ACCEPTED

#### According to
Index Fungorum

#### Published in
null

#### Original name
Colletotrichum lujae Verpl. & Clem.

### Remarks
null


text:
<!DOCTYPE html>

<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport"
	content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no">

<title>JULS - Sign Up</title>

<link rel="stylesheet" href="/static/css/reg.css">
<link rel="stylesheet" href="/static/css/bootstrap.css">

Fill in the registration form to proceed

		<form id="RegForm" method="post" action="../../new/user" enctype="x-www-form-urlencoded">
			<input class="form-control" id="emailReg" name= "email" type="email" placeholder="Email">

			<input class="form-control" id="PassReg1" name= "password1" type="password" placeholder="Password" required>
			<input class="form-control" id="PassReg2" name= "password2" type="password" placeholder="Repeat password" required>

			<input class="btn btn-lg btn-primary btn-block" type="submit" value="Sign Up">
		</form>


text:
require=$(foreach r,$(1),(require (quote $(r))))
EMACSFLAGS=-batch --no-site-file --eval '(setq debug-on-error t)' -l "../../../../cedet-remove-builtin.el"
EMACS=@echo "    > $@";emacs
AUTOGENFLAGS=-L ../../ --eval '(progn (require (quote cedet-compat)) (require (quote mode-local)))'

top=../../../
ede_FILES=Project.ede Makefile

symref_LISP=global.el grep.el list.el idutils.el cscope.el filter.el
LOADPATH= ../../../eieio/ ../../ ../../
VERSION=1.1beta
DISTDIR=$(top)cedet-topdir-$(VERSION)/cedet/semantic/symref

all: symref

%.elc: %.el
	$(EMACS) $(EMACSFLAGS) $(addprefix -L ,$(LOADPATH)) --eval '(progn $(call require, $(PRELOADS)))' -f batch-byte-compile $^

.PHONY: symref
symref: $(addsuffix c, $(symref_LISP))
symref: PRELOADS=cedet-compat

tags:

clean:
	rm -f *.elc

.PHONY: dist

dist:
	mkdir $(DISTDIR)
	cp $(symref_LISP) $(ede_FILES) $(DISTDIR)

Makefile: Project.ede
	@echo Makefile is out of date!  It needs to be regenerated by EDE.
	@echo If you have not modified Project.ede, you can use 'touch' to update the Makefile time stamp.
	@false

# End of Makefile


text:
		{{options.label}}
		{{options.required ? '*' : ''}}

		Campo obligatorio. Introduzca un valor


text: module.exports = require('./lib/socket.io');


text:
require 'test_helper'
require 'docsplit'
require 'seek/download_handling/http_streamer' # Needed to load exceptions that are tested later
require 'minitest/mock'
require 'time_test_helper'
require 'private_address_check'

class ContentBlobTest < ActiveSupport::TestCase

  include NelsTestHelper

  fixtures :content_blobs

  test 'search terms' do
    blob = ContentBlob.new
    blob.url = 'http://fish.com'
    assert_includes blob.search_terms, 'fish'
    assert_includes blob.search_terms, 'http://fish.com'
    refute_includes blob.search_terms, 'http'
    refute_includes blob.search_terms, 'com'

    blob = Factory(:txt_content_blob)
    assert_includes blob.search_terms, 'txt_test.txt'
    assert_includes blob.search_terms, 'txt'
  end

  test 'max indexable text size' do
    blob = Factory :large_txt_content_blob
    size = blob.file_size
    assert size > 1.megabyte
    assert size < 2.megabyte

    assert blob.is_text?

    with_config_value :max_indexable_text_size, 1 do
      refute blob.is_indexable_text?
    end

    with_config_value :max_indexable_text_size, 2 do
      assert blob.is_indexable_text?
    end

    with_config_value :max_indexable_text_size, '2' do
      assert blob.is_indexable_text?
    end
  end

  test 'md5sum_on_demand' do
    blob = Factory :rightfield_content_blob
    assert_not_nil blob.md5sum
    assert_equal '01788bca93265d80e8127ca0039bb69b', blob.md5sum
  end

  test 'sha1 sum on demand' do
    blob = Factory :rightfield_content_blob
    assert_not_nil blob.sha1sum
    assert_equal 'ffd634ac7564083ab7b66bc3eb2053cbc3d608f5', blob.sha1sum
  end

  test 'detects it is a webpage' do
    as_not_virtualliver do
      mock_remote_file "#{Rails.root}/test/fixtures/files/html_file.html", 'http://webpage.com', 'Content-Type' => 'text/html'
      blob = ContentBlob.create url: 'http://webpage.com', original_filename: nil, content_type: nil, external_link: true
      assert blob.is_webpage?
      assert_equal 'text/html', blob.content_type
    end
  end

  test 'detectes webpage if content-type includes charset info' do
    as_not_virtualliver do
      mock_remote_file "#{Rails.root}/test/fixtures/files/html_file.html", 'http://webpage.com', 'Content-Type' => 'text/html; charset=ascii'
      blob = ContentBlob.create url: 'http://webpage.com', original_filename: nil, content_type: nil, external_link: true
      assert blob.is_webpage?
      assert_equal 'text/html', blob.content_type
    end
  end

  test 'only overrides url content-type if not already known' do
    as_not_virtualliver do
      mock_remote_file "#{Rails.root}/test/fixtures/files/html_file.html", 'http://webpage.com', 'Content-Type' => 'text/html'
      mock_remote_file "#{Rails.root}/test/fixtures/files/file_picture.png", 'http://webpage.com/piccy.png', 'Content-Type' => 'image/png'

      blob = ContentBlob.create url: 'http://webpage.com', original_filename: nil, content_type: nil, external_link: true
      assert_equal 'text/html', blob.content_type

      blob = ContentBlob.create url: 'http://webpage.com/piccy.png', original_filename: nil, content_type: nil
      assert_equal 'image/png', blob.content_type

      blob = ContentBlob.create url: 'http://webpage.com/piccy.png', original_filename: nil, content_type: 'application/pdf'
      assert_equal 'application/pdf', blob.content_type
    end
  end

  test "detects it isn't a webpage" do
    as_not_virtualliver do
      mock_remote_file "#{Rails.root}/test/fixtures/files/file_picture.png", 'http://webpage.com/piccy.png', 'Content-Type' => 'image/png'
      blob = ContentBlob.create url: 'http://webpage.com/piccy.png', original_filename: nil, content_type: nil
      assert !blob.is_webpage?
      assert_equal 'image/png', blob.content_type
    end
  end

  test 'handles an unavailable url when checking for a webpage' do
    mock_remote_file "#{Rails.root}/test/fixtures/files/file_picture.png", 'http://webpage.com/piccy.png', { 'Content-Type' => 'image/png' }, 500
    blob = ContentBlob.create url: 'http://webpage.com/piccy.png', original_filename: nil, content_type: nil
    assert !blob.is_webpage?
  end

  test "content type for url type with binary doesn't try read file" do
    mock_remote_file "#{Rails.root}/test/fixtures/files/file_picture.png", 'http://webpage.com/binary-file', { 'Content-Type' => 'application/octet-stream' }, 500
    blob = ContentBlob.create url: 'http://webpage.com/binary-file', original_filename: nil, content_type: 'application/octet-stream'
    assert_equal 'application/octet-stream', blob.content_type
  end

  def test_cache_key
    blob = Factory :rightfield_content_blob
    assert_equal "content_blobs/#{blob.id}-ffd634ac7564083ab7b66bc3eb2053cbc3d608f5", blob.cache_key
  end

  def test_uuid_doesnt_change
    blob = content_blobs(:picture_blob)
    blob.uuid = 'zzz'
    assert_equal 'zzz', blob.uuid
    blob.save!
    assert_equal 'zzz', blob.uuid
  end

  def test_uuid_doesnt_change2
    blob = content_blobs(:picture_blob)
    blob.uuid = 'zzz'
    blob.save!
    blob = ContentBlob.find(blob.id)
    assert_equal 'zzz', blob.uuid
    blob.save!
    blob = ContentBlob.find(blob.id)
    assert_equal 'zzz', blob.uuid
  end

  def test_regenerate_uuid
    pic = content_blobs(:picture_blob)
    uuid = pic.uuid
    pic.regenerate_uuid
    assert_not_equal uuid, pic.uuid
  end

  def test_file_dump
    blob = ContentBlob.new(data: data_for_test('file_picture.png'), original_filename: 'piccy.jpg')
    blob.save!
    assert_not_nil blob.filepath
    data = nil
    File.open(blob.filepath, 'rb') do |f|
      data = f.read
    end
    assert_not_nil data
    assert_equal data_for_test('file_picture.png'), data
  end

  # checks that the data is assigned through the new method, stored to a file, and not written to the old data_old field
  def test_data_assignment
    blob = ContentBlob.new(data: data_for_test('file_picture.png'), original_filename: 'piccy.jpg')
    blob.save!
    blob = ContentBlob.find(blob.id)
    assert_equal data_for_test('file_picture.png'), blob.data_io_object.read

    assert_not_nil blob.filepath
    data = nil
    File.open(blob.filepath, 'rb') do |f|
      data = f.read
    end
    assert_not_nil data
    assert_equal data_for_test('file_picture.png'), data
  end

  # simply checks that get and set data returns the same thing
  def test_data_assignment2
    pic = Factory(:content_blob, data: data_for_test('file_picture.png'))
    pic.data = data_for_test('little_file.txt')
    pic.save!
    assert_equal data_for_test('little_file.txt'), pic.data_io_object.read

    # put it back, otherwise other tests fail
    pic.data = data_for_test('file_picture.png')
    pic.save!
  end

  #
  def test_will_overwrite_if_data_changes
    pic = Factory(:content_blob, data: data_for_test('file_picture.png'))
    pic.save!
    assert_equal data_for_test('file_picture.png'), File.open(pic.filepath, 'rb').read
    pic.data = data_for_test('little_file.txt')
    pic.save!
    assert_equal data_for_test('little_file.txt'), File.open(pic.filepath, 'rb').read
  end

  def test_uuid
    blob = Factory(:content_blob)
    blob.save!
    assert_not_nil blob.uuid
    assert_equal blob.uuid, ContentBlob.find(blob.id).uuid
  end

  def data_for_test(filename)
    file = "#{Rails.root}/test/fixtures/files/#{filename}"
    File.open(file, 'rb').read
  end

  def test_tmp_io_object
    io_object = Tempfile.new('tmp_io_object_test')
    io_object.write("blah blah\nmonkey_business")

    blob = ContentBlob.new(tmp_io_object: io_object, original_filename: 'monkey.txt')
    assert_difference('ContentBlob.count') do
      blob.save!
    end

    blob.reload
    assert_not_nil blob.filepath
    assert File.exist?(blob.filepath)
    data = nil
    File.open(blob.filepath, 'rb') do |f|
      data = f.read
    end

    assert_not_nil data
    assert_equal "blah blah\nmonkey_business", data.to_s
  end

  def test_string_io_object
    io_object = StringIO.new('frog')
    blob = ContentBlob.new(tmp_io_object: io_object, original_filename: 'frog.txt')
    assert_difference('ContentBlob.count') do
      blob.save!
    end

    blob.reload
    assert_not_nil blob.filepath
    assert File.exist?(blob.filepath)
    data = nil
    File.open(blob.filepath, 'rb') do |f|
      data = f.read
    end

    assert_not_nil data
    assert_equal 'frog', data.to_s
  end

  test 'validates by content blob or url' do
    blob = ContentBlob.new
    refute blob.valid?
    blob.original_filename = 'fish'
    assert blob.valid?
    blob.original_filename = nil
    blob.url = 'http://google.com'
    assert blob.valid?
  end

  def test_data_io
    io_object = StringIO.new('frog')
    blob = ContentBlob.new(tmp_io_object: io_object, original_filename: 'frog.txt')
    blob.save!
    blob.reload
    assert_equal 'frog', blob.data_io_object.read

    f = Tempfile.new('seek-data-io-test')
    f << 'fish'
    f.close

    io_object = File.new(f.path, 'r')
    blob = ContentBlob.new(tmp_io_object: io_object, original_filename: 'seek-data-io-test')
    blob.save!
    blob.reload
    io_object.rewind
    assert_equal io_object.read, blob.data_io_object.read

    mock_remote_file "#{Rails.root}/test/fixtures/files/file_picture.png", 'http://www.webpage.com/image.png'
    blob = ContentBlob.new(url: 'http://www.webpage.com/image.png')
    blob.save!
    blob.reload
    assert_nil blob.data_io_object

    blob = ContentBlob.new original_filename: 'nil'
    assert_nil blob.data_io_object
    blob.save!
    assert_nil blob.data_io_object
  end

  def test_filesize
    cb = Factory :content_blob, data: 'z'
    assert_equal 1, cb.file_size
    File.delete(cb.filepath)
    assert_equal 1, cb.file_size
    cb = Factory :rightfield_content_blob
    assert_not_nil cb.file_size
    assert_equal 9216, cb.file_size
  end

  def test_exception_when_both_data_and_io_object
    io_object = StringIO.new('frog')
    blob = ContentBlob.new(tmp_io_object: io_object, data: 'snake', original_filename: 'snake.txt')
    assert_raise Exception do
      blob.save
    end
  end

  test 'storage_directory and filepath' do
    content_blob = Factory(:content_blob)
    storage_directory = content_blob.data_storage_directory
    converted_storage_directory = content_blob.converted_storage_directory
    assert_equal "#{Rails.root}/tmp/testing-filestore/assets", storage_directory
    assert_equal "#{Rails.root}/tmp/testing-filestore/converted-assets", converted_storage_directory
    assert_equal (storage_directory + '/' + content_blob.uuid + '.dat'), content_blob.filepath
    assert_equal (converted_storage_directory + '/' + content_blob.uuid + '.pdf'), content_blob.filepath('pdf')
    assert_equal (converted_storage_directory + '/' + content_blob.uuid + '.txt'), content_blob.filepath('txt')
  end

  test 'file_exists?' do
    # specify uuid here to avoid repeating uuid of other content_blob when running the whole test file
    content_blob = Factory(:content_blob, uuid: '1111')
    assert content_blob.file_exists?
    content_blob = Factory(:content_blob, uuid: '2222', data: nil)
    assert !content_blob.file_exists?
  end

  test 'human content type' do
    content_blob = Factory(:docx_content_blob)
    assert_equal 'Word document', content_blob.human_content_type

    content_blob = Factory(:content_blob, content_type: 'application/msexcel')
    assert_equal 'Spreadsheet', content_blob.human_content_type

    content_blob = Factory(:xlsm_content_blob)
    assert_equal 'Spreadsheet (macro enabled)', content_blob.human_content_type

    content_blob = Factory.create(:pdf_content_blob, content_type: 'application/pdf')
    assert_equal 'PDF document', content_blob.human_content_type

    content_blob = Factory(:content_blob, content_type: 'text/html')
    assert_equal 'HTML document', content_blob.human_content_type

    content_blob = Factory(:content_blob, content_type: 'application/x-download')
    assert_equal 'Unknown file type', content_blob.human_content_type

    content_blob = Factory(:content_blob, content_type: '')
    assert_equal 'Unknown file type', content_blob.human_content_type

    content_blob = Factory(:content_blob)
    assert_equal 'Unknown file type', content_blob.human_content_type

    content_blob = Factory(:tiff_content_blob)
    assert_equal 'TIFF image', content_blob.human_content_type
  end

  test 'override mimetype presented, with detected type' do
    blob = Factory.build(:csv_content_blob, content_type:'application/vnd.excel')
    assert_equal 'application/vnd.excel',blob.content_type
    blob.save!
    assert_equal 'text/csv',blob.content_type

    blob = Factory.build(:xlsm_content_blob,content_type:'text/csv')
    assert_equal 'text/csv',blob.content_type
    blob.save!
    assert_equal 'application/vnd.ms-excel.sheet.macroEnabled.12',blob.content_type
  end

  test 'mimemagic updates content type on creation if binary' do
    tmp_blob = Factory(:pdf_content_blob) # to get the filepath
    content_blob = ContentBlob.create data: File.new(tmp_blob.filepath, 'rb').read, original_filename: 'blob', content_type: 'application/octet-stream'
    assert_equal 'application/pdf', content_blob.content_type
    assert content_blob.is_pdf?
  end

  test 'is_binary?' do
    content_blob = Factory(:pdf_content_blob)
    refute content_blob.is_binary?
    content_blob = Factory(:binary_content_blob)
    assert content_blob.is_binary?
  end

  test 'covert_office should doc to pdf and then docslit convert pdf to txt' do
    check_for_soffice
    content_blob = Factory(:doc_content_blob, uuid: 'doc_1')
    assert File.exist? content_blob.filepath
    pdf_path = content_blob.filepath('pdf')
    FileUtils.rm pdf_path if File.exist? pdf_path
    assert !File.exist?(pdf_path)

    content_blob.convert_to_pdf

    assert File.exist?(pdf_path), 'pdf was not created during conversion'

    content_blob.extract_text_from_pdf

    assert File.exist? content_blob.filepath('txt')

    content = File.open(content_blob.filepath('txt'), 'rb').read
    assert content.include?('This is a ms word doc format')
  end

  test 'convert_office should convert docx to pdf and then docsplit convert pdf to txt' do
    check_for_soffice
    content_blob = Factory(:docx_content_blob, uuid: 'docx_1')
    assert File.exist? content_blob.filepath
    FileUtils.rm content_blob.filepath('pdf') if File.exist? content_blob.filepath('pdf')
    assert !File.exist?(content_blob.filepath('pdf'))

    content_blob.convert_to_pdf

    assert File.exist?(content_blob.filepath('pdf')), 'pdf was not created during conversion'

    content_blob.extract_text_from_pdf

    assert File.exist? content_blob.filepath('txt')

    content = File.open(content_blob.filepath('txt'), 'rb').read
    assert content.include?('This is a ms word docx format')
  end

  test 'convert_office should convert odt to pdf and then docsplit converts pdf to txt' do
    check_for_soffice
    content_blob = Factory(:odt_content_blob, uuid: 'odt_1')
    assert File.exist? content_blob.filepath
    FileUtils.rm content_blob.filepath('pdf') if File.exist? content_blob.filepath('pdf')
    assert !File.exist?(content_blob.filepath('pdf'))

    content_blob.convert_to_pdf

    assert File.exist?(content_blob.filepath('pdf')), 'pdf was not created during conversion'

    content_blob.extract_text_from_pdf

    assert File.exist? content_blob.filepath('txt')

    content = File.open(content_blob.filepath('txt'), 'rb').read
    assert content.include?('This is an open office word odt format')
  end

  test 'convert_office should convert ppt to pdf and then docsplit converts pdf to txt' do
    check_for_soffice
    content_blob = Factory(:ppt_content_blob, uuid: 'ppt_1')
    assert File.exist? content_blob.filepath
    FileUtils.rm content_blob.filepath('pdf') if File.exist? content_blob.filepath('pdf')
    assert !File.exist?(content_blob.filepath('pdf'))

    content_blob.convert_to_pdf

    assert File.exist?(content_blob.filepath('pdf')), 'pdf was not created during conversion'

    content_blob.extract_text_from_pdf

    assert File.exist? content_blob.filepath('txt')

    content = File.open(content_blob.filepath('txt'), 'rb').read
    assert content.include?('This is a ms power point ppt format')
  end

  test 'convert_office should convert pptx to pdf and then docsplit converts pdf to txt' do
    check_for_soffice
    content_blob = Factory(:pptx_content_blob, uuid: 'pptx_1')
    assert File.exist? content_blob.filepath
    FileUtils.rm content_blob.filepath('pdf') if File.exist? content_blob.filepath('pdf')
    assert !File.exist?(content_blob.filepath('pdf'))

    content_blob.convert_to_pdf

    assert File.exist?(content_blob.filepath('pdf')), 'pdf was not created during conversion'

    content_blob.extract_text_from_pdf

    assert File.exist? content_blob.filepath('txt')

    content = File.open(content_blob.filepath('txt'), 'rb').read
    assert content.include?('This is a ms power point pptx format')
  end

  test 'convert_office should convert odp to pdf and then docsplit converts pdf to txt' do
    check_for_soffice
    content_blob = Factory(:odp_content_blob, uuid: 'odp_1')
    assert File.exist? content_blob.filepath
    FileUtils.rm content_blob.filepath('pdf') if File.exist? content_blob.filepath('pdf')
    assert !File.exist?(content_blob.filepath('pdf'))

    content_blob.convert_to_pdf

    assert File.exist?(content_blob.filepath('pdf')), 'pdf was not created during conversion'

    content_blob.extract_text_from_pdf

    assert File.exist? content_blob.filepath('txt')

    content = File.open(content_blob.filepath('txt'), 'rb').read
    assert content.include?('This is an open office power point odp format')
  end

  test 'convert_office should convert rtf to pdf and then docsplit converts pdf to txt' do
    check_for_soffice
    content_blob = Factory(:rtf_content_blob, uuid: 'rtf_1')
    assert File.exist? content_blob.filepath
    FileUtils.rm content_blob.filepath('pdf') if File.exist? content_blob.filepath('pdf')
    assert !File.exist?(content_blob.filepath('pdf'))

    content_blob.convert_to_pdf

    assert File.exist?(content_blob.filepath('pdf')), 'pdf was not created during conversion'

    content_blob.extract_text_from_pdf

    assert File.exist? content_blob.filepath('txt')

    content = File.open(content_blob.filepath('txt'), 'rb').read

    assert content.mb_chars.normalize.include?('This is a rtf format')
  end

  test 'convert_office should convert txt to pdf' do
    check_for_soffice
    content_blob = Factory(:txt_content_blob, uuid: 'txt_1')
    assert File.exist? content_blob.filepath
    FileUtils.rm content_blob.filepath('pdf') if File.exist? content_blob.filepath('pdf')
    assert !File.exist?(content_blob.filepath('pdf'))

    content_blob.convert_to_pdf

    assert File.exist?(content_blob.filepath('pdf')), 'pdf was not created during conversion'
  end

  test 'is_content_viewable?' do
    Seek::Config.stub(:soffice_available?, true) do
      viewable_formats = %w[application/pdf]
      viewable_formats << 'application/msword'
      viewable_formats << 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
      viewable_formats << 'application/vnd.ms-powerpoint'
      viewable_formats << 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
      viewable_formats << 'application/vnd.oasis.opendocument.text'
      viewable_formats << 'application/vnd.oasis.opendocument.presentation'
      viewable_formats << 'application/rtf'

      viewable_formats.each do |viewable_format|
        cb_with_content_viewable_format = Factory(:content_blob, content_type: viewable_format, asset: Factory(:sop), data: File.new("#{Rails.root}/test/fixtures/files/a_pdf_file.pdf", 'rb').read)
        User.with_current_user cb_with_content_viewable_format.asset.contributor do
          assert cb_with_content_viewable_format.is_viewable_format?
          assert cb_with_content_viewable_format.is_content_viewable?
        end
      end
      cb_with_no_viewable_format = Factory(:content_blob, content_type: 'application/excel', asset: Factory(:sop), data: File.new("#{Rails.root}/test/fixtures/files/spreadsheet.xls", 'rb').read)
      User.with_current_user cb_with_no_viewable_format.asset.contributor do
        assert !cb_with_no_viewable_format.is_viewable_format?
        assert !cb_with_no_viewable_format.is_content_viewable?
      end
    end
  end

  test 'is_content_viewable? without soffice' do
    Seek::Config.stub(:soffice_available?, false) do
      viewable_formats = %w[pdf_content_blob] # Can still view PDFs

      unviewable_formats = []
      unviewable_formats << 'doc_content_blob'
      unviewable_formats << 'docx_content_blob'
      unviewable_formats << 'ppt_content_blob'
      unviewable_formats << 'pptx_content_blob'
      unviewable_formats << 'odt_content_blob'
      unviewable_formats << 'odp_content_blob'
      unviewable_formats << 'rtf_content_blob'

      viewable_formats.each do |viewable_format|
        cb_with_content_viewable_format = Factory(viewable_format.to_s, asset:Factory(:sop))
        User.with_current_user cb_with_content_viewable_format.asset.contributor do
          assert cb_with_content_viewable_format.is_viewable_format?
          assert cb_with_content_viewable_format.is_content_viewable?
        end
      end
      unviewable_formats.each do |unviewable_format|
        cb_with_content_unviewable_format = Factory(unviewable_format.to_s, asset:Factory(:sop))
        User.with_current_user cb_with_content_unviewable_format.asset.contributor do
          refute cb_with_content_unviewable_format.is_viewable_format?
          refute cb_with_content_unviewable_format.is_content_viewable?
        end
      end
    end
  end

  test 'content needing conversion should not be viewable when pdf_conversion is disabled' do
    with_config_value :pdf_conversion_enabled, false do
      viewable_formats = %w[doc_content_blob]
      viewable_formats << 'docx_content_blob'
      viewable_formats << 'ppt_content_blob'
      viewable_formats << 'pptx_content_blob'
      viewable_formats << 'odt_content_blob'
      viewable_formats << 'odp_content_blob'

      viewable_formats.each do |format|
        cb = Factory(format.to_s, asset:Factory(:sop))
        User.with_current_user cb.asset.contributor do
          refute cb.is_content_viewable?
        end
      end
    end
  end

  test 'content not needing conversion should be viewable when pdf_conversion is disabled' do
    pdf_content_blob = Factory(:content_blob, content_type: 'application/pdf', asset: Factory(:sop), data: File.new("#{Rails.root}/test/fixtures/files/a_pdf_file.pdf", 'rb').read)

    with_config_value :pdf_conversion_enabled, false do
      viewable_formats = %w[txt_content_blob csv_content_blob tsv_content_blob pdf_content_blob]

      viewable_formats.each do |format|
        cb = Factory(format.to_s,asset:Factory(:sop))
        User.with_current_user cb.asset.contributor do
          assert cb.is_content_viewable?
        end
      end

    end
  end

  test 'filter_text_content' do
    check_for_soffice
    ms_word_sop_cb = Factory(:doc_content_blob)
    content = "test \n content \f only"
    filtered_content = ms_word_sop_cb.send(:filter_text_content, content)
    assert !filtered_content.include?('\n')
    assert !filtered_content.include?('\f')
  end

  test 'pdf_contents_for_search for a doc file' do
    check_for_soffice
    ms_word_sop_content_blob = Factory(:doc_content_blob)
    assert ms_word_sop_content_blob.is_pdf_convertable?
    content = ms_word_sop_content_blob.pdf_contents_for_search
    assert_equal ['This is a ms word doc format'], content
  end

  test 'pdf_contents_for_search for a pdf file' do
    check_for_soffice
    pdf_content_blob = Factory(:pdf_content_blob)
    content = pdf_content_blob.pdf_contents_for_search
    assert_equal ['This is a pdf format'], content
  end

  test 'calculates file size' do
    blob = Factory(:pdf_content_blob)
    assert_equal 8827, blob.file_size
  end

  test 'updates file size' do
    blob = Factory(:pdf_content_blob)
    blob.data = '123456'
    blob.save
    assert_equal 6, blob.file_size
  end

  test 'calculates file size for remote content' do
    stub_request(:head, 'http://www.abc.com').to_return(headers: { content_length: 500, content_type: 'text/plain' }, status: 200)
    blob = Factory(:url_content_blob)
    assert_equal 500, blob.file_size
  end

  test 'calculates checksums of remote content' do
    stub_request(:head, 'http://www.abc.com').to_return(
        headers: { content_length: nil, content_type: 'text/plain' }, status: 200
    )
    stub_request(:get, 'http://www.abc.com').to_return(
        body: File.new("#{Rails.root}/test/fixtures/files/checksums.txt"),
        headers: { content_type: 'text/plain' }, status: 200)

    blob = Factory(:url_content_blob)
    blob.save
    blob.reload
    assert !blob.file_exists?
    assert blob.md5sum.blank?
    assert blob.sha1sum.blank?

    blob.retrieve

    assert blob.reload.file_exists?
    assert_equal 'd41d8cd98f00b204e9800998ecf8427e', blob.md5sum
    assert_equal 'da39a3ee5e6b4b0d3255bfef95601890afd80709', blob.sha1sum
  end

  test 'can retrieve remote content' do
    stub_request(:head, 'http://www.abc.com').to_return(
      headers: { content_length: nil, content_type: 'text/plain' }, status: 200
    )
    stub_request(:get, 'http://www.abc.com').to_return(body: 'abcdefghij' * 500,
                                                       headers: { content_type: 'text/plain' }, status: 200)

    blob = Factory(:url_content_blob)
    assert !blob.file_exists?
    assert_nil blob.file_size

    blob.retrieve
    assert blob.file_exists?
    assert_equal 5000, blob.file_size
  end

  test "won't retrieve remote content over hard limit" do
    # Web server lies about content length:
    stub_request(:head, 'http://www.abc.com').to_return(
      headers: { content_length: 500, content_type: 'text/plain' }, status: 200
    )
    # Size is actually 6kb:
    stub_request(:get, 'http://www.abc.com').to_return(body: 'abcdefghij' * 600,
                                                       headers: { content_type: 'text/plain' }, status: 200)

    blob = Factory(:url_content_blob)
    assert !blob.file_exists?
    assert_equal 500, blob.file_size

    assert_raise Seek::DownloadHandling::SizeLimitExceededException do
      blob.retrieve
    end
    assert !blob.file_exists?
  end

  test "won't endlessly follow redirects when downloading remote content" do
    stub_request(:head, 'http://www.abc.com').to_return(
      headers: { content_length: 500, content_type: 'text/plain' }, status: 200
    )
    # Infinitely redirects
    stub_request(:get, 'http://www.abc.com').to_return(headers: { location: 'http://www.abc.com' }, status: 302)

    blob = Factory(:url_content_blob)
    assert !blob.file_exists?
    assert_equal 500, blob.file_size

    assert_raise Seek::DownloadHandling::RedirectLimitExceededException do
      blob.retrieve
    end
    assert !blob.file_exists?
  end

  test 'raises exception on bad response code when downloading remote content' do
    stub_request(:head, 'http://www.abc.com').to_return(
      headers: { content_length: 500, content_type: 'text/plain' }, status: 200
    )
    stub_request(:get, 'http://www.abc.com').to_return(status: 404)

    blob = Factory(:url_content_blob)
    assert !blob.file_exists?
    assert_equal 500, blob.file_size

    assert_raise Seek::DownloadHandling::BadResponseCodeException do
      blob.retrieve
    end
    assert !blob.file_exists?
  end

  test 'handles relative redirects when downloading remote content' do
    stub_request(:head, 'http://www.abc.com').to_return(headers: { location: '/xyz' }, status: 302)
    stub_request(:get, 'http://www.abc.com').to_return(headers: { location: '/xyz' }, status: 302)
    stub_request(:head, 'http://www.abc.com/xyz').to_return(
      headers: { content_length: nil, content_type: 'text/plain' }, status: 200
    )
    stub_request(:get, 'http://www.abc.com/xyz').to_return(body: 'abcdefghij' * 500,
                                                           headers: { content_type: 'text/plain' }, status: 200)

    blob = Factory(:url_content_blob)
    assert !blob.file_exists?
    assert_nil blob.file_size

    blob.retrieve
    assert blob.file_exists?
    assert_equal 5000, blob.file_size
  end

  test 'handles absolute redirects when downloading remote content' do
    stub_request(:head, 'http://www.abc.com').to_return(headers: { location: 'http://www.abc.com/xyz' }, status: 302)
    stub_request(:get, 'http://www.abc.com').to_return(headers: { location: 'http://www.abc.com/xyz' }, status: 302)
    stub_request(:head, 'http://www.abc.com/xyz').to_return(
      headers: { content_length: nil, content_type: 'text/plain' }, status: 200
    )
    stub_request(:get, 'http://www.abc.com/xyz').to_return(body: 'abcdefghij' * 500,
                                                           headers: { content_type: 'text/plain' }, status: 200)

    blob = Factory(:url_content_blob)
    assert !blob.file_exists?
    assert_nil blob.file_size

    blob.retrieve
    assert blob.file_exists?
    assert_equal 5000, blob.file_size
  end

  test 'handles mixed redirects when downloading remote content' do
    stub_request(:head, 'http://www.abc.com').to_return(headers: { location: 'http://www.xyz.com' }, status: 302)
    stub_request(:get, 'http://www.abc.com').to_return(headers: { location: 'http://www.xyz.com' }, status: 302)
    stub_request(:head, 'http://www.xyz.com').to_return(headers: { location: '/xyz' }, status: 302)
    stub_request(:get, 'http://www.xyz.com').to_return(headers: { location: '/xyz' }, status: 302)
    stub_request(:head, 'http://www.xyz.com/xyz').to_return(
      headers: { content_length: nil, content_type: 'text/plain' }, status: 200
    )
    stub_request(:get, 'http://www.xyz.com/xyz').to_return(body: 'abcdefghij' * 500,
                                                           headers: { content_type: 'text/plain' }, status: 200)

    blob = Factory(:url_content_blob)
    assert !blob.file_exists?
    assert_nil blob.file_size

    blob.retrieve
    assert blob.file_exists?
    assert_equal 5000, blob.file_size
  end

  test "won't retrieve remote content from internal network" do
    begin
      # Need to allow the request through so that `private_address_check` can catch it.
      WebMock.allow_net_connect!
      VCR.turned_off do
        assert PrivateAddressCheck.resolves_to_private_address?('localhost')

        blob = Factory(:url_content_blob, url: 'http://localhost/secrets')
        assert !blob.file_exists?

        assert_raise PrivateAddressCheck::PrivateConnectionAttemptedError do
          blob.retrieve
        end

        assert !blob.file_exists?
      end
    ensure
      WebMock.disable_net_connect!(allow_localhost: true)
    end
  end

  test 'is_text?' do
    assert Factory(:txt_content_blob).is_text?
    assert Factory(:csv_content_blob).is_text?
    assert Factory(:tsv_content_blob).is_text?
    assert Factory(:teusink_model_content_blob).is_text?
    assert Factory(:json_content_blob).is_text?

    refute Factory(:ppt_content_blob).is_text?
    refute Factory(:binary_content_blob).is_text?
  end

  test 'can retrieve from NeLS' do
    setup_nels_for_units

    blob = ContentBlob.create(url: "https://test-fe.cbu.uib.no/nels/pages/sbi/sbi.xhtml?ref=#{@reference}")

    VCR.use_cassette('nels/get_sample_metadata') do
      blob.retrieve_from_nels(@nels_access_token)
    end
    blob.reload

    assert blob.is_excel?
    assert blob.file_size > 0
  end

  test 'raises 404 when nels sample metadata missing' do
    setup_nels_for_units

    blob = ContentBlob.create(url: 'https://test-fe.cbu.uib.no/nels/pages/sbi/sbi.xhtml?ref=404')

    assert_raises(RestClient::ResourceNotFound) do
      VCR.use_cassette('nels/missing_sample_metadata') do
        blob.retrieve_from_nels(@nels_access_token)
      end
    end
  end

  test 'added timestamps' do
    t1 = 1.day.ago
    blob = nil
    pretend_now_is(t1) do
      blob = Factory(:content_blob)
      assert_equal t1.to_s, blob.created_at.to_s
      assert_equal t1.to_s, blob.updated_at.to_s
    end

    t2 = 1.hour.ago
    pretend_now_is(t2) do
      blob.content_type = 'text/xml'
      blob.save!
      assert_equal t2.to_s, blob.updated_at.to_s
    end
  end

  test 'deletes image files after destroy' do
    blob = Factory(:image_content_blob)
    filepath = blob.file_path
    refute_nil filepath
    assert File.exist?(filepath)
    assert_difference('ContentBlob.count', -1) do
      blob.destroy
    end
    refute File.exist?(filepath)
  end

  test 'deletes files after destroy' do
    blob = Factory(:spreadsheet_content_blob)
    filepath = blob.file_path
    refute_nil filepath
    assert File.exist?(filepath)
    assert_difference('ContentBlob.count', -1) do
      blob.destroy
    end
    refute File.exist?(filepath)
  end

  test 'deletes converted files after destroy' do
    blob = Factory(:doc_content_blob)
    pdf_path = blob.filepath('pdf')
    txt_path = blob.filepath('txt')

    # pretend the conversion has taken place
    FileUtils.touch(pdf_path)
    FileUtils.touch(txt_path)

    assert File.exist?(pdf_path)
    assert File.exist?(txt_path)
    assert_difference('ContentBlob.count', -1) do
      blob.destroy
    end
    refute File.exist?(pdf_path)
    refute File.exist?(txt_path)
  end
end


text:
/* Infowindow Roboto font override */
.gm-style div, .gm-style span, .gm-style label, .gm-style a {
  font-family: Arial, Helvetica, sans-serif; }

.bh-sl-error {
  clear: both;
  float: left;
  width: 100%;
  padding: 10px 0;
  color: #ae2118;
  font-weight: bold; }

.bh-sl-container {
  color: #555;
  /* Avoid image issues with Google Maps and CSS resets */
  /* Avoid issues with Google Maps and CSS frameworks */ }
  .bh-sl-container img {
    max-width: none !important;
    border-radius: 0 !important;
    box-shadow: none !important; }
  .bh-sl-container > * {
    box-sizing: content-box !important; }
  .bh-sl-container .jumbotron {
    margin-bottom: 0; }
  .bh-sl-container .form-input input, .bh-sl-container .form-input select, .bh-sl-container .form-input label {
    margin-right: 10px; }
  .bh-sl-container .loc-alt-dist {
    display: none; }
  .bh-sl-container .bh-sl-loading {
    float: left;
    margin: 4px 0 0 10px;
    width: 16px;
    height: 16px;
    background: url(../img/ajax-loader.gif) no-repeat; }
  .bh-sl-container .bh-sl-filters-container {
    clear: both;
    width: 100%;
    margin: 15px 0; }
    .bh-sl-container .bh-sl-filters-container .bh-sl-filters {
      list-style: none;
      float: left;
      padding: 0;
      margin: 0 100px 0 0; }
      .bh-sl-container .bh-sl-filters-container .bh-sl-filters li {
        display: block;
        clear: left;
        float: left;
        width: 100%;
        margin: 5px 0; }
        .bh-sl-container .bh-sl-filters-container .bh-sl-filters li label {
          display: inline; }
        .bh-sl-container .bh-sl-filters-container .bh-sl-filters li input {
          display: block;
          float: left;
          margin: 2px 8px 2px 0; }
  .bh-sl-container .bh-sl-map-container a {
    color: #005293;
    text-decoration: none; }
    .bh-sl-container .bh-sl-map-container a:hover, .bh-sl-container .bh-sl-map-container a:active {
      text-decoration: underline; }
  .bh-sl-container .bh-sl-loc-list {
    height: 530px;
    overflow-x: auto;
    font-size: 13px; }
    .bh-sl-container .bh-sl-loc-list ul {
      display: block;
      clear: left;
      float: left;
      width: 100%;
      list-style: none;
      margin: 0;
      padding: 0; }
      .bh-sl-container .bh-sl-loc-list ul li {
        display: block;
        clear: left;
        float: left;
        margin: 3% 4%;
        cursor: pointer;
        width: 92%;
        border: 1px solid #fff;
        /* Adding this to prevent moving li elements when adding the list-focus class*/ }
    .bh-sl-container .bh-sl-loc-list .list-label {
      float: left;
      margin: 10px 0 0 6px;
      padding: 4px;
      width: 27px;
      text-align: center;
      background: #00192d;
      color: #fff;
      font-weight: bold;
      border-radius: 15px; }
    .bh-sl-container .bh-sl-loc-list .list-details {
      float: left;
      margin-left: 6px;
      width: 80%; }
      .bh-sl-container .bh-sl-loc-list .list-details .list-content {
        padding: 10px; }
      .bh-sl-container .bh-sl-loc-list .list-details .loc-dist {
        font-weight: bold;
        font-style: italic;
        color: #8e8e8e; }
    .bh-sl-container .bh-sl-loc-list .list-focus {
      border: 1px solid rgba(0, 82, 147, 0.4);
      -moz-box-shadow: 0 0 8px rgba(0, 82, 147, 0.4);
      -webkit-box-shadow: 0 0 8px rgba(0, 82, 147, 0.4);
      box-shadow: 0 0 8px rgba(0, 100, 180, 0.4);
      transition: border 0.2s linear 0s, box-shadow 0.2s linear 0s; }
    .bh-sl-container .bh-sl-loc-list .bh-sl-close-directions-container {
      width: 100%;
      height: 20px;
      position: relative; }
      .bh-sl-container .bh-sl-loc-list .bh-sl-close-directions-container .bh-sl-close-icon {
        top: 0;
        right: 6px; }
    .bh-sl-container .bh-sl-loc-list .bh-sl-noresults-title {
      font-weight: bold; }
  .bh-sl-container .loc-name {
    /* Picked up by both list and infowindows */
    font-size: 15px;
    font-weight: bold; }
  .bh-sl-container .bh-sl-map {
    height: 530px; }
  .bh-sl-container .bh-sl-pagination-container {
    clear: both; }
    .bh-sl-container .bh-sl-pagination-container ol {
      list-style-type: none;
      text-align: center;
      margin: 0;
      padding: 10px 0; }
      .bh-sl-container .bh-sl-pagination-container ol li {
        display: inline-block;
        padding: 10px;
        cursor: pointer;
        font: bold 14px Arial, Helvetica, sans-serif;
        color: #005293; }
      .bh-sl-container .bh-sl-pagination-container ol .bh-sl-current {
        color: #555;
        cursor: auto;
        text-decoration: none; }

/* Modal window */
.bh-sl-overlay {
  position: fixed;
  left: 0;
  top: 0;
  width: 100%;
  height: 100%;
  z-index: 10000;
  background: url(../img/overlay-bg.png) repeat; }
  .bh-sl-overlay .bh-sl-modal-window {
    position: absolute;
    left: 50%;
    margin-left: -460px;
    /* width divided by 2 */
    margin-top: 60px;
    width: 920px;
    height: 620px;
    z-index: 10010;
    background: #fff;
    border-radius: 10px;
    box-shadow: 0 0 10px #656565; }
    .bh-sl-overlay .bh-sl-modal-window .bh-sl-map-container {
      margin-top: 50px;
      /* increase map container margin */ }
    .bh-sl-overlay .bh-sl-modal-window .bh-sl-modal-content {
      float: left;
      padding: 0 22px;
      /* there's already a margin on the top of the map-container div */ }
    .bh-sl-overlay .bh-sl-modal-window .bh-sl-close-icon {
      top: 13px;
      right: 22px; }

.bh-sl-close-icon {
  position: absolute;
  cursor: pointer;
  height: 24px;
  width: 24px; }
  .bh-sl-close-icon:after, .bh-sl-close-icon:before {
    position: absolute;
    top: 3px;
    right: 3px;
    bottom: 0;
    left: 50%;
    background: #ccc;
    content: '';
    display: block;
    height: 24px;
    margin: -3px 0 0 -1px;
    width: 3px;
    -webkit-transform: rotate(45deg);
    -moz-transform: rotate(45deg);
    -ms-transform: rotate(45deg);
    -o-transform: rotate(45deg);
    transform: rotate(45deg); }
  .bh-sl-close-icon:hover:after, .bh-sl-close-icon:hover:before {
    background: #b3b3b3; }
  .bh-sl-close-icon:before {
    -webkit-transform: rotate(-45deg);
    -moz-transform: rotate(-45deg);
    -ms-transform: rotate(-45deg);
    -o-transform: rotate(-45deg);
    transform: rotate(-45deg); }

/*# sourceMappingURL=bootstrap-example.css.map */


text:
/**
 *
 */
package redmart.impl;

import java.io.FileReader;
import java.io.IOException;
import java.util.List;

import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;

import redmart.CustomerPrize;
import redmart.MapImplTest;
import au.com.bytecode.opencsv.CSVReader;

/**
 * @author debmalyajash
 *
 */
public class CustomerPrizeImplTest {

	private static final String PRODUCT_FILE = MapImplTest.DIR + "products.csv";

	private static final Logger LOGGER = Logger.getLogger(CustomerPrizeImplTest.class);

	/**
	 * Test method for {@link redmart.impl.CustomerPrizeImpl#selectProduct(java.util.List)}.
	 */
	@Test
	public final void testSelectProduct() {
		CSVReader reader =  null;
		CustomerPrize prize = new CustomerPrizeImpl();
		try {
			reader = new CSVReader(new FileReader(PRODUCT_FILE));
			List<String[]> productList = reader.readAll();
			String productID = prize.selectProduct(productList);
			Assert.assertNotNull(productID);
			System.out.println(productID);
		} catch (Throwable e) {
			LOGGER.error(e.getMessage(),e);
			Assert.fail(e.getMessage());
		} finally {
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e) {
					LOGGER.error("CLOSE CALL:" + e.getMessage(), e );
				}
			}
		}
	}

}


text:
cask :v1 => 'mailfollowup' do
  version '1.6.2'
  sha256 'f656c121e04fac6185b24a96888613729c366275151c891258b634c71372b2af'

  url "https://www.cs.unc.edu/~welch/MailFollowup/media/MailFollowUp_#{version}.dmg.zip"
  homepage 'http://www.cs.unc.edu/~welch/MailFollowup/'
  license :unknown    # todo: change license and remove this comment; ':unknown' is a machine-generated placeholder

  container :nested => "MailFollowUp_#{version}.dmg"
  pkg 'Install MailFollowUp.pkg'

  uninstall :quit  => 'com.apple.mail',
            :pkgutil => [
                        'net.welchwold.mailfollowup',
                        'net.welchwold.mailfollowup.*',
                        ]
  zap       :delete => '~/Library/Mail/Bundles/MailFollowUp.mailbundle/'
end


text:
package org.jbehave.core.junit.needle;

import org.jbehave.core.configuration.AnnotationBuilder;
import org.jbehave.core.configuration.needle.NeedleAnnotationBuilder;
import org.jbehave.core.junit.AnnotatedEmbedderRunner;
import org.junit.runners.model.InitializationError;

/**
 * AnnotatedEmbedderRunner that uses {@link NeedleAnnotationBuilder}.
 */
public class NeedleAnnotatedEmbedderRunner extends AnnotatedEmbedderRunner {

	private NeedleAnnotationBuilder annotationBuilder;

	public NeedleAnnotatedEmbedderRunner(Class<?> testClass) throws InitializationError {
		super(testClass);
		this.annotationBuilder = new NeedleAnnotationBuilder(testClass());
	}

	@Override
    public AnnotationBuilder annotationBuilder() {
		return annotationBuilder;
	}

}


text:
/**
  * \brief example showing the usage of boundary splitting by using simulated annealing and threading
  *        to find the minimum of x^2
  */

#include <cmath>
#include <iostream>

#include "cppOpt.h"

using namespace cppOpt;

//define your own calculation
auto toOptimize = [](OptCalculation<double>& optCalculation) {
    //defined x^2 as function to be optimised
    optCalculation.result = pow(optCalculation.get_parameter("X"), 2);
};

using namespace std;

int main() {
    //setup the optimisation
    //define your total boundaries
    OptBoundaries<double> optBoundariesTotal;
    optBoundariesTotal.add_boundary({-5.0, 5.0, "X"});

    //and split them into 4
    vector<OptBoundaries<double>> optBoundariesSplit = optBoundariesTotal.split("X", 4);

    //number of calculations
    unsigned int maxCalculations = 300;

    //we want to find the minimum
    OptTarget optTarget = OptTarget::MINIMIZE;

    //how fast the simulated annealing algorithm slows down
    //http://en.wikipedia.org/wiki/Simulated_annealing
    double coolingFactor = 0.9;

    //the chance in the beginning to follow bad solutions
    double startChance = 0.25;

    //define your coordinator
    OptCoordinator<double, true> coordinator(
        maxCalculations,
        toOptimize,
        optTarget,
        0);

    //add 4 different simulated annealing children
    coordinator.add_child(make_unique<OptSimulatedAnnealing<double>>(
        optBoundariesSplit[0],
        coolingFactor,
        startChance));

    coordinator.add_child(make_unique<OptSimulatedAnnealing<double>>(
        optBoundariesSplit[1],
        coolingFactor,
        startChance));

    coordinator.add_child(make_unique<OptSimulatedAnnealing<double>>(
        optBoundariesSplit[2],
        coolingFactor,
        startChance));

    coordinator.add_child(make_unique<OptSimulatedAnnealing<double>>(
        optBoundariesSplit[3],
        coolingFactor,
        startChance));

    //let's go
    coordinator.run_optimisation(4);

    //print result
    OptCalculation<double> best = coordinator.get_best_calculation();
    cout << best.to_string_header() << endl;
    cout << best.to_string_values() << endl;

    return 0;
}