text:
package alluxio.worker.block;

import alluxio.Configuration;
import alluxio.Constants;
import alluxio.PropertyKey;
import alluxio.underfs.SeekableUnderFileInputStream;
import alluxio.underfs.UnderFileSystem;
import alluxio.underfs.options.OpenOptions;
import alluxio.util.IdUtils;
import alluxio.util.executor.ExecutorServiceFactories;

import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalListeners;
import com.google.common.cache.RemovalNotification;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;

import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.ThreadSafe;

/**
 * An under filesystem input stream manager that can cache seekable input streams for future reuse.
 * The manager caches the input streams, with a time-based eviction policy of expiration configured
 * in {@link PropertyKey.WORKER_UFS_INSTREAM_CACHE_EXPIRE_MS}.
 *
 * An under storage file maybe opened with multiple input streams at the same time. The manager uses
 * {@link UfsInputStreamIdSet} to track the in-use input stream and the available ones. The manager
 * closes the input streams after they are expired and not in-use anymore. Lock
 * {@link UfsInputStreamIdSet} before access, and in addition to that, lock
 * {@link #mFileIdToInputStreamIds} before the resource retrieval.
 */
@ThreadSafe
public class UfsInputStreamManager {
  private static final Logger LOG = LoggerFactory.getLogger(UfsInputStreamManager.class);
  private static final long UNAVAILABLE_RESOURCE_ID = -1;
  private static final boolean CACHE_ENABLED =
      Configuration.getBoolean(PropertyKey.WORKER_UFS_INSTREAM_CACHE_ENABLED);

  /**
   * A map from the ufs file id to the metadata of the input streams. Synchronization on this map
   * before access.
   */
  private final Map<Long, UfsInputStreamIdSet> mFileIdToInputStreamIds;
  /** Cache of the input streams, from the input stream id to the input stream. */
  @GuardedBy("mFileToInputStreamIds")
  private final Cache<Long, CachedSeekableInputStream> mUnderFileInputStreamCache;
  /** Thread pool for asynchronously removing the expired input streams. */
  private final ExecutorService mRemovalThreadPool;

  /**
   * Creates a new {@link UfsInputStreamManager}.
   */
  public UfsInputStreamManager() {
    mFileIdToInputStreamIds = new HashMap<>();
    mRemovalThreadPool = ExecutorServiceFactories
        .fixedThreadPool(Constants.UFS_INPUT_STREAM_CACHE_EXPIRATION, 2)
        .create();

    // A listener to the input stream removal.
    RemovalListener<Long, CachedSeekableInputStream> listener =
        (RemovalNotification<Long, CachedSeekableInputStream> removal) -> {
          CachedSeekableInputStream inputStream = removal.getValue();
          boolean shouldClose = false;
          synchronized (mFileIdToInputStreamIds) {
            if (mFileIdToInputStreamIds.containsKey(inputStream.getFileId())) {
              UfsInputStreamIdSet resources = mFileIdToInputStreamIds.get(inputStream.getFileId());
              synchronized (resources) {
                // remove the key
                resources.removeInUse(removal.getKey());
                if (resources.removeAvailable(removal.getKey())) {
                  // close the resource
                  LOG.debug("Removed the under file input stream resource of {}", removal.getKey());
                  shouldClose = true;
                }
                if (resources.isEmpty()) {
                  // remove the resources entry
                  mFileIdToInputStreamIds.remove(inputStream.getFileId());
                }
              }
            } else {
              LOG.warn("Try to remove the resource entry of {} but not exists any more",
                  removal.getKey());
            }
          }
          if (shouldClose) {
            try {
              inputStream.close();
            } catch (IOException e) {
              LOG.warn("Failed to close the input stream resource of file {} with file id {}",
                  " and resource id {}", inputStream.getFilePath(), inputStream.getFileId(),
                  removal.getKey());
            }
          }
        };
    mUnderFileInputStreamCache = CacheBuilder.newBuilder()
        .maximumSize(Configuration.getInt(PropertyKey.WORKER_UFS_INSTREAM_CACHE_MAX_SIZE))
        .expireAfterAccess(
            Configuration.getMs(PropertyKey.WORKER_UFS_INSTREAM_CACHE_EXPIRARTION_TIME),
            TimeUnit.MILLISECONDS)
        .removalListener(RemovalListeners.asynchronous(listener, mRemovalThreadPool)).build();
  }

  /**
   * Releases an input stream. The input stream is closed if it's already expired.
   *
   * @param inputStream the input stream to release
   * @throws IOException when input stream fails to close
   */
  public void release(InputStream inputStream) throws IOException {
    // for non-seekable input stream, close and return
    if (!(inputStream instanceof CachedSeekableInputStream) || !CACHE_ENABLED) {
      inputStream.close();
      return;
    }

    synchronized (mFileIdToInputStreamIds) {
      if (!mFileIdToInputStreamIds
          .containsKey(((CachedSeekableInputStream) inputStream).getFileId())) {
        LOG.debug("The resource {} is already expired",
            ((CachedSeekableInputStream) inputStream).getResourceId());
        // the cache no longer tracks this input stream
        inputStream.close();
        return;
      }
      UfsInputStreamIdSet resources =
          mFileIdToInputStreamIds.get(((CachedSeekableInputStream) inputStream).getFileId());
      if (!resources.release(((CachedSeekableInputStream) inputStream).getResourceId())) {
        LOG.debug("Close the expired input stream resource of {}",
            ((CachedSeekableInputStream) inputStream).getResourceId());
        // the input stream expired, close it
        inputStream.close();
      }
    }
  }

  /**
   * Invalidates an input stream from the cache.
   *
   * @param inputStream the cached input stream
   * @throws IOException when the invalidated input stream fails to release
   */
  public void invalidate(CachedSeekableInputStream inputStream) throws IOException {
    mUnderFileInputStreamCache.invalidate(inputStream.getResourceId());
    release(inputStream);
  }

  /**
   * Acquires an input stream. For seekable input streams, if there is an available input stream in
   * the cache, reuse it and repositions the offset, otherwise the manager opens a new input stream.
   *
   * @param ufs the under file system
   * @param path the path to the under storage file
   * @param fileId the file id
   * @param openOptions the open options
   * @return the acquired input stream
   * @throws IOException if the input stream fails to open
   */
  public InputStream acquire(UnderFileSystem ufs, String path, long fileId, OpenOptions openOptions)
      throws IOException {
    return acquire(ufs, path, fileId, openOptions, true);
  }

  /**
   * Acquires an input stream. For seekable input streams, if there is an available input stream in
   * the cache and reuse mode is specified, reuse it and repositions the offset, otherwise the
   * manager opens a new input stream.
   *
   * @param ufs the under file system
   * @param path the path to the under storage file
   * @param fileId the file id
   * @param openOptions the open options
   * @param reuse true to reuse existing input stream, otherwise acquire a new stream
   * @return the acquired input stream
   * @throws IOException if the input stream fails to open
   */
  public InputStream acquire(UnderFileSystem ufs, String path, long fileId, OpenOptions openOptions,
      boolean reuse) throws IOException {
    if (!ufs.isSeekable() || !CACHE_ENABLED) {
      // not able to cache, always return a new input stream
      return ufs.open(path, openOptions);
    }

    // explicit cache cleanup
    mUnderFileInputStreamCache.cleanUp();

    UfsInputStreamIdSet resources;
    synchronized (mFileIdToInputStreamIds) {
      if (mFileIdToInputStreamIds.containsKey(fileId)) {
        resources = mFileIdToInputStreamIds.get(fileId);
      } else {
        resources = new UfsInputStreamIdSet();
        mFileIdToInputStreamIds.put(fileId, resources);
      }
    }

    synchronized (resources) {
      long nextId = UNAVAILABLE_RESOURCE_ID;
      CachedSeekableInputStream inputStream = null;
      if (reuse) {
        // find the next available input stream from the cache
        for (long id : resources.availableIds()) {
          inputStream = mUnderFileInputStreamCache.getIfPresent(id);
          if (inputStream != null) {
            nextId = id;
            LOG.debug("Reused the under file input stream resource of {}", nextId);
            // for the cached ufs instream, seek to the requested position
            inputStream.seek(openOptions.getOffset());
            break;
          }
        }
      }
      // no cached input stream is available, open a new one
      if (nextId == UNAVAILABLE_RESOURCE_ID) {
        nextId = IdUtils.getRandomNonNegativeLong();
        final long newId = nextId;
        try {
          inputStream = mUnderFileInputStreamCache.get(nextId, () -> {
            SeekableUnderFileInputStream ufsStream = (SeekableUnderFileInputStream) ufs.open(path,
                OpenOptions.defaults().setOffset(openOptions.getOffset()));
            LOG.debug("Created the under file input stream resource of {}", newId);
            return new CachedSeekableInputStream(ufsStream, newId, fileId, path);
          });
        } catch (ExecutionException e) {
          LOG.warn("Failed to create a new cached ufs instream of file id {} and path {}", fileId,
              path);
          // fall back to a ufs creation.
          return ufs.open(path, OpenOptions.defaults().setOffset(openOptions.getOffset()));
        }
      }

      // mark the input stream id as acquired
      resources.acquire(nextId);
      return inputStream;
    }
  }

  /**
   * The metadata of the input streams associated with an under storage file that tracks which input
   * streams are in-use or available. Each input stream is identified by a unique id.
   */
  @ThreadSafe
  private static class UfsInputStreamIdSet {
    private final Set<Long> mInUseStreamIds;
    private final Set<Long> mAvailableStreamIds;

    /**
     * Creates a new {@link UfsInputStreamIdSet}.
     */
    UfsInputStreamIdSet() {
      mInUseStreamIds = new HashSet<>();
      mAvailableStreamIds = new HashSet<>();
    }

    /**
     * @return a view of the available input stream ids
     */
    Set<Long> availableIds() {
      return Collections.unmodifiableSet(mAvailableStreamIds);
    }

    /**
     * Marks an input stream as acquired.
     *
     * @param id the id of the input stream
     */
    synchronized void acquire(long id) {
      Preconditions.checkArgument(!mInUseStreamIds.contains(id), "%d is already in use", id);
      mAvailableStreamIds.remove(id);
      mInUseStreamIds.add(id);
    }

    /**
     * @return if there is any outstanding input streams of the file
     */
    synchronized boolean isEmpty() {
      return mInUseStreamIds.isEmpty() && mAvailableStreamIds.isEmpty();
    }

    /**
     * Marks an input stream as not in use.
     * @param id the id of the input stream
     */
    synchronized void removeInUse(long id) {
      mInUseStreamIds.remove(id);
    }

    /**
     * Removes the mark of the input stream as available.
     *
     * @param id the id of the input stream
     * @return true if the given input stream is available, false if the
     *         given input stream is not among available input streams
     */
    synchronized boolean removeAvailable(long id) {
      return mAvailableStreamIds.remove(id);
    }

    /**
     * Returns an id to the input stream pool. It marks the id from in use to available. If the
     * input stream is already removed from the cache, then do nothing.
     *
     * @param id id of the input stream
     * @return true if the id is marked from in use to available; false if the id no longer used for
     *         cache
     */
    synchronized boolean release(long id) {
      Preconditions.checkArgument(!mAvailableStreamIds.contains(id));
      if (mInUseStreamIds.contains(id)) {
        mInUseStreamIds.remove(id);
        mAvailableStreamIds.add(id);
        return true;
      }
      return false;
    }
  }
}


text:
<?php

/**
 * Command for unzipping zip files on a remote file system.
 *
 * @package MOXMAN_Core
 */
class MOXMAN_Core_UnZipCommand extends MOXMAN_Core_BaseCommand {
	/**
	 * Executes the command logic with the specified RPC parameters.
	 *
	 * @param Object $params Command parameters sent from client.
	 * @return Object Result object to be passed back to client.
	 */
	public function execute($params) {
		$fromFile = MOXMAN::getFile($params->from);
		$toFile = MOXMAN::getFile($params->to);
		$config = $toFile->getConfig();

		if ($config->get('general.demo')) {
			throw new MOXMAN_Exception(
				"This action is restricted in demo mode.",
				MOXMAN_Exception::DEMO_MODE
			);
		}

		if (!$fromFile->exists()) {
			throw new MOXMAN_Exception(
				"From file doesn't exist: " . $fromFile->getPublicPath(),
				MOXMAN_Exception::FILE_DOESNT_EXIST
			);
		}

		if (!$toFile->canWrite()) {
			throw new MOXMAN_Exception(
				"No write access to file: " . $toFile->getPublicPath(),
				MOXMAN_Exception::NO_WRITE_ACCESS
			);
		}

		$paths = array();
		$fileSystemManager = MOXMAN::getFileSystemManager();
		$zipArchive = new ZipArchive();
		$localTempFilePath = null;
		$result = array();

		if ($fromFile instanceof MOXMAN_Vfs_Local_File) {
			$res = $zipArchive->open($fromFile->getPath());
		} else {
			$localTempFilePath = $fileSystemManager->getLocalTempPath($fromFile);
			$fromFile->exportTo($localTempFilePath);
			$res = $zipArchive->open($localTempFilePath);
		}

		if ($res) {
			for ($i = 0; $i < $zipArchive->numFiles; $i++) {
				$stat = $zipArchive->statIndex($i);
				$paths[] = $stat["name"];
			}

			$filter = MOXMAN_Vfs_BasicFileFilter::createFromConfig($config);
			$fileSystem = $toFile->getFileSystem();

			foreach ($paths as $path) {
				$isFile = !preg_match('/\/$/', $path);
				$toPath = MOXMAN_Util_PathUtils::combine($toFile->getPath(), iconv('cp437', 'UTF-8', $path));
				$targetFile = MOXMAN::getFile($toPath);

				if ($filter->accept($targetFile, $isFile) === MOXMAN_Vfs_IFileFilter::ACCEPTED) {
					if ($isFile) {
						$content = $zipArchive->getFromName($path);

						// Fire before file action add event
						$args = new MOXMAN_Core_FileActionEventArgs("add", $targetFile);
						$args->getData()->fileSize = strlen($content);
						MOXMAN::getPluginManager()->get("core")->fire("BeforeFileAction", $args);
						$targetFile = $args->getFile();

						$targetFile = $this->mkdirs($targetFile, true);

						$stream = $targetFile->open(MOXMAN_Vfs_IFileStream::WRITE);
						$stream->write($content);
						$stream->close();
						//echo "Create file: ". $targetFile->getPublicPath() ."\n";

						$this->fireFileAction(MOXMAN_Core_FileActionEventArgs::ADD, $targetFile);
					} else {
						$targetFile = $this->mkdirs($targetFile);
					}

					$result[] = $this->fileToJson($targetFile);
				}
			}

			$zipArchive->close();

			if ($localTempFilePath) {
				$fileSystemManager->removeLocalTempFile($fromFile);
			}
		}

		return $result;
	}

	/** @ignore */
	private function mkdirs(MOXMAN_Vfs_IFile $file, $isFile=false) {
		$parents = array();

		$orgFile = $file;

		if ($isFile) {
			$file = $file->getParentFile();
		}

		$pathChunks = explode("/", $file->getPublicPath());

		// Ignore first slash
		array_shift($pathChunks);
		$path = "";
		$chunkFile = null;

		foreach($pathChunks as $chunk) {
			$path .= "/". $chunk;
			$chunkFile = MOXMAN::getFile($path);

			// Ignore root
			if (!$chunkFile->getParent()) {
				continue;
			}

			$args = new MOXMAN_Core_FileActionEventArgs("add", $chunkFile);
			MOXMAN::getPluginManager()->get("core")->fire("BeforeFileAction", $args);
			$chunkFile = $args->getFile();
			$path = $chunkFile->getPublicPath();

			if (!$chunkFile->exists()) {
				$chunkFile->mkdir();
				$this->fireFileAction(MOXMAN_Core_FileActionEventArgs::ADD, $chunkFile);
			}
		}

		if ($chunkFile) {
			if ($isFile) {
				return MOXMAN::getFile($chunkFile->getPath(), $orgFile->getName());
			}

			return $chunkFile;
		}

		return $orgFile;
	}
}

?>

redpajama_set_name: RedPajamaGithub


text:
Clazz.declarePackage ("JS");
Clazz.load (["JS.AbstractButton"], "JS.JCheckBox", null, function () {
c$ = Clazz.declareType (JS, "JCheckBox", JS.AbstractButton);
Clazz.makeConstructor (c$,
function () {
Clazz.superConstructor (this, JS.JCheckBox, ["chkJCB"]);
});
Clazz.overrideMethod (c$, "toHTML",
function () {
var s = "<label><input type=checkbox id='" + this.id + "' class='JCheckBox' style='" + this.getCSSstyle (0, 0) + "' " + (this.selected ? "checked='checked' " : "") + "onclick='SwingController.click(this)'>" + this.text + "</label>";
return s;
});
});


text:
package com.dremio.dac.homefiles;

import static java.lang.String.format;

import java.io.IOException;
import java.nio.file.attribute.PosixFilePermission;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.RejectedExecutionException;

import javax.inject.Provider;

import com.dremio.common.exceptions.UserException;
import com.dremio.common.logical.FormatPluginConfig;
import com.dremio.common.utils.PathUtils;
import com.dremio.connector.ConnectorException;
import com.dremio.connector.metadata.BytesOutput;
import com.dremio.connector.metadata.DatasetHandle;
import com.dremio.connector.metadata.DatasetMetadata;
import com.dremio.connector.metadata.DatasetNotFoundException;
import com.dremio.connector.metadata.EntityPath;
import com.dremio.connector.metadata.GetDatasetOption;
import com.dremio.connector.metadata.extensions.ValidateMetadataOption;
import com.dremio.connector.metadata.options.TimeTravelOption;
import com.dremio.dac.model.spaces.HomeName;
import com.dremio.exec.catalog.CurrentSchemaOption;
import com.dremio.exec.catalog.FileConfigOption;
import com.dremio.exec.catalog.MetadataObjectsUtils;
import com.dremio.exec.catalog.SortColumnsOption;
import com.dremio.exec.catalog.StoragePluginId;
import com.dremio.exec.server.SabotContext;
import com.dremio.exec.store.DatasetRetrievalOptions;
import com.dremio.exec.store.dfs.FileDatasetHandle;
import com.dremio.exec.store.dfs.FileSelection;
import com.dremio.exec.store.dfs.FormatPlugin;
import com.dremio.exec.store.dfs.MayBeDistFileSystemPlugin;
import com.dremio.exec.store.dfs.PhysicalDatasetUtils;
import com.dremio.exec.store.dfs.PreviousDatasetInfo;
import com.dremio.exec.store.file.proto.FileProtobuf;
import com.dremio.exec.store.file.proto.FileProtobuf.FileUpdateKey;
import com.dremio.io.file.FileAttributes;
import com.dremio.io.file.FileSystem;
import com.dremio.io.file.FileSystemUtils;
import com.dremio.io.file.Path;
import com.dremio.service.namespace.NamespaceException;
import com.dremio.service.namespace.NamespaceKey;
import com.dremio.service.namespace.NamespaceNotFoundException;
import com.dremio.service.namespace.dataset.proto.DatasetConfig;
import com.dremio.service.namespace.dataset.proto.DatasetType;
import com.dremio.service.namespace.file.proto.FileConfig;
import com.dremio.service.users.SystemUser;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;

/**
 * New storage plugin for home files
 */
public class HomeFileSystemStoragePlugin extends MayBeDistFileSystemPlugin<HomeFileConf> {
  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HomeFileSystemStoragePlugin.class);

  static final Set<PosixFilePermission> DEFAULT_PERMISSIONS = Sets.immutableEnumSet(
        PosixFilePermission.OWNER_READ,
        PosixFilePermission.OWNER_WRITE,
        PosixFilePermission.OWNER_EXECUTE);

  public static final String HOME_PLUGIN_NAME = "__home";
  private static final String UPLOADS = "_uploads";
  private static final String STAGING = "_staging";

  private final Path stagingDir;
  private final Path uploadsDir;
  private Object deleteHookKey;

  public HomeFileSystemStoragePlugin(final HomeFileConf config, final SabotContext context, final String name, Provider<StoragePluginId> idProvider) {
    super(config, context, name, idProvider);
    this.stagingDir = config.getPath().resolve(STAGING + "." + context.getDremioConfig().getThisNode());
    this.uploadsDir = config.getPath().resolve(UPLOADS);
  }

  @Override
  public HomeFileConf getConfig() {
    return super.getConfig();
  }

  @Override
  public void start() throws IOException {
    super.start();
    FileSystem fs = getSystemUserFS();
    fs.mkdirs(getConfig().getPath(), DEFAULT_PERMISSIONS);
    fs.mkdirs(stagingDir, DEFAULT_PERMISSIONS);
    fs.mkdirs(uploadsDir, DEFAULT_PERMISSIONS);
    deleteHookKey = FileSystemUtils.deleteOnExit(fs, stagingDir);
  }

  @Override
  public void close() {
    if (deleteHookKey != null) {
      try {
        if (getSystemUserFS().exists(stagingDir)) {
          getSystemUserFS().delete(stagingDir, true);
          FileSystemUtils.cancelDeleteOnExit(deleteHookKey);
        }
      } catch (IOException| RejectedExecutionException ex) {
        logger.warn("Unable to delete staging directory when closing HomeFileSystemPlugin.", ex);
      }
      deleteHookKey = null;
    }
    super.close();
  }

  @Override
  public MetadataValidity validateMetadata(BytesOutput signature, DatasetHandle datasetHandle, DatasetMetadata metadata,
      ValidateMetadataOption... options) throws DatasetNotFoundException {
    return MetadataValidity.VALID;
  }

  @Override
  public BytesOutput provideSignature(DatasetHandle datasetHandle, DatasetMetadata metadata) throws ConnectorException {
    return BytesOutput.NONE;
  }

  @Override
  public Optional<DatasetHandle> getDatasetHandle(EntityPath datasetPath, GetDatasetOption... options) throws ConnectorException {
    if (datasetPath.size() <= 1) {
      return Optional.empty();
    }
    NamespaceKey namespaceKey = MetadataObjectsUtils.toNamespaceKey(datasetPath);
    DatasetRetrievalOptions retrievalOptions = DatasetRetrievalOptions.of(options);

    FileConfig fileConfig = FileConfigOption.getFileConfig(options);
    PreviousDatasetInfo oldConfig = new PreviousDatasetInfo(fileConfig, CurrentSchemaOption.getSchema(options), SortColumnsOption.getSortColumns(options), null, null, true);
    FormatPluginConfig pluginConfig = null;
    try {
      final FileSystem fs = getSystemUserFS();

      if (fileConfig == null) {
        final DatasetConfig datasetConfig = getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(namespaceKey);
        if (datasetConfig.getPhysicalDataset() == null) {
          throw UserException.validationError().message("not a valid physical dataset").buildSilently();
        } else {
          fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings();
        }
      }

      pluginConfig = PhysicalDatasetUtils.toFormatPlugin(fileConfig, Collections.<String>emptyList());
      final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(pluginConfig);
      return Optional.ofNullable(getDataset(namespaceKey, oldConfig, formatPlugin, fs, fileConfig,
          retrievalOptions.maxMetadataLeafColumns(), retrievalOptions.getTimeTravelRequest()));
    } catch (NamespaceNotFoundException nfe) {
      return Optional.empty();
    } catch (NamespaceException | IOException e) {
      throw new ConnectorException(e);
    }
  }

  @Override
  protected FileDatasetHandle getDatasetWithFormat(NamespaceKey datasetPath, PreviousDatasetInfo oldConfig, FormatPluginConfig formatPluginConfig, DatasetRetrievalOptions retrievalOptions, String user) throws Exception {
    try {
      final FileSystem fs = getSystemUserFS();
      final DatasetConfig datasetConfig = getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath);

      if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) {
        throw new IllegalArgumentException(format("Table %s does not belong to home space", datasetPath.toString()));
      }

      final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(formatPluginConfig);

      return getDataset(datasetPath, oldConfig, formatPlugin, fs, datasetConfig.getPhysicalDataset().getFormatSettings(),
          retrievalOptions.maxMetadataLeafColumns(), retrievalOptions.getTimeTravelRequest());
    } catch (NamespaceNotFoundException nfe){
      if(formatPluginConfig == null) {
        // a home file can only be read from the namespace or using a format options. Without either, it is invalid, return nothing.
        return null;
      }

      // check that the user owns the home path
      final HomeName userHomePath = HomeName.getUserHomePath(user);
      final String stagingHome = PathUtils.toDottedPath(Path.mergePaths(Path.of(HOME_PLUGIN_NAME), this.stagingDir));

      final Path path = PathUtils.toFSPath(datasetPath.getPathComponents());
      if (PathUtils.toDottedPath(path).startsWith(stagingHome)) {
        final String userStaging = PathUtils.toDottedPath(Path.mergePaths(Path.of(HOME_PLUGIN_NAME), Path.mergePaths(this.stagingDir, Path.of(userHomePath.getName()))));

        if (!PathUtils.toDottedPath(path).startsWith(userStaging)) {
          return null;
        }
      }

      return super.getDatasetWithFormat(new NamespaceKey(relativePath(datasetPath.getPathComponents(), getConfig().getPath())), oldConfig, formatPluginConfig, retrievalOptions, SystemUser.SYSTEM_USERNAME);
    }
  }

  @Override
  public boolean containerExists(EntityPath key) {
    List<String> folderPath = key.getComponents();
    try {
    return getSystemUserFS().exists(PathUtils.toFSPath(
      ImmutableList.<String>builder()
        .addAll(folderPath.subList(1, folderPath.size()))
        .build()));
    }catch(IOException e) {
      logger.info("IOException while trying to retrieve home files.", e);
      return false;
    }
  }

  private FileDatasetHandle getDataset(
      NamespaceKey datasetPath,
      PreviousDatasetInfo oldConfig,
      FormatPlugin formatPlugin,
      FileSystem fs,
      FileConfig fileConfig,
      int maxLeafColumns,
      TimeTravelOption.TimeTravelRequest travelRequest
  ) throws IOException {

    final FileUpdateKey.Builder updateKey = FileUpdateKey.newBuilder();
    final FileAttributes rootAttributes = fs.getFileAttributes(Path.of(fileConfig.getLocation()));
    final Path combined = Path.of("/").resolve(PathUtils.removeLeadingSlash(fileConfig.getLocation()));
    final FileSelection fileSelection = FileSelection.create(fs, combined);

    if (fileSelection == null) {
      return null;
    }

    // first entity is always a root
    if (rootAttributes.isDirectory()) {
      updateKey.addCachedEntities(fromFileAttributes(rootAttributes));
    }

    for (FileAttributes dirAttributes: fileSelection.getAllDirectories()) {
      updateKey.addCachedEntities(fromFileAttributes(dirAttributes));
    }

    if(updateKey.getCachedEntitiesCount() == 0){
      // this is a single file.
      updateKey.addCachedEntities(fromFileAttributes(rootAttributes));
    }
    final boolean hasDirectories = fileSelection.containsDirectories();
    // Expand selection by copying it first used to check extensions of files in directory.
    final FileSelection fileSelectionWithoutDir =  hasDirectories? fileSelection.minusDirectories(): fileSelection;

    if(fileSelectionWithoutDir == null){
      // no files in the found directory, not a table.
      return null;
    }

    FileDatasetHandle.checkMaxFiles(datasetPath.getName(), fileSelectionWithoutDir.getFileAttributesList().size(), getContext(),
      getConfig().isInternal());
    return formatPlugin.getDatasetAccessor(DatasetType.PHYSICAL_DATASET_HOME_FILE, oldConfig, fs,
        fileSelectionWithoutDir, this, datasetPath, updateKey.build(), maxLeafColumns, travelRequest);
  }

  protected FileProtobuf.FileSystemCachedEntity fromFileAttributes(FileAttributes attributes) {
    return FileProtobuf.FileSystemCachedEntity.newBuilder()
            .setPath(attributes.getPath().toString())
            .setLastModificationTime(attributes.lastModifiedTime().toMillis())
            .build();
  }

  private static List<String> relativePath(List<String> tableSchemaPath, Path rootPath) {
    List<String> rootPathComponents = PathUtils.toPathComponents(rootPath);
    List<String> tablePathComponents = PathUtils.toPathComponents(PathUtils.toFSPathSkipRoot(tableSchemaPath, null));
    return tablePathComponents.subList(rootPathComponents.size(), tablePathComponents.size());
  }

  @Override
  protected boolean ctasToUseIceberg() {
    return false;
  }
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<!-- Generated by org.testng.reporters.JUnitReportReporter -->
<testsuite hostname="NJCreationsLab" name="OpdAttachmentTestCase" tests="4" failures="0" timestamp="27 Apr 2017 08:38:51 GMT" time="0.573" errors="0">
  <testcase name="getOpdTestRequestByPID" time="0.091" classname="OpdAttachmentTestCase"/>
  <testcase name="getOpdTestRequestByAttachID" time="0.216" classname="OpdAttachmentTestCase"/>
  <testcase name="updateOpdAttachmentTestCase" time="0.178" classname="OpdAttachmentTestCase"/>
  <testcase name="addOpdAttachmentTestCase" time="0.088" classname="OpdAttachmentTestCase"/>
</testsuite> <!-- OpdAttachmentTestCase -->


text:
.class public Lcom/android/settings/enterprise/ImePreferenceController;
.super Lcom/android/settings/core/DynamicAvailabilityPreferenceController;
.source "ImePreferenceController.java"

# instance fields
.field private final mFeatureProvider:Lcom/android/settings/enterprise/EnterprisePrivacyFeatureProvider;

# direct methods
.method public constructor <init>(Landroid/content/Context;Lcom/android/settings/core/lifecycle/Lifecycle;)V
    .locals 1

    invoke-direct {p0, p1, p2}, Lcom/android/settings/core/DynamicAvailabilityPreferenceController;-><init>(Landroid/content/Context;Lcom/android/settings/core/lifecycle/Lifecycle;)V

    invoke-static {p1}, Lcom/android/settings/overlay/FeatureFactory;->getFactory(Landroid/content/Context;)Lcom/android/settings/overlay/FeatureFactory;

    move-result-object v0

    invoke-virtual {v0, p1}, Lcom/android/settings/overlay/FeatureFactory;->getEnterprisePrivacyFeatureProvider(Landroid/content/Context;)Lcom/android/settings/enterprise/EnterprisePrivacyFeatureProvider;

    move-result-object v0

    iput-object v0, p0, Lcom/android/settings/enterprise/ImePreferenceController;->mFeatureProvider:Lcom/android/settings/enterprise/EnterprisePrivacyFeatureProvider;

    return-void
.end method

# virtual methods
.method public getPreferenceKey()Ljava/lang/String;
    .locals 1

    const-string/jumbo v0, "input_method"

    return-object v0
.end method

.method public isAvailable()Z
    .locals 1

    iget-object v0, p0, Lcom/android/settings/enterprise/ImePreferenceController;->mFeatureProvider:Lcom/android/settings/enterprise/EnterprisePrivacyFeatureProvider;

    invoke-interface {v0}, Lcom/android/settings/enterprise/EnterprisePrivacyFeatureProvider;->getImeLabelIfOwnerSet()Ljava/lang/String;

    move-result-object v0

    if-eqz v0, :cond_0

    const/4 v0, 0x1

    :goto_0
    return v0

    :cond_0
    const/4 v0, 0x0

    goto :goto_0
.end method

.method public updateState(Landroid/support/v7/preference/Preference;)V
    .locals 4

    iget-object v0, p0, Lcom/android/settings/enterprise/ImePreferenceController;->mContext:Landroid/content/Context;

    invoke-virtual {v0}, Landroid/content/Context;->getResources()Landroid/content/res/Resources;

    move-result-object v0

    const/4 v1, 0x1

    new-array v1, v1, [Ljava/lang/Object;

    iget-object v2, p0, Lcom/android/settings/enterprise/ImePreferenceController;->mFeatureProvider:Lcom/android/settings/enterprise/EnterprisePrivacyFeatureProvider;

    invoke-interface {v2}, Lcom/android/settings/enterprise/EnterprisePrivacyFeatureProvider;->getImeLabelIfOwnerSet()Ljava/lang/String;

    move-result-object v2

    const/4 v3, 0x0

    aput-object v2, v1, v3

    const v2, 0x7f12097e

    invoke-virtual {v0, v2, v1}, Landroid/content/res/Resources;->getString(I[Ljava/lang/Object;)Ljava/lang/String;

    move-result-object v0

    invoke-virtual {p1, v0}, Landroid/support/v7/preference/Preference;->setSummary(Ljava/lang/CharSequence;)V

    return-void
.end method


text:
title: "generateInternalStream(options)"
layout: default
section: api
---

__Description__ : Generates the complete zip file with the internal stream
implementation.

__Arguments__

name                | type     | default | description
--------------------|----------|---------|------------
options             | object   |         | the options to generate the zip file, see [the options of `generateAsync()`]({{site.baseurl}}/documentation/api_jszip/generate_async.html)

__Metadata__ : see [the metadata of `generateAsync()`]({{site.baseurl}}/documentation/api_jszip/generate_async.html).

__Returns__ : a [StreamHelper]({{site.baseurl}}/documentation/api_streamhelper.html).

__Throws__ : Nothing.

__Example__

```js
zip.generateInternalStream({type:"blob"}).accumulate(function callback(err, content) {
  if (err) {
    // handle error
  }
  // see FileSaver.js
  saveAs(content, "hello.zip");
}, function updateCallback(metadata) {
  // print progression with metadata.percent and metadata.currentFile
});
```


text:
'use strict';

var express = require('express');
var router  = express.Router();

var users = require('./user.controller');
var games = require('./game.controller');

router.use('/users', users);
router.use('/games', games);

module.exports = router;


text:
<!DOCTYPE html>

  <link href="http://gmpg.org/xfn/11" rel="profile">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta http-equiv="content-type" content="text/html; charset=utf-8">

  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1">

  <title>

      Bitmap Image · Sunny

  </title>

  <link rel="stylesheet" href="/public/css/poole.css">
  <link rel="stylesheet" href="/public/css/syntax.css">
  <link rel="stylesheet" href="/public/css/hyde.css">
  <link rel="stylesheet" href="http://fonts.googleapis.com/css?family=PT+Sans:400,400italic,700|Abril+Fatface">

  <link rel="apple-touch-icon-precomposed" sizes="144x144" href="/public/apple-touch-icon-144-precomposed.png">
                                 <link rel="shortcut icon" href="/public/favicon.ico">

  <link rel="alternate" type="application/rss+xml" title="RSS" href="/atom.xml">

          Sunny

This is my personal blog Welcome !@sunny.

      Home

            About

            Archives

      GitHub project
      Email

© 2015. All rights reserved.

Bitmap Image

  19 Mar 2014

Bitmap Image

位图图像 Bitmap Image 由像素组成,每一个像素都相当于图像中的一个点,JPEG,TIFF,PNG 类型的文件都是位图图像。在位图图像中每个像素都被分配一个特定位置和颜色值。在处理位图图像时,我们编辑的是像素而不是对象或形状,也就是说,编辑的是每一个点。位图图像被限制成正方形,但是我们通过填充透明像素来使图片呈现其他的形状包括旋转裁切等效果。

Quartz 支持很多流行的类型的图片格式,在iOS上包括 JPEG, GIF, PNG, TIF, ICO, GMP, XBM, and CUR。其他的图片类型 我们可以通过将格式信息传递给Quartz保证图片被正确的解析。

通常情况下我们在程序中大多使用 UIImagePNGRepresentation() 来将图片转换成二进制数据。但是这样的话实际上在转换的过程中被压缩了。那么我们可以通过 Quartz 来获得原始的图片数据。
首先我们要先把图片绘制到context上 然后使用 CGBitmapContextGetData()获取二进制数据。

 1 -(NSData *)bytesFromRGBImage:(UIImage *)sourceImage {
 2  if (!sourceImage) {
 3      return nil;
 4  }
 5
 6  CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();//创建设备的colorSpace
 7
 8  if (colorSpace == NULL) {
 9      return nil;
10  }
11
12  int width = sourceImage.size.width;//获取的像素点的个数
13  int height = sourceImage.size.height;//获取的像素点的个数
14  //8 是每一像素点的大小 8个bit(0-255)的空间
15  // width * 4 每一行的像素点占用的字节数,每个像素点的ARGB四个通道各占8个bit(0-255)的空间
16
17  CGContextRef context = CGBitmapContextCreate(NULL,width, height, 8, width *4, colorSpace, (CGBitmapInfo)kCGImageAlphaPremultipliedFirst);
18
19  CGColorSpaceRelease(colorSpace);
20  if (context == NULL) {
21      return nil;
22  }
23
24  CGRect rect = (CGRect){.size = sourceImage.size};
25  CGContextDrawImage(context, rect, sourceImage.CGImage);
26  NSData *data = [NSData dataWithBytes:CGBitmapContextGetData(context) length:width*height*4];
27  CGContextRelease(context);
28  return data;
29 }

同样我们还可以通过二进制数据获得图片对象

 1 -(UIImage *)imageFromBytes:(NSData *)data size:(CGSize)targetSize {
 2  int width = targetSize.width;
 3  int height = targetSize.height;
 4  if (data.length < width *height *4) {
 5      return nil;
 6  }
 7
 8  CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
 9  if (colorSpace == NULL) {
10      return nil;
11  }
12  Byte *bytes = (Byte *)data.bytes;
13  CGContextRef context = CGBitmapContextCreate(bytes, width, height, 8, width*4, colorSpace, (CGBitmapInfo) kCGImageAlphaPremultipliedFirst);
14  CGColorSpaceRelease(colorSpace);
15  if (context == NULL) {
16      return nil;
17  }
18
19  CGImageRef imageRef = CGBitmapContextCreateImage(context);
20  UIImage *image = [UIImage imageWithCGImage:imageRef];
21  CGContextRelease(context);
22  return image;
23 }

Alpha通道

Alpha通道增加像素的透明度信息。

色彩深度

色彩深度又叫色彩位数,即位图中要用多少个二进制位来表示每个点的颜色,是分辨率的一个重要指标。常用有1位(单色),2位(4色,CGA),4位(16色,VGA),8位(256色),16位(增强色),24位和32位(真彩色)等。色深16位以上的位图还可以根据其中分别表示RGB三原色或CMYK四原色(有的还包括Alpha通道)的位数进一步分类.

图像遮罩(image mask)

image mask 也是由一組像素组成的特殊的位图,能指定图片的绘制区域,常见就通过image mask来绘制各种各样的形状,在iOS实际开发中用到的最多的可能就是圆角矩形了。 说白了就是就是我们通过image mask想让图片怎么显示,就怎么显示。

Related Posts

            Swift
            24 Jan 2015

            iOS应用中实现图片滤镜的一个简单方法
            23 Jan 2015

            Core Image 旧胶片效果
            30 Mar 2014


text:
"""Defines functionality for pipelined execution of interfaces

The `Workflow` class provides core functionality for batch processing.

  .. testsetup::
     # Change directory to provide relative paths for doctests
     import os
     filepath = os.path.dirname(os.path.realpath( __file__ ))
     datadir = os.path.realpath(os.path.join(filepath, '../../testing/data'))
     os.chdir(datadir)

"""

from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import object

from datetime import datetime
from nipype.utils.misc import flatten, unflatten
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

from copy import deepcopy
import pickle
from glob import glob
import gzip
import inspect
import os
import os.path as op
import re
import shutil
import errno
import socket
from shutil import rmtree
import sys
from tempfile import mkdtemp
from warnings import warn
from hashlib import sha1

import numpy as np
import networkx as nx

from ...utils.misc import package_check, str2bool
package_check('networkx', '1.3')

from ... import config, logging
logger = logging.getLogger('workflow')
from ...interfaces.base import (traits, InputMultiPath, CommandLine,
                                Undefined, TraitedSpec, DynamicTraitedSpec,
                                Bunch, InterfaceResult, md5, Interface,
                                TraitDictObject, TraitListObject, isdefined)
from ...utils.misc import (getsource, create_function_from_source,
                           flatten, unflatten)
from ...utils.filemanip import (save_json, FileNotFoundError,
                                filename_to_list, list_to_filename,
                                copyfiles, fnames_presuffix, loadpkl,
                                split_filename, load_json, savepkl,
                                write_rst_header, write_rst_dict,
                                write_rst_list)
from ...external.six import string_types
from .utils import (generate_expanded_graph, modify_paths,
                    export_graph, make_output_dir, write_workflow_prov,
                    clean_working_directory, format_dot, topological_sort,
                    get_print_name, merge_dict, evaluate_connect_function,
                    _write_inputs, format_node)

from .base import EngineBase
from .nodes import Node, MapNode

class Workflow(EngineBase):
    """Controls the setup and execution of a pipeline of processes."""

    def __init__(self, name, base_dir=None):
        """Create a workflow object.

        Parameters
        ----------
        name : alphanumeric string
            unique identifier for the workflow
        base_dir : string, optional
            path to workflow storage

        """
        super(Workflow, self).__init__(name, base_dir)
        self._graph = nx.DiGraph()
        self.config = deepcopy(config._sections)

    # PUBLIC API
    def clone(self, name):
        """Clone a workflow

        .. note::

          Will reset attributes used for executing workflow. See
          _init_runtime_fields.

        Parameters
        ----------

        name: alphanumeric name
            unique name for the workflow

        """
        clone = super(Workflow, self).clone(name)
        clone._reset_hierarchy()
        return clone

    # Graph creation functions
    def connect(self, *args, **kwargs):
        """Connect nodes in the pipeline.

        This routine also checks if inputs and outputs are actually provided by
        the nodes that are being connected.

        Creates edges in the directed graph using the nodes and edges specified
        in the `connection_list`.  Uses the NetworkX method
        DiGraph.add_edges_from.

        Parameters
        ----------

        args : list or a set of four positional arguments

            Four positional arguments of the form::

              connect(source, sourceoutput, dest, destinput)

            source : nodewrapper node
            sourceoutput : string (must be in source.outputs)
            dest : nodewrapper node
            destinput : string (must be in dest.inputs)

            A list of 3-tuples of the following form::

             [(source, target,
                 [('sourceoutput/attribute', 'targetinput'),
                 ...]),
             ...]

            Or::

             [(source, target, [(('sourceoutput1', func, arg2, ...),
                                         'targetinput'), ...]),
             ...]
             sourceoutput1 will always be the first argument to func
             and func will be evaluated and the results sent ot targetinput

             currently func needs to define all its needed imports within the
             function as we use the inspect module to get at the source code
             and execute it remotely
        """
        if len(args) == 1:
            connection_list = args[0]
        elif len(args) == 4:
            connection_list = [(args[0], args[2], [(args[1], args[3])])]
        else:
            raise TypeError('connect() takes either 4 arguments, or 1 list of'
                            ' connection tuples (%d args given)' % len(args))

        disconnect = False
        if kwargs:
            disconnect = kwargs.get('disconnect', False)

        if disconnect:
            self.disconnect(connection_list)
            return

        newnodes = []
        for srcnode, destnode, _ in connection_list:
            if self in [srcnode, destnode]:
                msg = ('Workflow connect cannot contain itself as node:'
                       ' src[%s] dest[%s] workflow[%s]') % (srcnode,
                                                            destnode,
                                                            self.name)

                raise IOError(msg)
            if (srcnode not in newnodes) and not self._has_node(srcnode):
                newnodes.append(srcnode)
            if (destnode not in newnodes) and not self._has_node(destnode):
                newnodes.append(destnode)
        if newnodes:
            self._check_nodes(newnodes)
            for node in newnodes:
                if node._hierarchy is None:
                    node._hierarchy = self.name
        not_found = []
        connected_ports = {}
        for srcnode, destnode, connects in connection_list:
            if destnode not in connected_ports:
                connected_ports[destnode] = []
            # check to see which ports of destnode are already
            # connected.
            if not disconnect and (destnode in self._graph.nodes()):
                for edge in self._graph.in_edges_iter(destnode):
                    data = self._graph.get_edge_data(*edge)
                    for sourceinfo, destname in data['connect']:
                        if destname not in connected_ports[destnode]:
                            connected_ports[destnode] += [destname]
            for source, dest in connects:
                # Currently datasource/sink/grabber.io modules
                # determine their inputs/outputs depending on
                # connection settings.  Skip these modules in the check
                if dest in connected_ports[destnode]:
                    raise Exception("""
Trying to connect %s:%s to %s:%s but input '%s' of node '%s' is already
connected.
""" % (srcnode, source, destnode, dest, dest, destnode))
                if not (hasattr(destnode, '_interface') and
                        '.io' in str(destnode._interface.__class__)):
                    if not destnode._check_inputs(dest):
                        not_found.append(['in', destnode.name, dest])
                if not (hasattr(srcnode, '_interface') and
                        '.io' in str(srcnode._interface.__class__)):
                    if isinstance(source, tuple):
                        # handles the case that source is specified
                        # with a function
                        sourcename = source[0]
                    elif isinstance(source, string_types):
                        sourcename = source
                    else:
                        raise Exception(('Unknown source specification in '
                                         'connection from output of %s') %
                                        srcnode.name)
                    if sourcename and not srcnode._check_outputs(sourcename):
                        not_found.append(['out', srcnode.name, sourcename])
                connected_ports[destnode] += [dest]
        infostr = []
        for info in not_found:
            infostr += ["Module %s has no %sput called %s\n" % (info[1],
                                                                info[0],
                                                                info[2])]
        if not_found:
            raise Exception('\n'.join(['Some connections were not found'] +
                                      infostr))

        # turn functions into strings
        for srcnode, destnode, connects in connection_list:
            for idx, (src, dest) in enumerate(connects):
                if isinstance(src, tuple) and not isinstance(src[1], string_types):
                    function_source = getsource(src[1])
                    connects[idx] = ((src[0], function_source, src[2:]), dest)

        # add connections
        for srcnode, destnode, connects in connection_list:
            edge_data = self._graph.get_edge_data(srcnode, destnode, None)
            if edge_data:
                logger.debug('(%s, %s): Edge data exists: %s'
                             % (srcnode, destnode, str(edge_data)))
                for data in connects:
                    if data not in edge_data['connect']:
                        edge_data['connect'].append(data)
                    if disconnect:
                        logger.debug('Removing connection: %s' % str(data))
                        edge_data['connect'].remove(data)
                if edge_data['connect']:
                    self._graph.add_edges_from([(srcnode,
                                                 destnode,
                                                 edge_data)])
                else:
                    # pass
                    logger.debug('Removing connection: %s->%s' % (srcnode,
                                                                  destnode))
                    self._graph.remove_edges_from([(srcnode, destnode)])
            elif not disconnect:
                logger.debug('(%s, %s): No edge data' % (srcnode, destnode))
                self._graph.add_edges_from([(srcnode, destnode,
                                             {'connect': connects})])
            edge_data = self._graph.get_edge_data(srcnode, destnode)
            logger.debug('(%s, %s): new edge data: %s' % (srcnode, destnode,
                                                          str(edge_data)))

    def disconnect(self, *args):
        """Disconnect nodes
        See the docstring for connect for format.
        """
        if len(args) == 1:
            connection_list = args[0]
        elif len(args) == 4:
            connection_list = [(args[0], args[2], [(args[1], args[3])])]
        else:
            raise TypeError('disconnect() takes either 4 arguments, or 1 list '
                            'of connection tuples (%d args given)' % len(args))

        for srcnode, dstnode, conn in connection_list:
            logger.debug('disconnect(): %s->%s %s' % (srcnode, dstnode, conn))
            if self in [srcnode, dstnode]:
                raise IOError(
                    'Workflow connect cannot contain itself as node: src[%s] '
                    'dest[%s] workflow[%s]') % (srcnode, dstnode, self.name)

            # If node is not in the graph, not connected
            if not self._has_node(srcnode) or not self._has_node(dstnode):
                continue

            edge_data = self._graph.get_edge_data(
                srcnode, dstnode, {'connect': []})
            ed_conns = [(c[0], c[1]) for c in edge_data['connect']]

            remove = []
            for edge in conn:
                if edge in ed_conns:
                    idx = ed_conns.index(edge)
                    remove.append((edge[0], edge[1]))

            logger.debug('disconnect(): remove list %s' % remove)
            for el in remove:
                edge_data['connect'].remove(el)
                logger.debug('disconnect(): removed connection %s' % str(el))

            if not edge_data['connect']:
                self._graph.remove_edge(srcnode, dstnode)
            else:
                self._graph.add_edges_from([(srcnode, dstnode, edge_data)])

    def add_nodes(self, nodes):
        """ Add nodes to a workflow

        Parameters
        ----------
        nodes : list
            A list of EngineBase-based objects
        """
        newnodes = []
        all_nodes = self._get_all_nodes()
        for node in nodes:
            if self._has_node(node):
                raise IOError('Node %s already exists in the workflow' % node)
            if isinstance(node, Workflow):
                for subnode in node._get_all_nodes():
                    if subnode in all_nodes:
                        raise IOError(('Subnode %s of node %s already exists '
                                       'in the workflow') % (subnode, node))
            newnodes.append(node)
        if not newnodes:
            logger.debug('no new nodes to add')
            return
        for node in newnodes:
            if not issubclass(node.__class__, EngineBase):
                raise Exception('Node %s must be a subclass of EngineBase' %
                                str(node))
        self._check_nodes(newnodes)
        for node in newnodes:
            if node._hierarchy is None:
                node._hierarchy = self.name
        self._graph.add_nodes_from(newnodes)

    def remove_nodes(self, nodes):
        """ Remove nodes from a workflow

        Parameters
        ----------
        nodes : list
            A list of EngineBase-based objects
        """
        self._graph.remove_nodes_from(nodes)

    # Input-Output access
    @property
    def inputs(self):
        return self._get_inputs()

    @property
    def outputs(self):
        return self._get_outputs()

    def get_node(self, name):
        """Return an internal node by name
        """
        nodenames = name.split('.')
        nodename = nodenames[0]
        outnode = [node for node in self._graph.nodes() if
                   str(node).endswith('.' + nodename)]
        if outnode:
            outnode = outnode[0]
            if nodenames[1:] and issubclass(outnode.__class__, Workflow):
                outnode = outnode.get_node('.'.join(nodenames[1:]))
        else:
            outnode = None
        return outnode

    def list_node_names(self):
        """List names of all nodes in a workflow
        """
        outlist = []
        for node in nx.topological_sort(self._graph):
            if isinstance(node, Workflow):
                outlist.extend(['.'.join((node.name, nodename)) for nodename in
                                node.list_node_names()])
            else:
                outlist.append(node.name)
        return sorted(outlist)

    def write_graph(self, dotfilename='graph.dot', graph2use='hierarchical',
                    format="png", simple_form=True):
        """Generates a graphviz dot file and a png file

        Parameters
        ----------

        graph2use: 'orig', 'hierarchical' (default), 'flat', 'exec', 'colored'
            orig - creates a top level graph without expanding internal
            workflow nodes;
            flat - expands workflow nodes recursively;
            hierarchical - expands workflow nodes recursively with a
            notion on hierarchy;
            colored - expands workflow nodes recursively with a
            notion on hierarchy in color;
            exec - expands workflows to depict iterables

        format: 'png', 'svg'

        simple_form: boolean (default: True)
            Determines if the node name used in the graph should be of the form
            'nodename (package)' when True or 'nodename.Class.package' when
            False.

        """
        graphtypes = ['orig', 'flat', 'hierarchical', 'exec', 'colored']
        if graph2use not in graphtypes:
            raise ValueError('Unknown graph2use keyword. Must be one of: ' +
                             str(graphtypes))
        base_dir, dotfilename = op.split(dotfilename)
        if base_dir == '':
            if self.base_dir:
                base_dir = self.base_dir
                if self.name:
                    base_dir = op.join(base_dir, self.name)
            else:
                base_dir = os.getcwd()
        base_dir = make_output_dir(base_dir)
        if graph2use in ['hierarchical', 'colored']:
            dotfilename = op.join(base_dir, dotfilename)
            self.write_hierarchical_dotfile(dotfilename=dotfilename,
                                            colored=graph2use == "colored",
                                            simple_form=simple_form)
            format_dot(dotfilename, format=format)
        else:
            graph = self._graph
            if graph2use in ['flat', 'exec']:
                graph = self._create_flat_graph()
            if graph2use == 'exec':
                graph = generate_expanded_graph(deepcopy(graph))
            export_graph(graph, base_dir, dotfilename=dotfilename,
                         format=format, simple_form=simple_form)

    def write_hierarchical_dotfile(self, dotfilename=None, colored=False,
                                   simple_form=True):
        dotlist = ['digraph %s{' % self.name]
        dotlist.append(self._get_dot(prefix='  ', colored=colored,
                                     simple_form=simple_form))
        dotlist.append('}')
        dotstr = '\n'.join(dotlist)
        if dotfilename:
            fp = open(dotfilename, 'wt')
            fp.writelines(dotstr)
            fp.close()
        else:
            logger.info(dotstr)

    def export(self, filename=None, prefix="output", format="python",
               include_config=False):
        """Export object into a different format

        Parameters
        ----------
        filename: string
           file to save the code to; overrides prefix
        prefix: string
           prefix to use for output file
        format: string
           one of "python"
        include_config: boolean
           whether to include node and workflow config values

        """
        formats = ["python"]
        if format not in formats:
            raise ValueError('format must be one of: %s' % '|'.join(formats))
        flatgraph = self._create_flat_graph()
        nodes = nx.topological_sort(flatgraph)

        lines = ['# Workflow']
        importlines = ['from nipype.pipeline.engine import Workflow, '
                       'Node, MapNode']
        functions = {}
        if format == "python":
            connect_template = '%s.connect(%%s, %%s, %%s, "%%s")' % self.name
            connect_template2 = '%s.connect(%%s, "%%s", %%s, "%%s")' \
                                % self.name
            wfdef = '%s = Workflow("%s")' % (self.name, self.name)
            lines.append(wfdef)
            if include_config:
                lines.append('%s.config = %s' % (self.name, self.config))
            for idx, node in enumerate(nodes):
                nodename = node.fullname.replace('.', '_')
                # write nodes
                nodelines = format_node(node, format='python',
                                        include_config=include_config)
                for line in nodelines:
                    if line.startswith('from'):
                        if line not in importlines:
                            importlines.append(line)
                    else:
                        lines.append(line)
                # write connections
                for u, _, d in flatgraph.in_edges_iter(nbunch=node,
                                                       data=True):
                    for cd in d['connect']:
                        if isinstance(cd[0], tuple):
                            args = list(cd[0])
                            if args[1] in functions:
                                funcname = functions[args[1]]
                            else:
                                func = create_function_from_source(args[1])
                                funcname = [name for name in func.__globals__
                                            if name != '__builtins__'][0]
                                functions[args[1]] = funcname
                            args[1] = funcname
                            args = tuple([arg for arg in args if arg])
                            line_args = (u.fullname.replace('.', '_'),
                                         args, nodename, cd[1])
                            line = connect_template % line_args
                            line = line.replace("'%s'" % funcname, funcname)
                            lines.append(line)
                        else:
                            line_args = (u.fullname.replace('.', '_'),
                                         cd[0], nodename, cd[1])
                            lines.append(connect_template2 % line_args)
            functionlines = ['# Functions']
            for function in functions:
                functionlines.append(pickle.loads(function).rstrip())
            all_lines = importlines + functionlines + lines

            if not filename:
                filename = '%s%s.py' % (prefix, self.name)
            with open(filename, 'wt') as fp:
                fp.writelines('\n'.join(all_lines))
        return all_lines

    def run(self, plugin=None, plugin_args=None, updatehash=False):
        """ Execute the workflow

        Parameters
        ----------

        plugin: plugin name or object
            Plugin to use for execution. You can create your own plugins for
            execution.
        plugin_args : dictionary containing arguments to be sent to plugin
            constructor. see individual plugin doc strings for details.
        """
        if plugin is None:
            plugin = config.get('execution', 'plugin')
        if not isinstance(plugin, string_types):
            runner = plugin
        else:
            name = 'nipype.pipeline.plugins'
            try:
                __import__(name)
            except ImportError:
                msg = 'Could not import plugin module: %s' % name
                logger.error(msg)
                raise ImportError(msg)
            else:
                plugin_mod = getattr(sys.modules[name], '%sPlugin' % plugin)
                runner = plugin_mod(plugin_args=plugin_args)
        flatgraph = self._create_flat_graph()
        self.config = merge_dict(deepcopy(config._sections), self.config)
        if 'crashdump_dir' in self.config:
            warn(("Deprecated: workflow.config['crashdump_dir']\n"
                  "Please use config['execution']['crashdump_dir']"))
            crash_dir = self.config['crashdump_dir']
            self.config['execution']['crashdump_dir'] = crash_dir
            del self.config['crashdump_dir']
        logger.info(str(sorted(self.config)))
        self._set_needed_outputs(flatgraph)
        execgraph = generate_expanded_graph(deepcopy(flatgraph))
        for index, node in enumerate(execgraph.nodes()):
            node.config = merge_dict(deepcopy(self.config), node.config)
            node.base_dir = self.base_dir
            node.index = index
            if isinstance(node, MapNode):
                node.use_plugin = (plugin, plugin_args)
        self._configure_exec_nodes(execgraph)
        if str2bool(self.config['execution']['create_report']):
            self._write_report_info(self.base_dir, self.name, execgraph)
        runner.run(execgraph, updatehash=updatehash, config=self.config)
        datestr = datetime.utcnow().strftime('%Y%m%dT%H%M%S')
        if str2bool(self.config['execution']['write_provenance']):
            prov_base = op.join(self.base_dir,
                                'workflow_provenance_%s' % datestr)
            logger.info('Provenance file prefix: %s' % prov_base)
            write_workflow_prov(execgraph, prov_base, format='all')
        return execgraph

    # PRIVATE API AND FUNCTIONS

    def _write_report_info(self, workingdir, name, graph):
        if workingdir is None:
            workingdir = os.getcwd()
        report_dir = op.join(workingdir, name)
        if not op.exists(report_dir):
            os.makedirs(report_dir)
        shutil.copyfile(op.join(op.dirname(__file__),
                                'report_template.html'),
                        op.join(report_dir, 'index.html'))
        shutil.copyfile(op.join(op.dirname(__file__),
                                '..', '..', 'external', 'd3.js'),
                        op.join(report_dir, 'd3.js'))
        nodes, groups = topological_sort(graph, depth_first=True)
        graph_file = op.join(report_dir, 'graph1.json')
        json_dict = {'nodes': [], 'links': [], 'groups': [], 'maxN': 0}
        for i, node in enumerate(nodes):
            report_file = "%s/_report/report.rst" % \
                          node.output_dir().replace(report_dir, '')
            result_file = "%s/result_%s.pklz" % \
                          (node.output_dir().replace(report_dir, ''),
                           node.name)
            json_dict['nodes'].append(dict(name='%d_%s' % (i, node.name),
                                           report=report_file,
                                           result=result_file,
                                           group=groups[i]))
        maxN = 0
        for gid in np.unique(groups):
            procs = [i for i, val in enumerate(groups) if val == gid]
            N = len(procs)
            if N > maxN:
                maxN = N
            json_dict['groups'].append(dict(procs=procs,
                                            total=N,
                                            name='Group_%05d' % gid))
        json_dict['maxN'] = maxN
        for u, v in graph.in_edges_iter():
            json_dict['links'].append(dict(source=nodes.index(u),
                                           target=nodes.index(v),
                                           value=1))
        save_json(graph_file, json_dict)
        graph_file = op.join(report_dir, 'graph.json')
        template = '%%0%dd_' % np.ceil(np.log10(len(nodes))).astype(int)

        def getname(u, i):
            name_parts = u.fullname.split('.')
            # return '.'.join(name_parts[:-1] + [template % i + name_parts[-1]])
            return template % i + name_parts[-1]
        json_dict = []
        for i, node in enumerate(nodes):
            imports = []
            for u, v in graph.in_edges_iter(nbunch=node):
                imports.append(getname(u, nodes.index(u)))
            json_dict.append(dict(name=getname(node, i),
                                  size=1,
                                  group=groups[i],
                                  imports=imports))
        save_json(graph_file, json_dict)

    def _set_needed_outputs(self, graph):
        """Initialize node with list of which outputs are needed."""
        rm_outputs = self.config['execution']['remove_unnecessary_outputs']
        if not str2bool(rm_outputs):
            return
        for node in graph.nodes():
            node.needed_outputs = []
            for edge in graph.out_edges_iter(node):
                data = graph.get_edge_data(*edge)
                sourceinfo = [v1[0] if isinstance(v1, tuple) else v1
                              for v1, v2 in data['connect']]
                node.needed_outputs += [v for v in sourceinfo
                                        if v not in node.needed_outputs]
            if node.needed_outputs:
                node.needed_outputs = sorted(node.needed_outputs)

    def _configure_exec_nodes(self, graph):
        """Ensure that each node knows where to get inputs from
        """
        for node in graph.nodes():
            node.input_source = {}
            for edge in graph.in_edges_iter(node):
                data = graph.get_edge_data(*edge)
                for sourceinfo, field in sorted(data['connect']):
                    node.input_source[field] = \
                        (op.join(edge[0].output_dir(),
                                 'result_%s.pklz' % edge[0].name),
                         sourceinfo)

    def _check_nodes(self, nodes):
        """Checks if any of the nodes are already in the graph

        """
        node_names = [node.name for node in self._graph.nodes()]
        node_lineage = [node._hierarchy for node in self._graph.nodes()]
        for node in nodes:
            if node.name in node_names:
                idx = node_names.index(node.name)
                if node_lineage[idx] in [node._hierarchy, self.name]:
                    raise IOError('Duplicate node name %s found.' % node.name)
            else:
                node_names.append(node.name)

    def _has_attr(self, parameter, subtype='in'):
        """Checks if a parameter is available as an input or output
        """
        if subtype == 'in':
            subobject = self.inputs
        else:
            subobject = self.outputs
        attrlist = parameter.split('.')
        cur_out = subobject
        for attr in attrlist:
            if not hasattr(cur_out, attr):
                return False
            cur_out = getattr(cur_out, attr)
        return True

    def _get_parameter_node(self, parameter, subtype='in'):
        """Returns the underlying node corresponding to an input or
        output parameter
        """
        if subtype == 'in':
            subobject = self.inputs
        else:
            subobject = self.outputs
        attrlist = parameter.split('.')
        cur_out = subobject
        for attr in attrlist[:-1]:
            cur_out = getattr(cur_out, attr)
        return cur_out.traits()[attrlist[-1]].node

    def _check_outputs(self, parameter):
        return self._has_attr(parameter, subtype='out')

    def _check_inputs(self, parameter):
        return self._has_attr(parameter, subtype='in')

    def _get_inputs(self):
        """Returns the inputs of a workflow

        This function does not return any input ports that are already
        connected
        """
        inputdict = TraitedSpec()
        for node in self._graph.nodes():
            inputdict.add_trait(node.name, traits.Instance(TraitedSpec))
            if isinstance(node, Workflow):
                setattr(inputdict, node.name, node.inputs)
            else:
                taken_inputs = []
                for _, _, d in self._graph.in_edges_iter(nbunch=node,
                                                         data=True):
                    for cd in d['connect']:
                        taken_inputs.append(cd[1])
                unconnectedinputs = TraitedSpec()
                for key, trait in list(node.inputs.items()):
                    if key not in taken_inputs:
                        unconnectedinputs.add_trait(key,
                                                    traits.Trait(trait,
                                                                 node=node))
                        value = getattr(node.inputs, key)
                        setattr(unconnectedinputs, key, value)
                setattr(inputdict, node.name, unconnectedinputs)
                getattr(inputdict, node.name).on_trait_change(self._set_input)
        return inputdict

    def _get_outputs(self):
        """Returns all possible output ports that are not already connected
        """
        outputdict = TraitedSpec()
        for node in self._graph.nodes():
            outputdict.add_trait(node.name, traits.Instance(TraitedSpec))
            if isinstance(node, Workflow):
                setattr(outputdict, node.name, node.outputs)
            elif node.outputs:
                outputs = TraitedSpec()
                for key, _ in list(node.outputs.items()):
                    outputs.add_trait(key, traits.Any(node=node))
                    setattr(outputs, key, None)
                setattr(outputdict, node.name, outputs)
        return outputdict

    def _set_input(self, object, name, newvalue):
        """Trait callback function to update a node input
        """
        object.traits()[name].node.set_input(name, newvalue)

    def _set_node_input(self, node, param, source, sourceinfo):
        """Set inputs of a node given the edge connection"""
        if isinstance(sourceinfo, string_types):
            val = source.get_output(sourceinfo)
        elif isinstance(sourceinfo, tuple):
            if callable(sourceinfo[1]):
                val = sourceinfo[1](source.get_output(sourceinfo[0]),
                                    *sourceinfo[2:])
        newval = val
        if isinstance(val, TraitDictObject):
            newval = dict(val)
        if isinstance(val, TraitListObject):
            newval = val[:]
        logger.debug('setting node input: %s->%s', param, str(newval))
        node.set_input(param, deepcopy(newval))

    def _get_all_nodes(self):
        allnodes = []
        for node in self._graph.nodes():
            if isinstance(node, Workflow):
                allnodes.extend(node._get_all_nodes())
            else:
                allnodes.append(node)
        return allnodes

    def _has_node(self, wanted_node):
        for node in self._graph.nodes():
            if wanted_node == node:
                return True
            if isinstance(node, Workflow):
                if node._has_node(wanted_node):
                    return True
        return False

    def _create_flat_graph(self):
        """Make a simple DAG where no node is a workflow."""
        logger.debug('Creating flat graph for workflow: %s', self.name)
        workflowcopy = deepcopy(self)
        workflowcopy._generate_flatgraph()
        return workflowcopy._graph

    def _reset_hierarchy(self):
        """Reset the hierarchy on a graph
        """
        for node in self._graph.nodes():
            if isinstance(node, Workflow):
                node._reset_hierarchy()
                for innernode in node._graph.nodes():
                    innernode._hierarchy = '.'.join((self.name,
                                                     innernode._hierarchy))
            else:
                node._hierarchy = self.name

    def _generate_flatgraph(self):
        """Generate a graph containing only Nodes or MapNodes
        """
        logger.debug('expanding workflow: %s', self)
        nodes2remove = []
        if not nx.is_directed_acyclic_graph(self._graph):
            raise Exception(('Workflow: %s is not a directed acyclic graph '
                             '(DAG)') % self.name)
        nodes = nx.topological_sort(self._graph)
        for node in nodes:
            logger.debug('processing node: %s' % node)
            if isinstance(node, Workflow):
                nodes2remove.append(node)
                # use in_edges instead of in_edges_iter to allow
                # disconnections to take place properly. otherwise, the
                # edge dict is modified.
                for u, _, d in self._graph.in_edges(nbunch=node, data=True):
                    logger.debug('in: connections-> %s' % str(d['connect']))
                    for cd in deepcopy(d['connect']):
                        logger.debug("in: %s" % str(cd))
                        dstnode = node._get_parameter_node(cd[1], subtype='in')
                        srcnode = u
                        srcout = cd[0]
                        dstin = cd[1].split('.')[-1]
                        logger.debug('in edges: %s %s %s %s' %
                                     (srcnode, srcout, dstnode, dstin))
                        self.disconnect(u, cd[0], node, cd[1])
                        self.connect(srcnode, srcout, dstnode, dstin)
                # do not use out_edges_iter for reasons stated in in_edges
                for _, v, d in self._graph.out_edges(nbunch=node, data=True):
                    logger.debug('out: connections-> %s' % str(d['connect']))
                    for cd in deepcopy(d['connect']):
                        logger.debug("out: %s" % str(cd))
                        dstnode = v
                        if isinstance(cd[0], tuple):
                            parameter = cd[0][0]
                        else:
                            parameter = cd[0]
                        srcnode = node._get_parameter_node(parameter,
                                                           subtype='out')
                        if isinstance(cd[0], tuple):
                            srcout = list(cd[0])
                            srcout[0] = parameter.split('.')[-1]
                            srcout = tuple(srcout)
                        else:
                            srcout = parameter.split('.')[-1]
                        dstin = cd[1]
                        logger.debug('out edges: %s %s %s %s' % (srcnode,
                                                                 srcout,
                                                                 dstnode,
                                                                 dstin))
                        self.disconnect(node, cd[0], v, cd[1])
                        self.connect(srcnode, srcout, dstnode, dstin)
                # expand the workflow node
                # logger.debug('expanding workflow: %s', node)
                node._generate_flatgraph()
                for innernode in node._graph.nodes():
                    innernode._hierarchy = '.'.join((self.name,
                                                     innernode._hierarchy))
                self._graph.add_nodes_from(node._graph.nodes())
                self._graph.add_edges_from(node._graph.edges(data=True))
        if nodes2remove:
            self._graph.remove_nodes_from(nodes2remove)
        logger.debug('finished expanding workflow: %s', self)

    def _get_dot(self, prefix=None, hierarchy=None, colored=False,
                 simple_form=True, level=0):
        """Create a dot file with connection info
        """
        if prefix is None:
            prefix = '  '
        if hierarchy is None:
            hierarchy = []
        colorset = ['#FFFFC8', '#0000FF', '#B4B4FF', '#E6E6FF', '#FF0000',
                    '#FFB4B4', '#FFE6E6', '#00A300', '#B4FFB4', '#E6FFE6']

        dotlist = ['%slabel="%s";' % (prefix, self.name)]
        for node in nx.topological_sort(self._graph):
            fullname = '.'.join(hierarchy + [node.fullname])
            nodename = fullname.replace('.', '_')
            if not isinstance(node, Workflow):
                node_class_name = get_print_name(node, simple_form=simple_form)
                if not simple_form:
                    node_class_name = '.'.join(node_class_name.split('.')[1:])
                if hasattr(node, 'iterables') and node.iterables:
                    dotlist.append(('%s[label="%s", shape=box3d,'
                                    'style=filled, color=black, colorscheme'
                                    '=greys7 fillcolor=2];') % (nodename,
                                                                node_class_name))
                else:
                    if colored:
                        dotlist.append(('%s[label="%s", style=filled,'
                                        ' fillcolor="%s"];')
                                       % (nodename, node_class_name,
                                           colorset[level]))
                    else:
                        dotlist.append(('%s[label="%s"];')
                                       % (nodename, node_class_name))

        for node in nx.topological_sort(self._graph):
            if isinstance(node, Workflow):
                fullname = '.'.join(hierarchy + [node.fullname])
                nodename = fullname.replace('.', '_')
                dotlist.append('subgraph cluster_%s {' % nodename)
                if colored:
                    dotlist.append(prefix + prefix + 'edge [color="%s"];' % (colorset[level + 1]))
                    dotlist.append(prefix + prefix + 'style=filled;')
                    dotlist.append(prefix + prefix + 'fillcolor="%s";' % (colorset[level + 2]))
                dotlist.append(node._get_dot(prefix=prefix + prefix,
                                             hierarchy=hierarchy + [self.name],
                                             colored=colored,
                                             simple_form=simple_form, level=level + 3))
                dotlist.append('}')
                if level == 6:
                    level = 2
            else:
                for subnode in self._graph.successors_iter(node):
                    if node._hierarchy != subnode._hierarchy:
                        continue
                    if not isinstance(subnode, Workflow):
                        nodefullname = '.'.join(hierarchy + [node.fullname])
                        subnodefullname = '.'.join(hierarchy +
                                                   [subnode.fullname])
                        nodename = nodefullname.replace('.', '_')
                        subnodename = subnodefullname.replace('.', '_')
                        for _ in self._graph.get_edge_data(node,
                                                           subnode)['connect']:
                            dotlist.append('%s -> %s;' % (nodename,
                                                          subnodename))
                        logger.debug('connection: ' + dotlist[-1])
        # add between workflow connections
        for u, v, d in self._graph.edges_iter(data=True):
            uname = '.'.join(hierarchy + [u.fullname])
            vname = '.'.join(hierarchy + [v.fullname])
            for src, dest in d['connect']:
                uname1 = uname
                vname1 = vname
                if isinstance(src, tuple):
                    srcname = src[0]
                else:
                    srcname = src
                if '.' in srcname:
                    uname1 += '.' + '.'.join(srcname.split('.')[:-1])
                if '.' in dest and '@' not in dest:
                    if not isinstance(v, Workflow):
                        if 'datasink' not in \
                           str(v._interface.__class__).lower():
                            vname1 += '.' + '.'.join(dest.split('.')[:-1])
                    else:
                        vname1 += '.' + '.'.join(dest.split('.')[:-1])
                if uname1.split('.')[:-1] != vname1.split('.')[:-1]:
                    dotlist.append('%s -> %s;' % (uname1.replace('.', '_'),
                                                  vname1.replace('.', '_')))
                    logger.debug('cross connection: ' + dotlist[-1])
        return ('\n' + prefix).join(dotlist)


text:
(function($, _, chrome, document, window){
   "use strict";

    var ctxHEIGHT = window.ctxHEIGHT,
        ctxWIDTH = 358,
        linkAudit = {
            linkArray:[],
            linksToCheckLength:0,
            ctx:null,
            ctxColumnLength:0,
            ctxPendingHeight:ctxHEIGHT-20,
            ctxErrorHeight:10,
            ctxSuccessHeight:10,
            ctxMatchedHeight:10,
            ops:{
                ignoreHash:true,
                showStatusCode:false,
                showInfoInTitle:true,
                validateDomainOnly:false,
                whitelist:[],
                blacklist:[],
                className:"CE-linkAudit",
                contentSearch:false,
                colour:{
                    "success":["#080","#fff"],
                    "failed":["#800","#fff"],
                    "unknown":["#ff0","#000"],
                    "content-match":["rgba(255,204,0,1)","#000"],
                    "pending":["#cecece","#888"]
                },
                displayHeight:window.ctxHEIGHT
            },
            content:{
                /**
                 * Method, creates ui, gathers links on page
                 * @id init
                 * @param {Object} request
                 * @param {Object} sender
                 * @param {Object} sendResponse
                 * @return void
                 */
                init:function(request, sender, sendResponse){
                    linkAudit.content.log('Recived request, request.action '+ request.action);
                    if (request.action === "getLinkArray"){
                        linkAudit.content.setOptions(request.options);
                        linkAudit.content.getLinkArray(request, sendResponse);
                        linkAudit.content.createUI();
                    } else if (request.action === "updateDisplay"){
                        linkAudit.content.updateDisplay(request);
                        sendResponse({"message":"updated thanks"});
                    }
                },
                /**
                 * Method, sets content script options from background page
                 * @id setOptions
                 * @param {Object} options
                 * @return void
                 */
                setOptions:function(options){
                    if (options){
                        linkAudit.content.log('setting options');
                        for (var item in options){
                            if (options.hasOwnProperty( item )){
                                linkAudit.content.log(item + '='+ options[item] + ' type:'+ typeof options[item]);
                                linkAudit.ops[item] = options[item];
                            }
                        }
                        if (linkAudit.ops.contentSearch === true){
                            linkAudit.ctxColumnLength = 3;
                        } else {
                            linkAudit.ctxColumnLength = 2;
                        }
                        linkAudit.content.log('linkAudit.ctxColumnLength '+ linkAudit.ctxColumnLength);
                    }
                },
                /**
                 * Method, gathers unique links on page and returns them to the background page for processing
                 * @id getLinkArray
                 * @param {Object} request
                 * @param {Object} sendResponse
                 * @return void
                 */
                getLinkArray:function(request, sendResponse){

                    var linkArray = [],
                        item = {},
                        href = "",
                        link,
                        a = document.getElementsByTagName('a');

                    for (var i = 0; i < a.length; i++){
                        link = a[i];
                        var id = linkAudit.content.addHrefAndReturnId(link, linkArray);
                        if (id === "CE-linkAudit-notWhitelisted" || id === "CE-linkAudit-blacklisted"){
                            $(link).addClass( id );
                        } else if (id !== ""){
                            $(link).addClass( id ).addClass(linkAudit.ops.className);
                        }
                    }
                    linkAudit.content.log('Found '+ linkArray.length + ' links to validate');
                    linkAudit.linksToCheckLength = linkArray.length;
                    sendResponse({
                        "action" : "validateLinks",
                        "linkArray" : linkArray
                    });
                },
                /**
                 * Method, checks link href and if unique adds to array and returns the ID
                 * @id addHrefAndReturnId
                 * @param {Object} link
                 * @param {Array} linkArray
                 * @return string
                 */
                addHrefAndReturnId:function(link, linkArray){
                    var path = link.getAttribute("href"),
                        whitelist = linkAudit.ops.whitelist,
                        isWhitelist = true,
                        blacklist = linkAudit.ops.blacklist,
                        isBlacklist = false,
                        i = 0,
                        href = (linkAudit.ops.ignoreHash === true) ? link.href.split("#")[0] : link.href;

                    if (!path || path === "" || path.indexOf("#") === 0 || link.href.substring(0,4) !== "http"){
                        return "";
                    }
                    if (linkAudit.ops.validateDomainOnly && !link.href.match(location.hostname)){
                        //TODO do this check properly
                        linkAudit.content.log('not current domain');
                        return "";
                    }

                    //linkAudit.content.log('blacklist: ' + blacklist.length);
                    //linkAudit.content.log('white list: '+ whitelist.length);

                    if (whitelist.length){
                        isWhitelist = false;
                        for (i = 0; i < whitelist.length ; i ++){
                            if (href.match(whitelist[i])){
                                isWhitelist = true;
                                break;
                            }
                        }
                    }
                    if (isWhitelist === false){
                        linkAudit.content.log("Not in white list: " + href);
                        return "CE-linkAudit-notWhitelisted";
                    }
                    if (blacklist.length){
                        for (i = 0; i < blacklist.length ; i++){
                            if (href.match(blacklist[i])){
                                isBlacklist = true;
                                break;
                            }
                        }
                    }
                    if (isBlacklist === true ){
                        linkAudit.content.log("blacklisted: " + href);
                        return "CE-linkAudit-blacklisted";
                    }
                    var item = _.find(linkArray, function(item){ return item.href === href ;});
                    if (!item){
                        item = {
                            "id":createId(),
                            "href":href,
                            "status":"",
                            "httpStatusCode":-1,
                            "path":path
                        };
                        linkArray.push(item);
                    }
                    return item.id;

                    function createId(){
                        return "CE-linkAudit-"+ linkArray.length;
                    }
                },
                createUI:function(){
                    if ($("#CE-linkAuditDisplay-status span").length){
                        $("#CE-linkAuditDisplay-status").empty();
                    }
                    if ($('#CE-linkAuditDisplay').length === 0){

                        var bg = linkAudit.ops.colour.pending[0],
                            baseStyles = document.createTextNode("a."+linkAudit.ops.className+"{background-color:"+bg+"!important;color:"+linkAudit.ops.colour.pending[1]+";}#CE-linkAuditDisplay-status span.success{background-color:"+ linkAudit.ops.colour.success[0] +";}#CE-linkAuditDisplay-status span.failed{background-color:"+ linkAudit.ops.colour.failed[0] +";}#CE-linkAuditDisplay-status span.matched{background-color:"+ linkAudit.ops.colour["content-match"][0] +";}#CE-linkAuditDisplay-status span.unknown{background-color:"+ linkAudit.ops.colour.unknown[0] +";}#CE-linkAuditDisplay-status span.pending{background-color:"+ bg +";}");

                        linkAudit.content.log('Adding display HTML');
                        if ($("#CE-linkAuditStyleSheet").length){
                            $("#CE-linkAuditStyleSheet").remove();
                        }
                        $("head").append('


text:
package bytelang.parser.lexical.states;

import bytelang.CompilationErrorException;
import bytelang.parser.container.values.Value;
import bytelang.parser.container.values.ValueInteger;

public class StateInteger implements LexicalState, ReadableState {
	private long value;

	@Override
	public StateResult match(String input) {
		boolean hexadecimal = input.startsWith("0x");
		String  word        = "";

		if (hexadecimal) {
			input = input.substring(2);
		}

		while (
			input.length() > 0 &&
				Character.isDigit(input.charAt(0)) ||
				(hexadecimal && (
					(input.charAt(0) >= 'A' && input.charAt(0) <= 'Z') ||
					(input.charAt(0) >= 'a' && input.charAt(0) <= 'z')
				))
		) {
			word  += input.charAt(0);
			input  = input.substring(1);
		}

		try {
			value = Long.valueOf(word, hexadecimal ? 16 : 10);
			return new StateResult(input, null);
		} catch (NumberFormatException e) {
			throw new CompilationErrorException("Internal error: String couldn't be converted to an integer value.");
		}
	}

	@Override
	public LexicalStateType getType() {
		return LexicalStateType.INTEGER;
	}

	public long getValue() {
		return value;
	}

	@Override
	public Value toTypedValue() {
		return new ValueInteger(value);
	}

	@Override
	public String toString() {
		return String.valueOf(value);
	}
}


text:
===============================================

```
public class Pizza {
   XMLUtil xmlUtil = new XMLUtil();
   List ingredients;

   public void prepareFood() {
    gatherIngredients(); //uses list of ingredients
    mixIngredients();
    putInCookingVessel();
    cook(); //different implementation details
    plateFood();
   }

   public void cook() {
      // Do something pizza specific
   }
   public String toXML() {
      String returnXML = xmlUtil.simpleAttributesToXML(this);
      returnXML += xmlUtil.complexAttributesToXML(this);
      returnXML = xmlUtil.classToXML(this, returnXML);
      return returnXML;
   }
}
```

```
public class Cake {
   XMLUtil xmlUtil = new XMLUtil();
   List ingredients;

   public void prepareFood() {
      gatherIngredients(); //uses list of ingredients
      mixIngredients();
      putInCookingVessel();
      cook(); //different implementation details
      plateFood();
      lightCandles();
   }

   public void cook() {
      // Do something cake specific
   }

   public String toXML() {
      String returnXML = xmlUtil.simpleAttributesToXML(this);
      returnXML += xmlUtil.complexAttributesToXML(this);
      returnXML = xmlUtil.classToXML(this, returnXML);
      return returnXML;
   }
}
```

```
public class Refuge extends Land {
   XMLUtil xmlUtil = new XMLUtil();

   public String toXML() {
      String returnXML = xmlUtil.simpleAttributesToXML(this);
      returnXML += xmlUtil.complexAttributesToXML(this);
      returnXML = xmlUtil.classToXML(this, returnXML);
      return returnXML;
   }
}
```


text:
package org.mifos.test.acceptance.framework.client;

import com.thoughtworks.selenium.Selenium;
import org.mifos.test.acceptance.framework.MifosPage;

public class CreateClientPreviewDataPage extends MifosPage {

	public CreateClientPreviewDataPage() {
		super();
	}

	/**
	 * @param selenium
	 */
	public CreateClientPreviewDataPage(Selenium selenium) {
		super(selenium);
	}

    public void verifyPage() {
        this.verifyPage("preview_ClientDetails");
    }

    public CreateClientConfirmationPage submit() {
        selenium.click("preview_ClientDetails.button.submitForApproval");
        waitForPageToLoad();
        return new CreateClientConfirmationPage(selenium);
    }

    public CreateClientEnterFamilyDetailsPage edit() {
        selenium.click("preview_ClientDetails.button.editFamilyInformation");
        waitForPageToLoad();
        return new CreateClientEnterFamilyDetailsPage(selenium);
    }

}


text:
title: "Not About Heroes"
playwright: "Stephen MacDonald"
period: Spring
season: "In House"
season_sort: 260
date_start: 2015-03-11
date_end: 2015-03-14
venue: New Theatre

cast:
  - role: Siegfried Sassoon
    name: Ben Hollands
  - role: Wilfred Owen
    name: Jacob Hayes

crew:
  - role: Director
    name: Laura Jayne Bateman
  - role: Producer
    name: Lydia Smith
  - role: Assistant Producer
    name: Anna Scholes
  - role: Technical Director
    name: Gus Herbert
  - role: Lighting Designer
    name: David Taylor
  - role: Shadow Lighting Designer
    name: James Fox
  - role: Technical Operator
    name: Niamh Caines
  - role: Stage Manager
    name: Jake Leonard
  - role: Stage Manager
    name: Kathy Shepherd
  - role: Stage Manager
    name: Amy Naylor
  - role: Stage Manager
    name: Max Miller
  - role: Stage Manager
    name: Libby Boyd
  - role: Poster Designer
    name: Ben Hollands

assets:
  - type: poster
    image: NTvsBZ6
  - type: backstage
    video: kzTSNDL
    title: Backstage At

links:
  - type: Review
    href: https://impactnottingham.com/2015/03/not-about-heroes-nottingham-new-theatre/
    snapshot: agW3l
    publisher: Impact Nottingham
    author: Rose McGoldrick
    title: "Not About Heroes @ NNT"
    date: 2015-03-12
    rating: 8/10
    quote: "This play certainly taught its audience that \"war is not glorious\", serving as a platform for some very engaging, naturalistic acting and is well worth a watch."

prod_shots: hn8nZP
---

Edinburgh, July 1917. At Craiglockhart War Hospital for Nervous Disorders, Siegfried Sassoon crosses paths with Wilfred Owen. The former, arrogant and frustrated, has literary fame; the latter, diffident and traumatised, is completely unknown. Under Sassoon's tutelage, Owen creates the most revered war poetry ever written, including Anthem for Doomed Youth and Dulce et Decorum Est. Fifteen years later, after a string of affairs and spiralling into depression, Sassoon looks back on their sixteen months of passion- passion that was for both their poetry and for each other. "I don't know where either of us will go. But I'm glad I'm here, tonight, with you."