text:
'use strict';

angular.module('articles').controller('ArticlesController', ['$scope', '$rootScope', '$stateParams', '$location', '$animate', '$timeout', 'Authentication', 'Articles', 'Comments',
	function($scope, $rootScope, $stateParams, $location, $animate, $timeout, Authentication, Articles, Comments) {
		$scope.authentication = Authentication;

		$scope.switchShowFull = function(repeatScope){
			repeatScope.showFull = !repeatScope.showFull;
		};

		$scope.createVisible = false;
		$scope.switchCreateVisible = function(){
			$scope.createVisible = !$scope.createVisible;
		};

		$scope.canEdit = function(post) {
			return ($rootScope.adminMode || post.user._id === Authentication.user._id)
		}

		$scope.create = function(parentString) {
			var article = new Articles({
				title: this.title,
				content: this.content,
				link: this.link
			});

			article.parent = parentString; //by default the articles list only shows where parent = 'top'
			article.user = this.user;
			article.imageurl = this.imageurl;

			article.$save(function(response) {
				//$location.path('articles/' + response._id);

				$scope.title = '';
				$scope.content = '';
				$scope.imageurl= '';
				article.user = Authentication.user;
				$scope.articles.unshift(article); //push it to the display
				$scope.createVisible = !$scope.createVisible;
			}, function(errorResponse) {
				$scope.error = errorResponse.data.message;
			});
		};

		$scope.createComment = function(parentId) {
			var article = new Articles({
				//parent: this.parentId,
				//title: $scope.parentId.toString(),
				content: this.content
			});
			article.title = 'comment';
			article.parent = $scope.article._id; //this.parentId;
			article.$save(function(response) {
				//$location.path('articles/' + response._id);

				//$scope.title = '';
				$scope.content = '';
				article.user = Authentication.user;
				$scope.comments.unshift(article); //display the new comment
				$scope.showComment = !$scope.showComment;
			}, function(errorResponse) {
				$scope.error = errorResponse.data.message;
			});
		};

		$scope.remove = function(article) {
			$scope.confirmDelete = false;
			if (article) {
				article.$remove();

				for (var i in $scope.articles) {
					if ($scope.articles[i] === article) {
						$scope.articles.splice(i, 1);
					}
				}
			} else {
				$scope.article.$remove(function() {
					$location.path('articles');
				});
			}
		};

		$scope.update = function() {
			var article = $scope.article;

			article.$update(function() {
				$location.path('articles/' + article._id);
			}, function(errorResponse) {
				$scope.error = errorResponse.data.message;
			});
		};

		$scope.sortBy = 'created';
		$scope.sortDesc = true;
		$scope.sortAndUpdate = function(sorter){
			if(sorter==='reverse')
				$scope.sortDesc = !$scope.sortDesc;
			else
			{
				$scope.sortBy = sorter;
			}

			$scope.find();
		};

		//$scope.now = Date.now();
		//$scope.fuck = 'ug'; //$scope.articles[1].created;
		//$scope.check = $scope.dateDiffInDays(now,$scope.articles[1].created);

		$scope.dateDiffInDays = function (a, b) {
			var _MS_PER_DAY = 1000 * 60 * 60 * 24;
			// Discard the time and time-zone information.
			var utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate());
			var utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate());
			return (utc2 - utc1) / _MS_PER_DAY;
			//return Math.floor((utc2 - utc1) / _MS_PER_DAY);
		};

/*
		$scope.sortArticles = function(){
			articles.sort(function(a,b){
				var ageA = Date.Now - a.created;
				var ageB = Date.Now - b.created;

				if(a.Created)
			})
		}
*/
		$scope.find = function() {
			//$scope.articles = Articles.query();
			$scope.articles = Articles.list({parent:'top', limit:4, sortBy:($scope.sortDesc?'-':'') + $scope.sortBy});
		};

		$scope.listMsgs = function() {
			$scope.articles = Articles.list({parent:'msg', limit:25, sortBy:($scope.sortDesc?'-':'') + $scope.sortBy});
		}

		$scope.findOne = function() {
			$scope.article = Articles.get({
				articleId: $stateParams.articleId
			});
			$scope.comments = //['one','two','three'];
			Comments.query({
				parentId: $stateParams.articleId
			});
		};

	}
]);


text:
(function() {
  var CLI, Meteor, Spacejam, expect, fs, path, _,
    __bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; };

  require('./log');

  fs = require("fs");

  path = require("path");

  _ = require("underscore");

  expect = require("chai").expect;

  Spacejam = require('./Spacejam');

  Meteor = require('./Meteor');

  require.extensions['.txt'] = function(module, filename) {
    return module.exports = fs.readFileSync(filename, 'utf8');
  };

  CLI = (function() {
    var instance;

    instance = null;

    CLI.get = function() {
      return instance != null ? instance : instance = new CLI();
    };

    CLI.prototype.commands = {
      "test-packages": "testPackages",
      "test-in-velocity": "testInVelocity"
    };

    CLI.prototype.options = null;

    CLI.prototype.spacejam = null;

    CLI.prototype.pidPath = null;

    function CLI() {
      this.onProcessExit = __bind(this.onProcessExit, this);
      this.spacejam = new Spacejam();
      log.debug("CLI.constructor()");
      process.on('SIGPIPE', (function(_this) {
        return function(code) {
          var _ref;
          log.info("spacejam: Received a SIGPIPE signal. Killing all child processes...");
          return (_ref = _this.spacejam) != null ? _ref.killChildren() : void 0;
        };
      })(this));
    }

    CLI.prototype.pidFileInit = function(pidFile) {
      var err, pid, pidPath;
      log.debug("CLI.checkAndCreatePidFile()", arguments);
      expect(pidFile).to.be.a('string').that.has.length.above(0);
      pidPath = path.resolve(pidFile);
      if (fs.existsSync(pidPath)) {
        pid = +fs.readFileSync(pidPath);
        log.info("spacejam: Found pid file " + pidFile + " with pid " + pid + ", checking if process is alive.");
        try {
          process.kill(pid, 0);
          log.warn("spacejam: process with pid " + pid + " is already running, exiting.");
          process.exit(Spacejam.DONE.ALREADY_RUNNING);
        } catch (_error) {
          err = _error;
          log.trace(err);
          log.warn("spacejam: pid file " + pidFile + " exists, but process is dead, ignoring it.");
        }
      }
      log.info("spacejam: Saving pid " + process.pid + " to " + pidPath);
      fs.writeFileSync(pidPath, "" + process.pid);
      this.pidPath = pidPath;
      process.on('exit', this.onProcessExit);
    };

    CLI.prototype.onProcessExit = function(code) {
      var err;
      log.info("spacejam: spacejam is exiting with code " + code + ", deleting pid file.");
      try {
        return fs.unlinkSync(this.pidPath);
      } catch (_error) {
        err = _error;
        log.trace(err);
        return log.error("spacejam: Error deleting pid file " + this.pidPath, err);
      }
    };

    CLI.prototype.exec = function() {
      var command, err, version;
      log.debug("CLI.exec()");
      expect(this.options, "You can only call CLI.exec() once").to.be["null"];
      this.options = require("rc")("spacejam", {});
      if (this.options['pid-file'] != null) {
        this.pidFileInit(this.options['pid-file']);
      }
      command = this.options._[0];
      log.debug("command: " + command);
      if (command === 'help') {
        this.printHelp();
        process.exit(0);
      } else if (command === 'package-version') {
        version = Meteor.getPackageVersion();
        console.log(version);
        process.exit(0);
      }
      if (!_.has(this.commands, command)) {
        if (command) {
          log.error("spacejam: Error: \n'" + command + "' is not a recognized command\n");
        }
        this.printHelp();
      }
      this.options.packages = this.options._.slice(1);
      delete this.options._;
      log.debug("CLI.exec() options:", this.options);
      this.spacejam.on('done', (function(_this) {
        return function(code) {
          return process.exit(code);
        };
      })(this));
      try {
        return this.spacejam[this.commands[command]](this.options);
      } catch (_error) {
        err = _error;
        console.trace(err);
        return process.exit(1);
      }
    };

    CLI.prototype.printHelp = function() {
      log.debug("CLI.printHelp()");
      return process.stdout.write(require('../bin/help.txt'));
    };

    return CLI;

  })();

  module.exports = CLI;

}).call(this);

redpajama_set_name: RedPajamaGithub


text:
<!DOCTYPE html>

    <meta http-equiv="refresh" content="0;URL=constant.SDL_SCANCODE_W.html">

Redirecting to constant.SDL_SCANCODE_W.html...


text:
package alluxio.underfs.hdfs.activesync;

import alluxio.AlluxioURI;
import alluxio.Constants;
import alluxio.SyncInfo;
import alluxio.collections.ConcurrentHashSet;
import alluxio.conf.PropertyKey;
import alluxio.exception.InvalidPathException;
import alluxio.resource.LockResource;
import alluxio.underfs.UnderFileSystemConfiguration;
import alluxio.underfs.hdfs.HdfsActiveSyncProvider;
import alluxio.util.LogUtils;
import alluxio.util.ThreadFactoryUtils;
import alluxio.util.io.PathUtils;
import alluxio.util.logging.SamplingLogger;

import org.apache.hadoop.hdfs.DFSInotifyEventInputStream;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.inotify.Event;
import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.inotify.MissingEventsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

/**
 * Supported Hdfs Active Sync Provider.
 */
public class SupportedHdfsActiveSyncProvider implements HdfsActiveSyncProvider {
  private static final Logger LOG = LoggerFactory.getLogger(SupportedHdfsActiveSyncProvider.class);
  private static final Logger SAMPLING_LOG = new SamplingLogger(LOG, 10L * Constants.MINUTE_MS);

  private final HdfsAdmin mHdfsAdmin;
  // These read write locks protect the state (maps) managed by this class.
  private final Lock mReadLock;
  private final Lock mWriteLock;
  private final ThreadPoolExecutor mExecutorService;
  private final int mActiveUfsSyncMaxActivity;
  private final int mActiveUfsSyncMaxAge;
  private final long mActiveUfsPollTimeoutMs;
  private final long mActiveUfsSyncEventRateInterval;
  private Future<?> mPollingThread;
  private List<AlluxioURI> mUfsUriList;
  private final Queue<Future<Integer>> mProcessTasks;

  // a map mapping SyncPoints to a set of files that have been changed under that syncPoint
  private Map<String, Set<AlluxioURI>> mChangedFiles;
  // Use an integer to indicate the activity level of the sync point
  // TODO(yuzhu): Merge the three maps into one map
  private Map<String, Integer> mActivity;
  private Map<String, Integer> mAge;
  private Map<String, Long> mTxIdMap;
  private long mCurrentTxId;
  private boolean mEventMissed;

  private final int mBatchSize;

  /**
   * Constructor for supported Hdfs Active Sync Provider.
   *
   * @param uri the hdfs uri
   * @param conf the hdfs conf
   * @param ufsConf Alluxio UFS configuration
   */
  public SupportedHdfsActiveSyncProvider(URI uri, org.apache.hadoop.conf.Configuration conf,
      UnderFileSystemConfiguration ufsConf)
      throws IOException {
    mHdfsAdmin = new HdfsAdmin(uri, conf);
    mChangedFiles = new ConcurrentHashMap<>();
    mActivity = new ConcurrentHashMap<>();
    mAge = new ConcurrentHashMap<>();
    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
    mReadLock = lock.readLock();
    mWriteLock = lock.writeLock();
    mExecutorService = new ThreadPoolExecutor(
        ufsConf.getInt(PropertyKey.MASTER_UFS_ACTIVE_SYNC_THREAD_POOL_SIZE),
        ufsConf.getInt(PropertyKey.MASTER_UFS_ACTIVE_SYNC_THREAD_POOL_SIZE),
        1, TimeUnit.MINUTES, new LinkedBlockingQueue<>(),
        ThreadFactoryUtils.build("SupportedHdfsActiveSyncProvider-%d", false),
        new ThreadPoolExecutor.CallerRunsPolicy());
    mExecutorService.allowCoreThreadTimeOut(true);
    mPollingThread = null;
    mUfsUriList = new CopyOnWriteArrayList<>();
    mEventMissed = false;
    mTxIdMap = new ConcurrentHashMap<>();
    mCurrentTxId = SyncInfo.INVALID_TXID;
    mActiveUfsSyncMaxActivity = ufsConf.getInt(PropertyKey.MASTER_UFS_ACTIVE_SYNC_MAX_ACTIVITIES);
    mActiveUfsSyncMaxAge = ufsConf.getInt(PropertyKey.MASTER_UFS_ACTIVE_SYNC_MAX_AGE);
    mActiveUfsPollTimeoutMs = ufsConf.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_POLL_TIMEOUT);
    mActiveUfsSyncEventRateInterval =
        ufsConf.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_EVENT_RATE_INTERVAL);
    mProcessTasks = new LinkedBlockingQueue<>();
    mBatchSize = ufsConf.getInt(PropertyKey.MASTER_UFS_ACTIVE_SYNC_POLL_BATCH_SIZE);
  }

  /**
   * Start the accounting for the next window of events.
   *
   * This includes increasing the age of the unsynced syncpoints and decrease the activity of
   * unsynced syncpoints.
   */
  private void initNextWindow() {
    for (String syncPoint : mActivity.keySet()) {
      mActivity.put(syncPoint, mActivity.get(syncPoint) / 10);
      mAge.put(syncPoint, mAge.get(syncPoint) + 1);
    }
  }

  private void recordFileChanged(String syncPoint, String filePath, long txId) {
    AlluxioURI syncPointUri = new AlluxioURI(syncPoint);

    mChangedFiles.computeIfAbsent(syncPoint, (key) -> {
      mActivity.put(syncPoint, 0);
      mAge.put(syncPoint, 0);
      mTxIdMap.put(syncPoint, txId);
      return new ConcurrentHashSet<>();
    });
    try (LockResource r = new LockResource(mWriteLock)) {
      mChangedFiles.get(syncPoint).add(
          new AlluxioURI(syncPointUri.getScheme(), syncPointUri.getAuthority(), filePath));
      mActivity.put(syncPoint, mActivity.get(syncPoint) + 1);
    }
  }

  private void syncSyncPoint(String syncPoint) {
    mChangedFiles.remove(syncPoint);
    mActivity.remove(syncPoint);
    mAge.remove(syncPoint);
    mTxIdMap.remove(syncPoint);
  }

  private boolean processEvent(Event event, List<AlluxioURI> syncPointList, long txId) {
    boolean fileMatch = false;
    String filePath = "";
    String renameFilePath = "";

    switch (event.getEventType()) {
      case CREATE:
        Event.CreateEvent createEvent = (Event.CreateEvent) event;
        filePath = createEvent.getPath();
        break;
      case UNLINK:
        Event.UnlinkEvent unlinkEvent = (Event.UnlinkEvent) event;
        filePath = unlinkEvent.getPath();
        break;
      case APPEND:
        Event.AppendEvent appendEvent = (Event.AppendEvent) event;
        filePath = appendEvent.getPath();
        break;
      case RENAME:
        Event.RenameEvent renameEvent = (Event.RenameEvent) event;
        filePath = renameEvent.getSrcPath();
        renameFilePath = renameEvent.getDstPath();
        break;
      case METADATA:
        Event.MetadataUpdateEvent metadataUpdateEvent = (Event.MetadataUpdateEvent) event;
        filePath = metadataUpdateEvent.getPath();
        break;
      default:
        break;
    }
    if (filePath.isEmpty()) {
      return false;
    }
    for (AlluxioURI syncPoint :  syncPointList) {
      try {
        // find out if the changed file falls under one of the sync points
        if (PathUtils.hasPrefix(filePath, syncPoint.getPath())) {
          fileMatch = true;
          recordFileChanged(syncPoint.toString(), filePath, txId);
        }
      } catch (InvalidPathException e) {
        LOG.info("Invalid path encountered {} ", filePath);
      }

      try {
        // find out if the changed file falls under one of the sync points
        if ((!renameFilePath.isEmpty())
            && PathUtils.hasPrefix(renameFilePath, syncPoint.getPath())) {
          fileMatch = true;
          recordFileChanged(syncPoint.toString(), renameFilePath, txId);
        }
      } catch (InvalidPathException e) {
        LOG.info("Invalid path encountered {} ", renameFilePath);
      }
    }
    try (LockResource r = new LockResource(mWriteLock)) {
      mCurrentTxId = txId;
    }
    return fileMatch;
  }

  @Override
  public boolean startPolling(long txId) throws IOException {
    if (mPollingThread == null) {
      DFSInotifyEventInputStream eventStream;
      LOG.info("Start polling from event txID {}", txId);
      if (txId != SyncInfo.INVALID_TXID) {
        eventStream = mHdfsAdmin.getInotifyEventStream(txId);
      } else {
        eventStream = mHdfsAdmin.getInotifyEventStream();
      }
      mPollingThread = mExecutorService.submit(() -> {
        pollEvent(eventStream);
      });
      return true;
    }
    return false;
  }

  @Override
  public boolean stopPolling() {
    if (mPollingThread != null) {
      mPollingThread.cancel(true);
      mPollingThread = null;
      return true;
    }
    return false;
  }

  /**
   * startSync on a ufs uri.
   *
   * @param ufsUri the ufs uri to monitor for sync
   */
  @Override
  public void startSync(AlluxioURI ufsUri) {
    LOG.debug("Add {} as a sync point", ufsUri.toString());
    mUfsUriList.add(ufsUri);
  }

  /**
   * stop sync on a ufs uri.
   *
   * @param ufsUri the ufs uri to stop monitoring for sync
   */
  @Override
  public void stopSync(AlluxioURI ufsUri) {
    LOG.debug("attempt to remove {} from sync point list", ufsUri.toString());
    mUfsUriList.remove(ufsUri);
  }

  /**
   * Fetch and process events.
   * @param eventStream event stream
   */
  public void pollEvent(DFSInotifyEventInputStream eventStream) {
    LOG.debug("Polling thread starting, with timeout {} ms", mActiveUfsPollTimeoutMs);
    long start = System.currentTimeMillis();

    long behind = eventStream.getTxidsBehindEstimate();

    while (!Thread.currentThread().isInterrupted()) {
      try {

        List<Callable<Integer>> process = new LinkedList<>();
        for (int i = 0; i < mBatchSize; i++) {
          EventBatch batch = eventStream.poll(mActiveUfsPollTimeoutMs, TimeUnit.MILLISECONDS);
          if (batch == null) {
            break;
          }
          process.add(() -> {
            for (Event event : batch.getEvents()) {
              processEvent(event, mUfsUriList, batch.getTxid());
            }
            return batch.getEvents().length;
          });
        }
        mProcessTasks.add(mExecutorService.submit(() -> process.stream().map(
            callable -> {
              try {
                return callable.call();
              } catch (Exception e) {
                LogUtils.warnWithException(LOG, "Failed to process event", e);
                return 0;
              }
            }).reduce(0, Integer::sum)
        ));
        long end = System.currentTimeMillis();
        if (end > (start + mActiveUfsSyncEventRateInterval)) {
          long currentlyBehind = eventStream.getTxidsBehindEstimate();
          long processedEvents = getCountSinceLastLog();
          long hdfsEvents = processedEvents + currentlyBehind - behind;
          long durationMs = end - start;
          if (LOG.isDebugEnabled()) {
            // for debug, print every interval
            LOG.debug(
                "HDFS sync stats. past duration: {} ms. HDFS generated events: {} ({} events/s). "
                    + "Processed events: {} ({} events/s). TxidsBehindEstimate: {}",
                durationMs,
                hdfsEvents, String.format("%.2f", hdfsEvents * 1000.0f / durationMs),
                processedEvents, String.format("%.2f", processedEvents * 1000.0f / durationMs),
                currentlyBehind);
          } else {
            // for info, print with the sampling logger
            SAMPLING_LOG.info(
                "HDFS sync stats. past duration: {} ms. HDFS generated events: {} ({} events/s). "
                    + "Processed events: {} ({} events/s). TxidsBehindEstimate: {}",
                durationMs,
                hdfsEvents, String.format("%.2f", hdfsEvents * 1000.0f / durationMs),
                processedEvents, String.format("%.2f", processedEvents * 1000.0f / durationMs),
                currentlyBehind);
          }
          behind = currentlyBehind;
          start = end;
        }
      } catch (IOException e) {
        LOG.warn("IOException occured during polling inotify: {}", e.toString());
        if (e.getCause() instanceof InterruptedException) {
          return;
        }
      } catch (MissingEventsException e) {
        LOG.warn("MissingEventException during polling: {}", e.toString());
        mEventMissed = true;
        // need to sync all syncpoints at this point
      } catch (InterruptedException e) {
        LOG.warn("InterruptedException during polling: {}", e.toString());
        return;
      }
    }
  }

  /**
   * Removes any tasks from the head of the task queue which have been completed.
   *
   * Operation is not idempotent.
   *
   * @return the number of events processed since this method was last called
   */
  private long getCountSinceLastLog() {
    // Take any completed tasks off of the queue and update the count.
    int count = 0;
    while (mProcessTasks.peek() != null && mProcessTasks.peek().isDone()) {
      try {
        Future<Integer> task = mProcessTasks.poll();
        if (task == null) {
          // the while condition should ensure this never happens.
          throw new ConcurrentModificationException("Head of queue modified while polling");
        }
        count += task.get();
      } catch (InterruptedException | ExecutionException e) {
        LogUtils.warnWithException(LOG, "EventBatch process task failed: ", e);
        if (e instanceof InterruptedException) {
          Thread.currentThread().interrupt();
          break;
        }
      }
    }
    return count;
  }

  private long getLastTxId() {
    if (mTxIdMap.isEmpty()) {
      return mCurrentTxId;
    } else {
      return Collections.min(mTxIdMap.values());
    }
  }

  /**
   * Get the activity sync info.
   *
   * @return SyncInfo object which encapsulates the necessary information about changes
   */
  public SyncInfo getActivitySyncInfo() {
    // The overview of this method is
    // 1. setup a source of event
    // 2. Filter based on the paths associated with this mountId
    // 3. Build History for each of the syncPoint
    // 4. If heuristics function returns sync, then we sync the syncPoint

    if (mPollingThread == null) {
      return SyncInfo.emptyInfo();
    }

    Map<AlluxioURI, Set<AlluxioURI>> syncPointFiles = new HashMap<>();
    long txId = 0;
    try (LockResource r = new LockResource(mWriteLock)) {
      initNextWindow();
      if (mEventMissed) {
        // force sync every syncpoint
        for (AlluxioURI uri : mUfsUriList) {
          syncPointFiles.put(uri, null);
          syncSyncPoint(uri.toString());
        }
        mEventMissed = false;
        LOG.debug("Missed event, syncing all sync points\n{}",
            Arrays.toString(syncPointFiles.keySet().toArray()));
        SyncInfo syncInfo = new SyncInfo(syncPointFiles, true, getLastTxId());
        return syncInfo;
      }
      for (String syncPoint : mActivity.keySet()) {
        AlluxioURI syncPointURI = new AlluxioURI(syncPoint);
        // if the activity level is below the threshold or the sync point is too old, we sync
        if (mActivity.get(syncPoint) < mActiveUfsSyncMaxActivity
            || mAge.get(syncPoint) > mActiveUfsSyncMaxAge) {
          if (!syncPointFiles.containsKey(syncPointURI)) {
            syncPointFiles.put(syncPointURI, mChangedFiles.get(syncPoint));
          }
          syncSyncPoint(syncPoint);
        }
      }
      txId = getLastTxId();
    }
    LOG.debug("Syncing {} files", syncPointFiles.size());
    LOG.debug("Last transaction id {}", txId);

    SyncInfo syncInfo = new SyncInfo(syncPointFiles, false, txId);
    return syncInfo;
  }
}


text:
<?xml version="1.0" encoding="utf-8"?>
<application xmlns="http://www.kinoma.com/kpr/application/1" id="orientation.example.kinoma.marvell.com" program="src/main" title="Device Orientation Example">

</application>


text:
001    /*
002     * To change this template, choose Tools | Templates
003     * and open the template in the editor.
004     */
005    package com.fairportfirst.frc2014.commands;
006
007    /**
008     *
009     * @author Tyler
010     */
011    public class RetractShooterCommand extends CommandBase {
012
013        public RetractShooterCommand() {
014            // Use requires() here to declare subsystem dependencies
015            // eg. requires(chassis);
016            requires(shooterSubsystem);
017        }
018
019        // Called just before this Command runs the first time
020        protected void initialize() {
021        }
022
023        // Called repeatedly when this Command is scheduled to run
024        protected void execute() {
025            shooterSubsystem.retract();
026        }
027
028        // Make this return true when this Command no longer needs to run execute()
029        protected boolean isFinished() {
030            return true;
031        }
032
033        // Called once after isFinished returns true
034        protected void end() {
035        }
036
037        // Called when another command which requires one or more of the same
038        // subsystems is scheduled to run
039        protected void interrupted() {
040        }
041    }


text:
"""Multiple pagination integration tests."""

from __future__ import unicode_literals

from endless_pagination.tests.integration import SeleniumTestCase

class MultiplePaginationTest(SeleniumTestCase):

    view_name = 'multiple'

    def test_new_elements_loaded(self):
        # Ensure a new page is loaded on click for each paginated elements.
        self.get()
        with self.assertNewElements('object', range(4, 7)):
            with self.assertNewElements('item', range(7, 10)):
                with self.assertNewElements('entry', range(1, 5)):
                    self.click_link(2, 0)
                    self.click_link(3, 1)
                    self.click_link(self.MORE)

    def test_url_not_changed(self):
        # Ensure the requests are done using Ajax (the page does not refresh).
        self.get()
        with self.assertSameURL():
            self.click_link(2, 0)
            self.click_link(3, 1)
            self.click_link(self.MORE)

    def test_direct_link(self):
        # Ensure direct links work.
        self.get(data={'objects-page': 3, 'items-page': 4, 'entries-page': 5})
        self.assertElements('object', range(7, 10))
        self.assertElements('item', range(10, 13))
        self.assertElements('entry', range(11, 14))
        self.assertIn('objects-page=3', self.selenium.current_url)
        self.assertIn('items-page=4', self.selenium.current_url)
        self.assertIn('entries-page=5', self.selenium.current_url)

    def test_subsequent_pages(self):
        # Ensure elements are correctly loaded starting from a subsequent page.
        self.get(data={'objects-page': 2, 'items-page': 2, 'entries-page': 2})
        with self.assertNewElements('object', range(1, 4)):
            with self.assertNewElements('item', range(7, 10)):
                with self.assertNewElements('entry', range(2, 8)):
                    self.click_link(self.PREVIOUS, 0)
                    self.click_link(self.NEXT, 1)
                    self.click_link(self.MORE)

    def test_no_more_link_in_last_page(self):
        # Ensure there is no more or forward links on the last pages.
        self.get(data={'objects-page': 7, 'items-page': 7, 'entries-page': 8})
        self.asserLinksEqual(0, self.NEXT)
        self.asserLinksEqual(0, self.MORE)


text:
 $("#login-button").click(function(event){
		 event.preventDefault();

	 $('form').fadeOut(500);
	 $('.wrapper').addClass('form-success');
});


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <meta name="generator" content="rustdoc">
    <meta name="description" content="API documentation for the Rust `GraphemeIndices` struct in crate `unicode`.">
    <meta name="keywords" content="rust, rustlang, rust-lang, GraphemeIndices">

    <title>unicode::str::GraphemeIndices - Rust</title>

    <link rel="stylesheet" type="text/css" href="../../main.css">

    <link rel="shortcut icon" href="http://www.rust-lang.org/favicon.ico">

unicode::<wbr>str

        <form class="search-form js-only">

                <input class="search-input" name="search"
                       autocomplete="off"
                       placeholder="Click or press 'S' to search, '?' for more options..."
                       type="search">

        </form>

Struct unicode::<wbr>str::<wbr>GraphemeIndices<wbr>Unstable
            [-] [+]
        [src]

pub struct GraphemeIndices<'a> {
    // some fields omitted
}

External iterator for grapheme clusters and byte offsets.

Trait Implementations

impl<'a> Iterator for GraphemeIndices<'a>

type Item = (usize, &'a str)

fn next(&mut self) -> Option<(usize, &'a str)>

fn size_hint(&self) -> (usize, Option<usize>)

impl<'a> DoubleEndedIterator for GraphemeIndices<'a>

fn next_back(&mut self) -> Option<(usize, &'a str)>

Derived Implementations

impl<'a> Clone for GraphemeIndices<'a>

fn clone(&self) -> GraphemeIndices<'a>

fn clone_from(&mut self, source: &Self)

Keyboard shortcuts

?

Show this help dialog

S

Focus the search field

⇤

Move up in search results

⇥

Move down in search results

⏎

Go to active search result

Search tricks

                Prefix searches with a type followed by a colon (e.g.
                fn:) to restrict the search to a given type.

                Accepted types are: fn, mod,
                struct, enum,
                trait, typedef (or
                tdef).


text:
typedef void (^ChangeIdBlock)();

@interface MyInfoPage : BaseSetViewController

- (instancetype)initWithUserInfo:(AccountInfo *)userInfo;

@property(strong, nonatomic) ChangeIdBlock changeIdBlock;

@end


text:
using Aurora.Profiles;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Aurora.Settings
{
    public interface IEffectScript
    {
        string ID { get; }

        VariableRegistry Properties { get; }

        object UpdateLights(VariableRegistry properties, IGameState state = null);
    }
}


text:
"""Runs mesh network benchmarks.

Runs TCP_RR, TCP_STREAM benchmarks from netperf and compute total throughput
and average latency inside mesh network.
"""

import logging
import re
import threading
from absl import flags
from perfkitbenchmarker import configs
from perfkitbenchmarker import errors
from perfkitbenchmarker import sample
from perfkitbenchmarker import vm_util
from perfkitbenchmarker.linux_packages import netperf
from six.moves import range

flags.DEFINE_integer('num_connections', 1,
                     'Number of connections between each pair of vms.')

flags.DEFINE_integer('num_iterations', 1,
                     'Number of iterations for each run.')

FLAGS = flags.FLAGS

BENCHMARK_NAME = 'mesh_network'
BENCHMARK_CONFIG = """
mesh_network:
  description: >
    Measures VM to VM cross section bandwidth in
    a mesh network. Specify the number of VMs in the network
    with --num_vms.
  vm_groups:
    default:
      vm_spec: *default_single_core
"""

NETPERF_BENCHMARKSS = ['TCP_RR', 'TCP_STREAM']
VALUE_INDEX = 1
RESULT_LOCK = threading.Lock()

def GetConfig(user_config):
  config = configs.LoadConfig(BENCHMARK_CONFIG, user_config, BENCHMARK_NAME)
  config['vm_groups']['default']['vm_count'] = FLAGS.num_vms
  if FLAGS.num_vms < 2:  # Needs at least 2 vms to run the benchmark.
    config['vm_groups']['default']['vm_count'] = 2
  return config

def PrepareVM(vm):
  """Prepare netperf on a single VM.

  Args:
    vm: The VM that needs to install netperf package.
  """
  vm.RemoteCommand('./netserver')

def Prepare(benchmark_spec):
  """Install vms with necessary softwares.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the banchmark.
  """
  vms = benchmark_spec.vms
  logging.info('Preparing netperf on %s', vms[0])
  vms[0].Install('netperf')
  for vm in vms:
    vms[0].MoveFile(vm, netperf.NETPERF_PATH)
    vms[0].MoveFile(vm, netperf.NETSERVER_PATH)
  vm_util.RunThreaded(PrepareVM, vms, len(vms))

def RunNetperf(vm, benchmark_name, servers, result):
  """Spawns netperf on a remote VM, parses results.

  Args:
    vm: The VM running netperf.
    benchmark_name: The netperf benchmark to run.
    servers: VMs running netserver.
    result: The result variable shared by all threads.
  """
  cmd = ''
  if FLAGS.duration_in_seconds:
    cmd_duration_suffix = '-l %s' % FLAGS.duration_in_seconds
  else:
    cmd_duration_suffix = ''
  for server in servers:
    if vm != server:
      cmd += ('./netperf -t '
              '{benchmark_name} -H {server_ip} -i {iterations} '
              '{cmd_suffix} & ').format(
                  benchmark_name=benchmark_name,
                  server_ip=server.internal_ip,
                  iterations=FLAGS.num_iterations,
                  cmd_suffix=cmd_duration_suffix)
  netperf_cmd = ''
  for _ in range(FLAGS.num_connections):
    netperf_cmd += cmd
  netperf_cmd += 'wait'
  output, _ = vm.RemoteCommand(netperf_cmd)
  logging.info(output)

  match = re.findall(r'(\d+\.\d+)\s+\n', output)
  value = 0
  expected_num_match = (len(servers) - 1) * FLAGS.num_connections
  if len(match) != expected_num_match:
    raise errors.Benchmarks.RunError(
        'Netserver not reachable. Expecting %s results, got %s.' %
        (expected_num_match, len(match)))
  for res in match:
    if benchmark_name == 'TCP_RR':
      value += 1.0 / float(res) * 1000.0
    else:
      value += float(res)
  with RESULT_LOCK:
    result[VALUE_INDEX] += value

def Run(benchmark_spec):
  """Run netperf on target vms.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.

  Returns:
    Total throughput, average latency in the form of tuple. The tuple contains
        the sample metric (string), value (float), unit (string).
  """
  vms = benchmark_spec.vms
  num_vms = len(vms)
  results = []
  for netperf_benchmark in NETPERF_BENCHMARKSS:
    args = []
    metadata = {
        'number_machines': num_vms,
        'number_connections': FLAGS.num_connections
    }

    if netperf_benchmark == 'TCP_STREAM':
      metric = 'TCP_STREAM_Total_Throughput'
      unit = 'Mbits/sec'
      value = 0.0
    else:
      metric = 'TCP_RR_Average_Latency'
      unit = 'ms'
      value = 0.0
    result = [metric, value, unit, metadata]
    args = [((source, netperf_benchmark, vms, result), {}) for source in vms]
    vm_util.RunThreaded(RunNetperf, args, num_vms)
    result = sample.Sample(*result)
    if netperf_benchmark == 'TCP_RR':
      denom = ((num_vms - 1) *
               num_vms *
               FLAGS.num_connections)
      result = result._replace(value=result.value / denom)

    results.append(result)
  logging.info(results)
  return results

def Cleanup(benchmark_spec):
  """Cleanup netperf on the target vm (by uninstalling).

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
        required to run the benchmark.
  """
  vms = benchmark_spec.vms
  for vm in vms:
    logging.info('uninstalling netperf on %s', vm)
    vm.RemoteCommand('pkill -9 netserver')
    vm.RemoteCommand('rm netserver')
    vm.RemoteCommand('rm netperf')


text:
#pragma once

#include <memory>

#include <folly/io/IOBuf.h>
#include <folly/Memory.h>
#include <folly/Optional.h>

#include "mcrouter/lib/IOBufUtil.h"
#include "mcrouter/lib/mc/msg.h"
#include "mcrouter/lib/McMsgRef.h"
#include "mcrouter/lib/McOperation.h"
#include "mcrouter/lib/McResUtil.h"
#include "mcrouter/lib/OperationTraits.h"
#include "mcrouter/lib/Reply.h"

namespace facebook { namespace memcache {

class McReply;
struct AccessPoint;

namespace detail {
inline void mcReplySetMcMsgRef(McReply& reply, McMsgRef&& msg);
}  // detail

/**
 * mc_msg_t-based Reply implementation.
 */
class McReply {
 public:

  /* Reply interface */

  /**
   * Constructs a default successful reply for a given operation.
   *
   * Example uses would be an immediate reply for an async operation;
   * a reply for a delete queued for replay, etc.
   */
  template <int op>
  McReply(DefaultReplyT, McOperation<op>) noexcept;

  template <class Request>
  McReply(DefaultReplyT, const Request&) noexcept;

  /**
   * Constructs an "error" reply, meaning that there was a routing error.
   */
  explicit McReply(ErrorReplyT) noexcept : McReply(mc_res_local_error) {
  }

  /**
   * Same as above, but additionally stores error value.
   */
  McReply(ErrorReplyT, folly::StringPiece valueToSet)
      : McReply(mc_res_local_error, valueToSet) {
  }

  /**
   * Constructs a TKO reply.
   *
   * Used to signal that the Route Handle didn't attempt to send out a request.
   * A sending Route Handle might attempt an immediate failover on a TKO reply.
   */
  explicit McReply(TkoReplyT) noexcept
      : McReply(mc_res_tko) {
  }

  /**
   * Picks one McReply from the iterator range.
   *
   * Used to reduce replies for AllSync and similar.
   *
   * @param begin Points to the first McReply object in the range
   * @param end Points to the first McReply past the end of the range
   *
   * @return Iterator to one of the objects from the input range
   */
  template <typename InputIterator>
  static InputIterator reduce(InputIterator begin, InputIterator end);

  /**
   * @return True if this reply's result is worse than other.result()
   */
  bool worseThan(const McReply& other) const noexcept;

  bool isError() const noexcept {
    return isErrorResult(result_);
  }

  bool isFailoverError() const noexcept {
    return isFailoverErrorResult(result_);
  }

  bool isSoftTkoError() const noexcept {
    return isSoftTkoErrorResult(result_);
  }

  bool isHardTkoError() const noexcept {
    return isHardTkoErrorResult(result_);
  }

  bool isTko() const noexcept {
    return isTkoResult(result_);
  }

  bool isLocalError() const noexcept {
    return isLocalErrorResult(result_);
  }

  bool isConnectError() const noexcept {
    return isConnectErrorResult(result_);
  }

  bool isConnectTimeout() const noexcept {
    return isConnectTimeoutResult(result_);
  }

  bool isDataTimeout() const noexcept {
    return isDataTimeoutResult(result_);
  }

  bool isRedirect() const noexcept {
    return isRedirectResult(result_);
  }

  bool isHit() const noexcept {
    return isHitResult(result_);
  }

  bool isMiss() const noexcept {
    return isMissResult(result_);
  }

  bool isHotMiss() const noexcept {
    return isHotMissResult(result_);
  }

  bool isStored() const noexcept {
    return isStoredResult(result_);
  }

  /**
   * Functions to update value and result
   */
  void setValue(folly::IOBuf valueData);
  void setValue(folly::StringPiece str);
  void setResult(mc_res_t res) noexcept {
    result_ = res;
  }

  mc_res_t result() const noexcept {
    return result_;
  }

  /**
   * TypedThriftReply allows syntax like
   *   reply->get_value()
   * In order to ease the transition to TypedThriftReply, we hack McReply
   * to allow similar syntax.
   */
  const McReply* operator->() const {
    return this;
  }

  McReply* operator->() {
    return this;
  }

  bool hasValue() const {
    return valueData_.hasValue();
  }

  const folly::IOBuf& value() const {
    static folly::IOBuf emptyBuffer;
    return hasValue() ? valueData_.value() : emptyBuffer;
  }

  /**
   * Hack. TypedThriftReply comment above applies here, as well.
   */
  const folly::IOBuf& get_value() const {
    return value();
  }

  folly::IOBuf* valuePtrUnsafe() {
    return valueData_.get_pointer();
  }

  const folly::IOBuf* valuePtrUnsafe() const {
    return valueData_.get_pointer();
  }

  folly::StringPiece valueRangeSlow() const {
    return valueData_.hasValue() ? folly::StringPiece(valueData_->coalesce())
                                 : folly::StringPiece();
  }

  const std::shared_ptr<const AccessPoint>& destination() const noexcept {
    return destination_;
  }

  void setDestination(std::shared_ptr<const AccessPoint> ap) noexcept {
    destination_ = std::move(ap);
  }

  uint32_t appSpecificErrorCode() const noexcept {
    return errCode_;
  }

  void setAppSpecificErrorCode(uint32_t ecode) noexcept {
    errCode_ = ecode;
  }

  uint64_t flags() const noexcept {
    return flags_;
  }

  void setFlags(uint64_t fl) noexcept {
    flags_ = fl;
  }

  uint32_t exptime() const noexcept {
    return exptime_;
  }

  void setExptime(uint32_t et) noexcept {
    exptime_ = et;
  }

  uint32_t number() const noexcept {
    return number_;
  }

  void setNumber(uint32_t num) noexcept {
    number_ = num;
  }

  uint64_t leaseToken() const noexcept {
    return leaseToken_;
  }

  void setLeaseToken(uint64_t lt) noexcept {
    leaseToken_ = lt;
  }

  uint64_t cas() const noexcept {
    return cas_;
  }

  void setCas(uint64_t c) noexcept {
    cas_ = c;
  }

  uint64_t delta() const noexcept {
    return delta_;
  }

  void setDelta(uint64_t d) noexcept {
    delta_ = d;
  }

  uint8_t ipv() const {
    return msg_.get() ? msg_->ipv : 0;
  }

  const struct in6_addr& ipAddress() const {
    static struct in6_addr addr;
    return msg_.get() ? msg_->ip_addr : addr;
  }

  void setIpAddress(const struct in6_addr& addr, uint8_t ipVersion) {
    // TODO(prateekj): Task 8315662 - Store ipAddress in McReply itself
    auto msg = createMcMsgRef();
    msg->ip_addr = addr;
    msg->ipv = ipVersion;
    msg_ = std::move(msg);
  }

  /**
   * Hacks for compatibility with TypedThriftReply API. Use sparingly.
   */
  void set_casToken(uint64_t c) noexcept {
    setCas(c);
  }
  void set_leaseToken(uint64_t lt) noexcept {
    setLeaseToken(lt);
  }
  void set_delta(uint64_t d) noexcept {
    setDelta(d);
  }
  void set_exptime(uint64_t et) noexcept {
    setExptime(et);
  }
  void set_age(uint64_t a) noexcept {
    setNumber(a);
  }

  /**
   * Fills out a provided mc_msg_t so that it represents this McReply
   * for the given op.
   * The msg fields might reference the data owned by this McReply,
   * so the msg is only valid as long as the McReply is valid.
   */
  void dependentMsg(mc_op_t op, mc_msg_t* out) const;

  /**
   * Returns a self-contained mc_msg_t representing this McReply
   * for the given op.
   *
   * NOTE: this McReply is still valid after the call.  The obtained
   * McMsgRef and the existing McReply do not depend on each other.
   */
  McMsgRef releasedMsg(mc_op_t op) const;

  /**
   * Set the destructor. If destructor/ctx are non-null, will call
   * destructor(ctx) when this reply is destroyed.
   */
  void setDestructor(void (*destructor) (void*), void* ctx) {
    assert(!destructor_.hasValue());
    destructor_.assign(CUniquePtr(ctx, destructor));
  }

  ~McReply() = default;

  explicit McReply(mc_res_t result) noexcept;
  McReply() = default;
  McReply(mc_res_t result, McMsgRef&& reply);

  McReply(mc_res_t result, folly::IOBuf value);
  McReply(mc_res_t result, folly::StringPiece value);
  McReply(mc_res_t result, const char* value);
  McReply(mc_res_t result, std::string value);
  McReply(McReply&& other) = default;
  McReply& operator=(McReply&& other) = default;

 private:
  McMsgRef msg_;
  mc_res_t result_{mc_res_unknown};
  mutable folly::Optional<folly::IOBuf> valueData_;
  std::shared_ptr<const AccessPoint> destination_;
  uint64_t flags_{0};
  uint64_t leaseToken_{0};
  uint64_t delta_{0};
  uint64_t cas_{0};
  uint32_t errCode_{0};
  uint32_t number_{0};
  uint32_t exptime_{0};

  /**
   * Container for a C-style destructor
   */
  using CUniquePtr = std::unique_ptr<void, void(*)(void*)>;
  folly::Optional<CUniquePtr> destructor_;

  friend class McClientAsciiParser;
  inline friend void detail::mcReplySetMcMsgRef(McReply& reply, McMsgRef&& msg);
};

}}  // facebook::memcache

#include "McReply-inl.h"


text:
package org.elasticsearch.indices.recovery;

import com.google.common.base.Predicate;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFormatTooNewException;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RateLimiter;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateObserver;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.CancellableThreads;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.engine.RecoveryEngineException;
import org.elasticsearch.index.mapper.MapperException;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.*;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.*;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;

import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;

/**
 * The recovery target handles recoveries of peer shards of the shard+node to recover to.
 *

 *
Note, it can be safely assumed that there will only be a single recovery per shard (index+id) and
 * not several of them (since we don't allocate several shard replicas to the same node).
 */
public class RecoveryTarget extends AbstractComponent {

    public static class Actions {
        public static final String FILES_INFO = "internal:index/shard/recovery/filesInfo";
        public static final String FILE_CHUNK = "internal:index/shard/recovery/file_chunk";
        public static final String CLEAN_FILES = "internal:index/shard/recovery/clean_files";
        public static final String TRANSLOG_OPS = "internal:index/shard/recovery/translog_ops";
        public static final String PREPARE_TRANSLOG = "internal:index/shard/recovery/prepare_translog";
        public static final String FINALIZE = "internal:index/shard/recovery/finalize";
    }

    private final ThreadPool threadPool;

    private final TransportService transportService;

    private final RecoverySettings recoverySettings;
    private final ClusterService clusterService;

    private final RecoveriesCollection onGoingRecoveries;

    @Inject
    public RecoveryTarget(Settings settings, ThreadPool threadPool, TransportService transportService,
                          IndicesLifecycle indicesLifecycle, RecoverySettings recoverySettings, ClusterService clusterService) {
        super(settings);
        this.threadPool = threadPool;
        this.transportService = transportService;
        this.recoverySettings = recoverySettings;
        this.clusterService = clusterService;
        this.onGoingRecoveries = new RecoveriesCollection(logger, threadPool);

        transportService.registerRequestHandler(Actions.FILES_INFO, RecoveryFilesInfoRequest.class, ThreadPool.Names.GENERIC, new FilesInfoRequestHandler());
        transportService.registerRequestHandler(Actions.FILE_CHUNK, RecoveryFileChunkRequest.class, ThreadPool.Names.GENERIC, new FileChunkTransportRequestHandler());
        transportService.registerRequestHandler(Actions.CLEAN_FILES, RecoveryCleanFilesRequest.class, ThreadPool.Names.GENERIC, new CleanFilesRequestHandler());
        transportService.registerRequestHandler(Actions.PREPARE_TRANSLOG, RecoveryPrepareForTranslogOperationsRequest.class, ThreadPool.Names.GENERIC, new PrepareForTranslogOperationsRequestHandler());
        transportService.registerRequestHandler(Actions.TRANSLOG_OPS, RecoveryTranslogOperationsRequest.class, ThreadPool.Names.GENERIC, new TranslogOperationsRequestHandler());
        transportService.registerRequestHandler(Actions.FINALIZE, RecoveryFinalizeRecoveryRequest.class, ThreadPool.Names.GENERIC, new FinalizeRecoveryRequestHandler());

        indicesLifecycle.addListener(new IndicesLifecycle.Listener() {
            @Override
            public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard,
                                               @IndexSettings Settings indexSettings) {
                if (indexShard != null) {
                    onGoingRecoveries.cancelRecoveriesForShard(shardId, "shard closed");
                }
            }
        });
    }

    /**
     * cancel all ongoing recoveries for the given shard, if their status match a predicate
     *
     * @param reason       reason for cancellation
     * @param shardId      shardId for which to cancel recoveries
     * @param shouldCancel a predicate to check if a recovery should be cancelled or not. Null means cancel without an extra check.
     *                     note that the recovery state can change after this check, but before it is being cancelled via other
     *                     already issued outstanding references.
     * @return true if a recovery was cancelled
     */
    public boolean cancelRecoveriesForShard(ShardId shardId, String reason, @Nullable Predicate<RecoveryStatus> shouldCancel) {
        return onGoingRecoveries.cancelRecoveriesForShard(shardId, reason, shouldCancel);
    }

    public void startRecovery(final IndexShard indexShard, final RecoveryState.Type recoveryType, final DiscoveryNode sourceNode, final RecoveryListener listener) {
        try {
            indexShard.recovering("from " + sourceNode, recoveryType, sourceNode);
        } catch (IllegalIndexShardStateException e) {
            // that's fine, since we might be called concurrently, just ignore this, we are already recovering
            logger.debug("{} ignore recovery. already in recovering process, {}", indexShard.shardId(), e.getMessage());
            return;
        }
        // create a new recovery status, and process...
        final long recoveryId = onGoingRecoveries.startRecovery(indexShard, sourceNode, listener, recoverySettings.activityTimeout());
        threadPool.generic().execute(new RecoveryRunner(recoveryId));

    }

    protected void retryRecovery(final RecoveryStatus recoveryStatus, final String reason, TimeValue retryAfter, final StartRecoveryRequest currentRequest) {
        logger.trace("will retrying recovery with id [{}] in [{}] (reason [{}])", recoveryStatus.recoveryId(), retryAfter, reason);
        try {
            recoveryStatus.resetRecovery();
        } catch (Throwable e) {
            onGoingRecoveries.failRecovery(recoveryStatus.recoveryId(), new RecoveryFailedException(currentRequest, e), true);
        }
        threadPool.schedule(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(recoveryStatus.recoveryId()));
    }

    private void doRecovery(final RecoveryStatus recoveryStatus) {
        assert recoveryStatus.sourceNode() != null : "can't do a recovery without a source node";

        logger.trace("collecting local files for {}", recoveryStatus);
        Store.MetadataSnapshot metadataSnapshot = null;
        try {
            metadataSnapshot = recoveryStatus.store().getMetadataOrEmpty();
        } catch (IOException e) {
            logger.warn("error while listing local files, recover as if there are none", e);
            metadataSnapshot = Store.MetadataSnapshot.EMPTY;
        } catch (Exception e) {
            // this will be logged as warning later on...
            logger.trace("unexpected error while listing local files, failing recovery", e);
            onGoingRecoveries.failRecovery(recoveryStatus.recoveryId(),
                    new RecoveryFailedException(recoveryStatus.state(), "failed to list local files", e), true);
            return;
        }
        final StartRecoveryRequest request = new StartRecoveryRequest(recoveryStatus.shardId(), recoveryStatus.sourceNode(), clusterService.localNode(),
                false, metadataSnapshot, recoveryStatus.state().getType(), recoveryStatus.recoveryId());

        final AtomicReference<RecoveryResponse> responseHolder = new AtomicReference<>();
        try {
            logger.trace("[{}][{}] starting recovery from {}", request.shardId().index().name(), request.shardId().id(), request.sourceNode());
            recoveryStatus.indexShard().prepareForIndexRecovery();
            recoveryStatus.CancellableThreads().execute(new CancellableThreads.Interruptable() {
                @Override
                public void run() throws InterruptedException {
                    responseHolder.set(transportService.submitRequest(request.sourceNode(), RecoverySource.Actions.START_RECOVERY, request, new FutureTransportResponseHandler<RecoveryResponse>() {
                        @Override
                        public RecoveryResponse newInstance() {
                            return new RecoveryResponse();
                        }
                    }).txGet());
                }
            });
            final RecoveryResponse recoveryResponse = responseHolder.get();
            assert responseHolder != null;
            final TimeValue recoveryTime = new TimeValue(recoveryStatus.state().getTimer().time());
            // do this through ongoing recoveries to remove it from the collection
            onGoingRecoveries.markRecoveryAsDone(recoveryStatus.recoveryId());
            if (logger.isTraceEnabled()) {
                StringBuilder sb = new StringBuilder();
                sb.append('[').append(request.shardId().index().name()).append(']').append('[').append(request.shardId().id()).append("] ");
                sb.append("recovery completed from ").append(request.sourceNode()).append(", took[").append(recoveryTime).append("]\n");
                sb.append("   phase1: recovered_files [").append(recoveryResponse.phase1FileNames.size()).append("]").append(" with total_size of [").append(new ByteSizeValue(recoveryResponse.phase1TotalSize)).append("]")
                        .append(", took [").append(timeValueMillis(recoveryResponse.phase1Time)).append("], throttling_wait [").append(timeValueMillis(recoveryResponse.phase1ThrottlingWaitTime)).append(']')
                        .append("\n");
                sb.append("         : reusing_files   [").append(recoveryResponse.phase1ExistingFileNames.size()).append("] with total_size of [").append(new ByteSizeValue(recoveryResponse.phase1ExistingTotalSize)).append("]\n");
                sb.append("   phase2: start took [").append(timeValueMillis(recoveryResponse.startTime)).append("]\n");
                sb.append("         : recovered [").append(recoveryResponse.phase2Operations).append("]").append(" transaction log operations")
                        .append(", took [").append(timeValueMillis(recoveryResponse.phase2Time)).append("]")
                        .append("\n");
                logger.trace(sb.toString());
            } else {
                logger.debug("{} recovery done from [{}], took [{}]", request.shardId(), recoveryStatus.sourceNode(), recoveryTime);
            }
        } catch (CancellableThreads.ExecutionCancelledException e) {
            logger.trace("recovery cancelled", e);
        } catch (Throwable e) {

            if (logger.isTraceEnabled()) {
                logger.trace("[{}][{}] Got exception on recovery", e, request.shardId().index().name(), request.shardId().id());
            }
            Throwable cause = ExceptionsHelper.unwrapCause(e);
            if (cause instanceof RecoveryEngineException) {
                // unwrap an exception that was thrown as part of the recovery
                cause = cause.getCause();
            }
            // do it twice, in case we have double transport exception
            cause = ExceptionsHelper.unwrapCause(cause);
            if (cause instanceof RecoveryEngineException) {
                // unwrap an exception that was thrown as part of the recovery
                cause = cause.getCause();
            }

            // here, we would add checks against exception that need to be retried (and not removeAndClean in this case)

            if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException || cause instanceof ShardNotFoundException) {
                // if the target is not ready yet, retry
                retryRecovery(recoveryStatus, "remote shard not ready", recoverySettings.retryDelayStateSync(), request);
                return;
            }

            if (cause instanceof DelayRecoveryException) {
                retryRecovery(recoveryStatus, cause.getMessage(), recoverySettings.retryDelayStateSync(), request);
                return;
            }

            if (cause instanceof ConnectTransportException) {
                logger.debug("delaying recovery of {} for [{}] due to networking error [{}]", recoveryStatus.shardId(), recoverySettings.retryDelayNetwork(), cause.getMessage());
                retryRecovery(recoveryStatus, cause.getMessage(), recoverySettings.retryDelayNetwork(), request);
                return;
            }

            if (cause instanceof IndexShardClosedException) {
                onGoingRecoveries.failRecovery(recoveryStatus.recoveryId(), new RecoveryFailedException(request, "source shard is closed", cause), false);
                return;
            }

            if (cause instanceof AlreadyClosedException) {
                onGoingRecoveries.failRecovery(recoveryStatus.recoveryId(), new RecoveryFailedException(request, "source shard is closed", cause), false);
                return;
            }

            onGoingRecoveries.failRecovery(recoveryStatus.recoveryId(), new RecoveryFailedException(request, e), true);
        }
    }

    public interface RecoveryListener {
        void onRecoveryDone(RecoveryState state);

        void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure);
    }

    class PrepareForTranslogOperationsRequestHandler implements TransportRequestHandler<RecoveryPrepareForTranslogOperationsRequest> {

        @Override
        public void messageReceived(RecoveryPrepareForTranslogOperationsRequest request, TransportChannel channel) throws Exception {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
                final RecoveryStatus recoveryStatus = statusRef.status();
                recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps());
                recoveryStatus.indexShard().skipTranslogRecovery(false);
            }
            channel.sendResponse(TransportResponse.Empty.INSTANCE);
        }
    }

    class FinalizeRecoveryRequestHandler implements TransportRequestHandler<RecoveryFinalizeRecoveryRequest> {

        @Override
        public void messageReceived(RecoveryFinalizeRecoveryRequest request, TransportChannel channel) throws Exception {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
                final RecoveryStatus recoveryStatus = statusRef.status();
                recoveryStatus.indexShard().finalizeRecovery();
            }
            channel.sendResponse(TransportResponse.Empty.INSTANCE);
        }
    }

    class TranslogOperationsRequestHandler implements TransportRequestHandler<RecoveryTranslogOperationsRequest> {

        @Override
        public void messageReceived(final RecoveryTranslogOperationsRequest request, final TransportChannel channel) throws Exception {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
                final ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger);
                final RecoveryStatus recoveryStatus = statusRef.status();
                final RecoveryState.Translog translog = recoveryStatus.state().getTranslog();
                translog.totalOperations(request.totalTranslogOps());
                assert recoveryStatus.indexShard().recoveryState() == recoveryStatus.state();
                try {
                    recoveryStatus.indexShard().performBatchRecovery(request.operations());
                } catch (TranslogRecoveryPerformer.BatchOperationException exception) {
                    MapperException mapperException = (MapperException) ExceptionsHelper.unwrap(exception, MapperException.class);
                    if (mapperException == null) {
                        throw exception;
                    }
                    // in very rare cases a translog replay from primary is processed before a mapping update on this node
                    // which causes local mapping changes. we want to wait until these mappings are processed.
                    logger.trace("delaying recovery due to missing mapping changes (rolling back stats for [{}] ops)", exception, exception.completedOperations());
                    translog.decrementRecoveredOperations(exception.completedOperations());
                    // we do not need to use a timeout here since the entire recovery mechanism has an inactivity protection (it will be
                    // canceled)
                    observer.waitForNextChange(new ClusterStateObserver.Listener() {
                        @Override
                        public void onNewClusterState(ClusterState state) {
                            try {
                                messageReceived(request, channel);
                            } catch (Exception e) {
                                onFailure(e);
                            }
                        }

                        protected void onFailure(Exception e) {
                            try {
                                channel.sendResponse(e);
                            } catch (IOException e1) {
                                logger.warn("failed to send error back to recovery source", e1);
                            }
                        }

                        @Override
                        public void onClusterServiceClose() {
                            onFailure(new ElasticsearchException("cluster service was closed while waiting for mapping updates"));
                        }

                        @Override
                        public void onTimeout(TimeValue timeout) {
                            // note that we do not use a timeout (see comment above)
                            onFailure(new ElasticsearchTimeoutException("timed out waiting for mapping updates (timeout [" + timeout + "])"));
                        }
                    });
                }
            }
            channel.sendResponse(TransportResponse.Empty.INSTANCE);

        }
    }

    class FilesInfoRequestHandler implements TransportRequestHandler<RecoveryFilesInfoRequest> {

        @Override
        public void messageReceived(RecoveryFilesInfoRequest request, TransportChannel channel) throws Exception {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
                final RecoveryStatus recoveryStatus = statusRef.status();
                final RecoveryState.Index index = recoveryStatus.state().getIndex();
                for (int i = 0; i < request.phase1ExistingFileNames.size(); i++) {
                    index.addFileDetail(request.phase1ExistingFileNames.get(i), request.phase1ExistingFileSizes.get(i), true);
                }
                for (int i = 0; i < request.phase1FileNames.size(); i++) {
                    index.addFileDetail(request.phase1FileNames.get(i), request.phase1FileSizes.get(i), false);
                }
                recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps);
                recoveryStatus.state().getTranslog().totalOperationsOnStart(request.totalTranslogOps);
                // recoveryBytesCount / recoveryFileCount will be set as we go...
                channel.sendResponse(TransportResponse.Empty.INSTANCE);
            }
        }
    }

    class CleanFilesRequestHandler implements TransportRequestHandler<RecoveryCleanFilesRequest> {

        @Override
        public void messageReceived(RecoveryCleanFilesRequest request, TransportChannel channel) throws Exception {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
                final RecoveryStatus recoveryStatus = statusRef.status();
                recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps());
                // first, we go and move files that were created with the recovery id suffix to
                // the actual names, its ok if we have a corrupted index here, since we have replicas
                // to recover from in case of a full cluster shutdown just when this code executes...
                recoveryStatus.indexShard().deleteShardState(); // we have to delete it first since even if we fail to rename the shard might be invalid
                recoveryStatus.renameAllTempFiles();
                final Store store = recoveryStatus.store();
                // now write checksums
                recoveryStatus.legacyChecksums().write(store);
                Store.MetadataSnapshot sourceMetaData = request.sourceMetaSnapshot();
                try {
                    store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetaData);
                } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) {
                    // this is a fatal exception at this stage.
                    // this means we transferred files from the remote that have not be checksummed and they are
                    // broken. We have to clean up this shard entirely, remove all files and bubble it up to the
                    // source shard since this index might be broken there as well? The Source can handle this and checks
                    // its content on disk if possible.
                    try {
                        try {
                            store.removeCorruptionMarker();
                        } finally {
                            Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files
                        }
                    } catch (Throwable e) {
                        logger.debug("Failed to clean lucene index", e);
                        ex.addSuppressed(e);
                    }
                    throw new RecoveryFailedException(recoveryStatus.state(), "failed to clean after recovery", ex);
                } catch (Exception ex) {
                    throw new RecoveryFailedException(recoveryStatus.state(), "failed to clean after recovery", ex);
                }
                channel.sendResponse(TransportResponse.Empty.INSTANCE);
            }
        }
    }

    class FileChunkTransportRequestHandler implements TransportRequestHandler<RecoveryFileChunkRequest> {

        // How many bytes we've copied since we last called RateLimiter.pause
        final AtomicLong bytesSinceLastPause = new AtomicLong();

        @Override
        public void messageReceived(final RecoveryFileChunkRequest request, TransportChannel channel) throws Exception {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
                final RecoveryStatus recoveryStatus = statusRef.status();
                final Store store = recoveryStatus.store();
                recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps());
                final RecoveryState.Index indexState = recoveryStatus.state().getIndex();
                if (request.sourceThrottleTimeInNanos() != RecoveryState.Index.UNKNOWN) {
                    indexState.addSourceThrottling(request.sourceThrottleTimeInNanos());
                }
                IndexOutput indexOutput;
                if (request.position() == 0) {
                    indexOutput = recoveryStatus.openAndPutIndexOutput(request.name(), request.metadata(), store);
                } else {
                    indexOutput = recoveryStatus.getOpenIndexOutput(request.name());
                }
                BytesReference content = request.content();
                if (!content.hasArray()) {
                    content = content.toBytesArray();
                }
                RateLimiter rl = recoverySettings.rateLimiter();
                if (rl != null) {
                    long bytes = bytesSinceLastPause.addAndGet(content.length());
                    if (bytes > rl.getMinPauseCheckBytes()) {
                        // Time to pause
                        bytesSinceLastPause.addAndGet(-bytes);
                        long throttleTimeInNanos = rl.pause(bytes);
                        indexState.addTargetThrottling(throttleTimeInNanos);
                        recoveryStatus.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos);
                    }
                }
                indexOutput.writeBytes(content.array(), content.arrayOffset(), content.length());
                indexState.addRecoveredBytesToFile(request.name(), content.length());
                if (indexOutput.getFilePointer() >= request.length() || request.lastChunk()) {
                    try {
                        Store.verify(indexOutput);
                    } finally {
                        // we are done
                        indexOutput.close();
                    }
                    // write the checksum
                    recoveryStatus.legacyChecksums().add(request.metadata());
                    final String temporaryFileName = recoveryStatus.getTempNameForFile(request.name());
                    assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName);
                    store.directory().sync(Collections.singleton(temporaryFileName));
                    IndexOutput remove = recoveryStatus.removeOpenIndexOutputs(request.name());
                    assert remove == null || remove == indexOutput; // remove maybe null if we got finished
                }
            }
            channel.sendResponse(TransportResponse.Empty.INSTANCE);
        }
    }

    class RecoveryRunner extends AbstractRunnable {

        final long recoveryId;

        RecoveryRunner(long recoveryId) {
            this.recoveryId = recoveryId;
        }

        @Override
        public void onFailure(Throwable t) {
            try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatus(recoveryId)) {
                if (statusRef != null) {
                    logger.error("unexpected error during recovery [{}], failing shard", t, recoveryId);
                    onGoingRecoveries.failRecovery(recoveryId,
                            new RecoveryFailedException(statusRef.status().state(), "unexpected error", t),
                            true // be safe
                    );
                } else {
                    logger.debug("unexpected error during recovery, but recovery id [{}] is finished", t, recoveryId);
                }
            }
        }

        @Override
        public void doRun() {
            RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatus(recoveryId);
            if (statusRef == null) {
                logger.trace("not running recovery with id [{}] - can't find it (probably finished)", recoveryId);
                return;
            }
            try {
                doRecovery(statusRef.status());
            } finally {
                statusRef.close();
            }
        }
    }

}


text:
=========================================
Commandlinefu presentation at Confoo 2011
=========================================
-------------------------------------------------------------
Link to slides available here :: commandlinefu, presentations
-------------------------------------------------------------

The presentation I gave at `Confoo 2011`_ is now `available online`_, there is
feedback on `joind.in`_.

.. _`Confoo 2011`: http://confoo.ca/en
.. _`available online`: http://presentations.codeinthehole.com/confoo2011/
.. _`joind.in`: http://joind.in/talk/view/2896

.. image:: /static/images/clf-presentation.png
    :width: 800px

The slides are written in `Eric Meyer's S5`_, which is a XHTML, CSS and Javascript
based slide show format. I enjoyed using this as it was easy to port my styles
across from the website CSS file, it also means the presentation is viewable as
a normal webpage (no flash required). However, the downside is that it is not
easy to submit the slides to Slideshare which is the common thing to do. There
are a few work-arounds out there but nothing good enough that I could find.

.. _`Eric Meyer's S5`: http://meyerweb.com/eric/tools/s5/


text:
var _mainWin, apps = []

document.addEventListener("DOMContentLoaded", function(){
    init();
});
//------

function init(){
    try{
        fin.desktop.main(function(){
            initWithOpenFin();
        })
    }catch(err){
        initNoOpenFin();
    }
};

function initWithOpenFin(){
    // NB it is 'Window' not 'Application' that the EventListener is being attached to
    _mainWin = fin.desktop.Window.getCurrent();

    document.querySelector("#min-btt").addEventListener('click', function(e){
        minAll()
    });

    document.querySelector("#restore-btt").addEventListener('click', function(e){
        restorAll()
    });

    document.querySelector("#max-btt").addEventListener('click', function(e){
        maxAll();
    });

    _mainWin.addEventListener('close-requested', function(e) {
        var challenge = confirm('are you sure? This will terminate all child applications.');
        if (challenge == true) {
            terminateAllApps();
            _mainWin.close(true);
        }else{
            console.log("The confirm was false")
        }
    });
//create an new app
    initNewApp("BGCIROVolumeMatch").then(function(value){
        var _childWin = value.getWindow()
        _childWin.addEventListener('close-requested', function(e){
            console.log("close requested, but blocked. Close me from the main app.");
            _childWin.minimize();
        });
        apps.push(value);
    });
// and a second - for good measure...
    initNewApp("BGCIROVolumeMatch2").then(function(value){
        var _childWin2 = value.getWindow()
        _childWin2.addEventListener('close-requested', function(e){
            console.log("close requested, but blocked. Close me from the main app.");
            _childWin2.minimize();
        });
        apps.push(value);
    });

    // and a third - for even better measure...
    initNewApp("BGCIROVolumeMatch3").then(function(value){
        var _childWin3 = value.getWindow()
        _childWin3.addEventListener('close-requested', function(e){
            console.log("close requested, but blocked. Close me from the main app.");
            _childWin3.minimize();
        });
        apps.push(value);
    });
}

function initNoOpenFin(){
    alert("OpenFin is not available - you are probably running in a browser.");
}

function terminateAllApps(){
    for(var app in apps ){
        apps[app].terminate();
    }
}

function minAll(){
    for(var app in apps ){
        apps[app].getWindow().minimize();
    }
}

function restorAll(){
    for(var app in apps ){
        apps[app].getWindow().restore();
    }
}

function maxAll(){
    for(var app in apps ){
        apps[app].getWindow().maximize();
    }
}

function initNewApp(uuid){
    return new Promise(function(resolve, reject){
        var volumeMatchApplication = new fin.desktop.Application({
            name: "BGC IRO Volume Match",
            uuid: uuid,
            url: "http://localhost:3030/child.html",
            mainWindowOptions: {
                name: "BGC IRO Volume Match",
                autoShow: false,
                defaultCentered: true,
                alwaysOnTop: false,
                state: "minimized",
                windowState: "minimized",
                saveWindowState: false,
                icon: "favicon.ico"
            }
        }, function () {
            // Ensure the Volume Match application is closed when the main application is closed.
            console.log("running");
            volumeMatchApplication.run();
            resolve(volumeMatchApplication)
        });
    });
}


text:
import os
import numpy as np
import chainer
import chainer.cuda
from chainer import cuda, serializers, Variable
from chainer import training
import chainer.functions as F
import argparse

import sys
sys.path.insert(0, '../')
from common.models.generators import *
from common.utils import *

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='gan analogy testing script')
    parser.add_argument('--gpu', '-g', type=int, default=0, help='GPU ID (negative value indicates CPU)')
    parser.add_argument('--gen_class', '-c', default='', help='Default generator class')
    parser.add_argument("--load_gen_model", '-l', default='', help='load generator model')
    parser.add_argument('--out', '-o', default='output.jpg', help='output image path')
    parser.add_argument("--latent_len", type=int, default=128, help='latent vector length')
    parser.add_argument("--image_channels", type=int, default=3, help='number of image channels')

    args = parser.parse_args()
    print(args)

    if args.gen_class != '':
        gen = eval(args.gen_class)
    else:
        gen = DCGANGenerator(latent=args.latent_len, out_ch=args.image_channels)

    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
    serializers.load_npz(args.load_gen_model, gen)
    print("Generator model loaded")

    if args.gpu >= 0:
        gen.to_gpu()
        print("use gpu {}".format(args.gpu))
    analogy(gen, args.out, latent_len=args.latent_len)


text:
package com.android.server.location;

import android.content.Context;
import android.location.Address;
import android.location.GeocoderParams;
import android.location.IGeocodeProvider;
import android.os.Handler;
import android.os.RemoteException;
import android.os.UserHandle;
import android.util.Log;

import com.android.server.ServiceWatcher;
import java.util.List;

/**
 * Proxy for IGeocodeProvider implementations.
 */
public class GeocoderProxy {
    private static final String TAG = "GeocoderProxy";

    private static final String SERVICE_ACTION = "com.android.location.service.GeocodeProvider";

    private final Context mContext;
    private final ServiceWatcher mServiceWatcher;

    public static GeocoderProxy createAndBind(Context context,
            int overlaySwitchResId, int defaultServicePackageNameResId,
            int initialPackageNamesResId, Handler handler) {
        GeocoderProxy proxy = new GeocoderProxy(context, overlaySwitchResId,
            defaultServicePackageNameResId, initialPackageNamesResId, handler);
        if (proxy.bind()) {
            return proxy;
        } else {
            return null;
        }
    }

    private GeocoderProxy(Context context,
            int overlaySwitchResId, int defaultServicePackageNameResId,
            int initialPackageNamesResId, Handler handler) {
        mContext = context;

        mServiceWatcher = new ServiceWatcher(mContext, TAG, SERVICE_ACTION, overlaySwitchResId,
            defaultServicePackageNameResId, initialPackageNamesResId, null, handler);
    }

    private boolean bind () {
        return mServiceWatcher.start();
    }

    private IGeocodeProvider getService() {
        return IGeocodeProvider.Stub.asInterface(mServiceWatcher.getBinder());
    }

    public String getConnectedPackageName() {
        return mServiceWatcher.getBestPackageName();
    }

    public String getFromLocation(double latitude, double longitude, int maxResults,
            GeocoderParams params, List<Address> addrs) {
        IGeocodeProvider provider = getService();
        if (provider != null) {
            try {
                return provider.getFromLocation(latitude, longitude, maxResults, params, addrs);
            } catch (RemoteException e) {
                Log.w(TAG, e);
            }
        }
        return "Service not Available";
    }

    public String getFromLocationName(String locationName,
            double lowerLeftLatitude, double lowerLeftLongitude,
            double upperRightLatitude, double upperRightLongitude, int maxResults,
            GeocoderParams params, List<Address> addrs) {
        IGeocodeProvider provider = getService();
        if (provider != null) {
            try {
                return provider.getFromLocationName(locationName, lowerLeftLatitude,
                        lowerLeftLongitude, upperRightLatitude, upperRightLongitude,
                        maxResults, params, addrs);
            } catch (RemoteException e) {
                Log.w(TAG, e);
            }
        }
        return "Service not Available";
    }

}


text:
[![Linux and Mac Build Status](https://travis-ci.org/google/shaderc.svg)](https://travis-ci.org/google/shaderc "Linux and Mac Build Status")
[![Windows Build status](https://ci.appveyor.com/api/projects/status/g6c372blna7vnk1l?svg=true)](https://ci.appveyor.com/project/dneto0/shaderc "Windows Build Status")
[![Coverage Status](https://coveralls.io/repos/google/shaderc/badge.svg?branch=master&service=github)](https://coveralls.io/github/google/shaderc?branch=master)

A collection of tools, libraries and tests for shader compilation.
At the moment it includes:

- `glslc`, a command line compiler for GLSL to SPIR-V, and
- `libshaderc` a library API for doing the same.

## Status

Shaderc is experimental, and subject to significant incompatible changes.

For licensing terms, please see the [`LICENSE`](LICENSE) file.  If interested in
contributing to this project, please see [`CONTRIBUTING.md`](CONTRIBUTING.md)

This is not an official Google product (experimental or otherwise), it is just
code that happens to be owned by Google.  That may change if Shaderc gains
contributions from others.  See the [`CONTRIBUTING.md`](CONTRIBUTING.md) file
for more information. See also the [`AUTHORS`](AUTHORS) and
[`CONTRIBUTORS`](CONTRIBUTORS) files.

## File organization

- `android_test/` : a small Android application to verify compilation
- `cmake/`: CMake utility functions and configuration for Shaderc
- `examples/`: Example programs
- `glslc/`: an executable to compile GLSL to SPIR-V
- `libshaderc/`: a library for compiling shader strings into SPIR-V
- `libshaderc_util/`: a utility library used by multiple shaderc components
- `third_party/`: third party open source packages; see below
- `utils/`: utility scripts for Shaderc

Shaderc depends on `glslang`, the Khronos reference compiler for GLSL.
Sometimes a change updates both Shaderc and glslang.  In that case the
glslang change will appear in [google/glslang](https://github.com/google/glslang)
before it appears upstream in
[KhronosGroup/glslang](https://github.com/KhronosGroup/glslang).
We intend to upstream all changes to glslang. We maintain the separate
copy only to stage those changes for review, and to provide something for
Shaderc to build against in the meantime.  Please see
[DEVELOPMENT.howto.md](DEVELOPMENT.howto.md) for more details.

Shaderc depends on [SPIRV-Tools](https://github.com/KhronosGroup/SPIRV-Tools)
for assembling and disassembling SPIR-V binaries.

Shaderc depends on the [Google Test](https://github.com/google/googletest)
testing framework.

In the following sections, `$SOURCE_DIR` is the directory you intend to clone
Shaderc into.

## Getting and building Shaderc

1) Check out the source code:

```sh
git clone https://github.com/google/shaderc $SOURCE_DIR
cd $SOURCE_DIR/third_party
git clone https://github.com/google/googletest.git
git clone https://github.com/google/glslang.git
git clone https://github.com/KhronosGroup/SPIRV-Tools.git spirv-tools
cd $SOURCE_DIR/
```

2) Ensure you have the requisite tools -- see the tools subsection below.

3) Decide where to place the build output. In the following steps, we'll call it
   `$BUILD_DIR`. Any new directory should work. We recommend building outside
   the source tree, but it is also common to build in a (new) subdirectory of
   `$SOURCE_DIR`, such as `$SOURCE_DIR/build`.

4a) Build (and test) with Ninja on Linux or Windows:

```sh
cd $BUILD_DIR
cmake -GNinja -DCMAKE_BUILD_TYPE={Debug|Release|RelWithDebInfo} $SOURCE_DIR
ninja
ctest # optional
```

4b) Or build (and test) with MSVC on Windows:

```sh
cd $BUILD_DIR
cmake $SOURCE_DIR
cmake --build . --config {Release|Debug|MinSizeRel|RelWithDebInfo}
ctest -C {Release|Debug|MinSizeRel|RelWithDebInfo}
```

4c) Or build with MinGW on Linux for Windows:
(Skip building threaded unit tests due to
[Googletest bug 606](https://github.com/google/googletest/issues/606))

```sh
cd $BUILD_DIR
cmake -GNinja -DCMAKE_BUILD_TYPE={Debug|Release|RelWithDebInfo} $SOURCE_DIR \
   -DCMAKE_TOOLCHAIN_FILE=$SOURCE_DIR/cmake/linux-mingw-toolchain.cmake \
   -Dgtest_disable_pthreads=ON
ninja
```

After a successful build, you should have a `glslc` executable somewhere under
the `$BUILD_DIR/glslc/` directory, as well as a `libshaderc` library somewhere
under the `$BUILD_DIR/libshaderc/` directory.

The default behavior on MSVC is to link with the static CRT. If you would like
to change this behavior `-DSHADERC_ENABLE_SHARED_CRT` may be passed on the
cmake configure line.

See [the libshaderc README](libshaderc/README.md) for more on using the library
API in your project.

### Tools you'll need

For building, testing, and profiling Shaderc, the following tools should be
installed regardless of your OS:

- [CMake](http://www.cmake.org/): for generating compilation targets.
- [Python](http://www.python.org/): for utility scripts and running the test suite.

On Linux, the following tools should be installed:

- [`gcov`](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): for testing code
    coverage, provided by the `gcc` package on Ubuntu.
- [`lcov`](http://ltp.sourceforge.net/coverage/lcov.php): a graphical frontend
    for `gcov`, provided by the `lcov` package on Ubuntu.
- [`genhtml`](http://linux.die.net/man/1/genhtml): for creating reports in html
    format from `lcov` output, provided by the `lcov` package on Ubuntu.

On Linux, if cross compiling to Windows:
- [`mingw`](http://www.mingw.org): A GCC-based cross compiler targeting Windows
    so that generated executables use the Micrsoft C runtime libraries.

On Windows, the following tools should be installed and available on your path:

- Visual Studio 2013 Update 4 or later. Previous versions of Visual Studio
  will likely work but are untested.
- Git - including the associated tools, Bash, `diff`.

Optionally, the following tools may be installed on any OS:

 - [`asciidoctor`](http://asciidoctor.org/): for generating documenation.
   - [`pygments.rb`](https://rubygems.org/gems/pygments.rb) required by
     `asciidoctor` for syntax highlighting.
 - [`nosetests`](https://nose.readthedocs.org): for testing the Python code.

### Building and running Shderc using Docker

Please make sure you have the Docker engine
[installed](https://docs.docker.com/engine/installation/) on your machine.

To create a Docker image containing Shaderc command line tools, issue the
following command in `${SOURCE_DIR}`: `docker build -t <IMAGE-NAME> .`.
The created image will have all the command line tools installed at
`/usr/local` interally, and a data volume mounted at `/code`.

Assume `<IMAGE-NAME>` is `shaderc/shaderc` from now on.

To invoke a tool from the above created image in a Docker container:

```bash
docker run shaderc/shaderc glslc --version
```

Alternatively, you can mount a host directory (e.g., `example`) containing
the shaders you want to manipulate and run different kinds of tools via
an interactive shell in the container:

```bash
$ docker run -i -t -v `pwd`/example:/code shaderc/shaderc
/code $ ls
test.vert
/code $ glslc -c -o - test.vert | spirv-dis
```

## Bug tracking

We track bugs using GitHub -- click on the "Issues" button on
[the project's GitHub page](https://github.com/google/shaderc).

## Test coverage

On Linux, you can obtain test coverage as follows:

```sh
cd $BUILD_DIR
cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_CODE_COVERAGE=ON $SOURCE_DIR
ninja
ninja report-coverage
```

Then the coverage report can be found under the `$BUILD_DIR/coverage-report`
directory.


text:
package io.cdap.wrangler.service.filesystem;

import io.cdap.cdap.api.app.AbstractApplication;
import io.cdap.cdap.api.dataset.lib.FileSet;
import io.cdap.cdap.api.dataset.lib.FileSetProperties;
import io.cdap.wrangler.service.explorer.FilesystemExplorer;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;

public class TestApp extends AbstractApplication {
  /**
   * Override this method to declare and configure the application.
   */
  @Override
  public void configure() {
    setName("dataprep");
    setDescription("DataPrep Backend Service");
    // Used by the file service.
    createDataset("indexds", FileSet.class, FileSetProperties.builder()
      .setBasePath("dataprep/indexds")
      .setInputFormat(TextInputFormat.class)
      .setOutputFormat(TextOutputFormat.class)
      .setDescription("Store Dataset Index files")
      .build());
    addService("service", new FilesystemExplorer());
  }
}


text:
#include "singa/singa_config.h"
#ifdef USE_OPENCV
#include "ilsvrc12.h"
#include "singa/utils/channel.h"
#include "singa/utils/string.h"
int main(int argc, char **argv) {
  int pos = singa::ArgPos(argc, argv, "-h");
  if (pos != -1) {
    std::cout << "Usage:\n"
              << "\t-trainlist <file>: the file of training list;\n"
              << "\t-trainfolder <folder>: the folder of training images;\n"
              << "\t-testlist <file>: the file of test list;\n"
              << "\t-testfolder <floder>: the folder of test images;\n"
              << "\t-outdata <folder>: the folder to save output files;\n"
              << "\t-filesize <int>: number of training images that stores in "
                 "each binary file.\n";
    return 0;
  }
  pos = singa::ArgPos(argc, argv, "-trainlist");
  string train_image_list = "imagenet/label/train.txt";
  if (pos != -1) train_image_list = argv[pos + 1];

  pos = singa::ArgPos(argc, argv, "-trainfolder");
  string train_image_folder = "imagenet/ILSVRC2012_img_train";
  if (pos != -1) train_image_folder = argv[pos + 1];

  pos = singa::ArgPos(argc, argv, "-testlist");
  string test_image_list = "imagenet/label/val.txt";
  if (pos != -1) test_image_list = argv[pos + 1];

  pos = singa::ArgPos(argc, argv, "-testfolder");
  string test_image_folder = "imagenet/ILSVRC2012_img_val";
  if (pos != -1) test_image_folder = argv[pos + 1];

  pos = singa::ArgPos(argc, argv, "-outdata");
  string bin_folder = "imagenet_data";
  if (pos != -1) bin_folder = argv[pos + 1];

  pos = singa::ArgPos(argc, argv, "-filesize");
  size_t train_file_size = 1280;
  if (pos != -1) train_file_size = atoi(argv[pos + 1]);
  singa::ILSVRC data;
  LOG(INFO) << "Creating training and test data...";
  data.CreateTrainData(train_image_list, train_image_folder, bin_folder,
                       train_file_size);
  data.CreateTestData(test_image_list, test_image_folder, bin_folder);
  LOG(INFO) << "Data created!";
  return 0;
}
#endif  // USE_OPENCV


text:
<?php
$app->group("/kb", function () use ($app) {
    // Frontpage
    $app->get("/", function () use ($app) {
        (new \ProjectRena\Controller\Killboard\IndexController($app))->index();
    });

    // Killmail page
    $app->group("/killmail", function () use ($app) {
        $app->get("/:killID/", function () use ($app) {
            (new \ProjectRena\Controller\Killboard\KillmailController($app))->index();
        });
    });

    // Character page
    $app->group("/character", function () use ($app) {
        $app->get("/:character/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "character")["character"][0]["characterID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\CharacterController($app))->index($id);
        });
    });

    // Corporation page
    $app->group("/corporation", function () use ($app) {
        $app->get("/:corporation/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "corporation")["corporation"][0]["corporationID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\CorporationController($app))->index($id);
        });
    });

    // Alliance page
    $app->group("/alliance", function () use ($app) {
        $app->get("/:alliance/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "alliance")["alliance"][0]["allianceID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\AllianceController($app))->index($id);
        });
    });

    // Faction page
    $app->group("/faction", function () use ($app) {
        $app->get("/:faction/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "faction")["faction"][0]["factionID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\FactionController($app))->index($id);
        });
    });

    // System page
    $app->group("/system", function () use ($app) {
        $app->get("/:system/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "system")["system"][0]["solarSystemID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\SystemController($app))->index($id);
        });
    });

    // Region page
    $app->group("/region", function () use ($app) {
        $app->get("/:region/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "region")["region"][0]["regionID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\RegionController($app))->index($id);
        });
    });

    // Ship page
    $app->group("/ship", function () use ($app) {
        $app->get("/:ship/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "item")["item"][0]["typeID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\ShipController($app))->index($id);
        });
    });

    // Group page
    $app->group("/group", function () use ($app) {
        $app->get("/:group/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "item")["item"][0]["typeID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\GroupController($app))->index($id);
        });
    });

    // Weapon page
    $app->group("/weapon", function () use ($app) {
        $app->get("/:weapon/", function ($entity) use ($app) {
            if (!is_numeric($entity))
                $id = $app->Search->search($entity, "item")["item"][0]["typeID"];
            else
                $id = $entity;

            if ($id == 0 || $id == null)
                $app->redirect("/kb/", 302);

            (new \ProjectRena\Controller\Killboard\WeaponController($app))->index($id);
        });
    });
});


text:
using System.Text;
using System.Web;
using System.Web.Services;
using WebsitePanel.Providers.ResultObjects;

namespace WebsitePanel.Portal.VPS
{
    /// <summary>
    /// Summary description for $codebehindclassname$
    /// </summary>
    [WebService(Namespace = "http://tempuri.org/")]
    [WebServiceBinding(ConformsTo = WsiProfiles.BasicProfile1_1)]
    public class VirtualMachineImage : IHttpHandler
    {

        public void ProcessRequest(HttpContext context)
        {
            context.Response.Clear();
            context.Response.ContentType = "image/png";

            int itemId =  Utils.ParseInt(context.Request.QueryString["ItemID"]);
            byte[] buffer = ES.Services.VPS.GetVirtualMachineThumbnail(itemId,
                WebsitePanel.Providers.Virtualization.ThumbnailSize.Medium160x120);
            if (buffer != null)
            {
                context.Response.OutputStream.Write(buffer, 0, buffer.Length);
            }
        }

        public bool IsReusable
        {
            get
            {
                return false;
            }
        }
    }
}