text:
package rx.operators;

import java.util.LinkedList;
import java.util.Queue;
import rx.Subscriber;
import rx.subscriptions.CompositeSubscription;

/**
 * Buffers the incoming events until notified, then replays the
 * buffered events and continues as a simple pass-through subscriber.
 * @param <T> the streamed value type
 */
public class BufferUntilSubscriber<T> extends Subscriber<T> {
    /** The actual subscriber. */
    private final Subscriber<? super T> actual;
    /** Indicate the pass-through mode. */
    private volatile boolean passthroughMode;
    /** Protect mode transition. */
    private final Object gate = new Object();
    /** The buffered items. */
    private final Queue<Object> queue = new LinkedList<Object>();
    /** The queue capacity. */
    private final int capacity;
    /** Null sentinel (in case queue type is changed). */
    private static final Object NULL_SENTINEL = new Object();
    /** Complete sentinel. */
    private static final Object COMPLETE_SENTINEL = new Object();
    /**
     * Container for an onError event.
     */
    private static final class ErrorSentinel {
        final Throwable t;

        public ErrorSentinel(Throwable t) {
            this.t = t;
        }

    }
    /**
     * Constructor that wraps the actual subscriber and shares its subscription.
     * @param capacity the queue capacity to accept before blocking, negative value indicates an unbounded queue
     * @param actual
     */
    public BufferUntilSubscriber(int capacity, Subscriber<? super T> actual) {
        super(actual);
        this.actual = actual;
        this.capacity = capacity;
    }
    /**
     * Constructor that wraps the actual subscriber and uses the given composite
     * subscription.
     * @param capacity the queue capacity to accept before blocking, negative value indicates an unbounded queue
     * @param actual
     * @param cs
     */
    public BufferUntilSubscriber(int capacity, Subscriber<? super T> actual, CompositeSubscription cs) {
        super(cs);
        this.actual = actual;
        this.capacity = capacity;
    }

    /**
     * Call this method to replay the buffered events and continue as a pass-through subscriber.
     * If already in pass-through mode, this method is a no-op.
     */
    public void enterPassthroughMode() {
        if (!passthroughMode) {
            synchronized (gate) {
                if (!passthroughMode) {
                    while (!queue.isEmpty()) {
                        Object o = queue.poll();
                        if (!actual.isUnsubscribed()) {
                            if (o == NULL_SENTINEL) {
                                actual.onNext(null);
                            } else
                            if (o == COMPLETE_SENTINEL) {
                                actual.onCompleted();
                            } else
                            if (o instanceof ErrorSentinel) {
                                actual.onError(((ErrorSentinel)o).t);
                            } else
                            if (o != null) {
                                @SuppressWarnings("unchecked")
                                T v = (T)o;
                                actual.onNext(v);
                            } else {
                                throw new NullPointerException();
                            }
                        }
                    }
                    passthroughMode = true;
                    gate.notifyAll();
                }
            }
        }
    }
    @Override
    public void onNext(T t) {
        if (!passthroughMode) {
            synchronized (gate) {
                if (!passthroughMode) {
                    if (capacity < 0 || queue.size() < capacity) {
                        queue.offer(t != null ? t : NULL_SENTINEL);
                        return;
                    }
                    try {
                        while (!passthroughMode) {
                            gate.wait();
                        }
                        if (actual.isUnsubscribed()) {
                            return;
                        }
                    } catch (InterruptedException ex) {
                        Thread.currentThread().interrupt();
                        actual.onError(ex);
                        return;
                    }
                }
            }
        }
        actual.onNext(t);
    }

    @Override
    public void onError(Throwable e) {
        if (!passthroughMode) {
            synchronized (gate) {
                if (!passthroughMode) {
                    if (capacity < 0 || queue.size() < capacity) {
                        queue.offer(new ErrorSentinel(e));
                        return;
                    }
                    try {
                        while (!passthroughMode) {
                            gate.wait();
                        }
                        if (actual.isUnsubscribed()) {
                            return;
                        }
                    } catch (InterruptedException ex) {
                        Thread.currentThread().interrupt();
                        actual.onError(ex);
                        return;
                    }
                }
            }
        }
        actual.onError(e);
    }

    @Override
    public void onCompleted() {
        if (!passthroughMode) {
            synchronized (gate) {
                if (!passthroughMode) {
                    if (capacity < 0 || queue.size() < capacity) {
                        queue.offer(COMPLETE_SENTINEL);
                        return;
                    }
                    try {
                        while (!passthroughMode) {
                            gate.wait();
                        }
                        if (actual.isUnsubscribed()) {
                            return;
                        }
                    } catch (InterruptedException ex) {
                        Thread.currentThread().interrupt();
                        actual.onError(ex);
                        return;
                    }
                }
            }
        }
        actual.onCompleted();
    }

}


text:
import * as React from "react"

export interface JsonEditorProps<T> {
  value: T
  setValue: (value: T) => void
  style?: React.CSSProperties
  readOnly?: boolean
}

export const controlStyle: React.CSSProperties = {
  outline: 0,
  padding: '0.375rem 0.75rem',
  fontSize: '1rem',
  fontWeight: 400,
  fontFamily: 'monospace',
  lineHeight: 1.5,
  color: '#212529',
  backgroundColor: '#fff',
  backgroundClip: 'padding-box',
  border: '1px solid #ced4da',
  appearance: 'none',
  borderRadius: '0.25rem',
  flex: 1,
  marginBottom: '3px',
}

export const buttonStyle: React.CSSProperties = {
  display: 'inline-block',
  fontWeight: 400,
  lineHeight: 1,
  color: '#212529',
  textAlign: 'center',
  textDecoration: 'none',
  verticalAlign: 'middle',
  cursor: 'pointer',
  userSelect: 'none',
  backgroundColor: 'transparent',
  border: '1px solid transparent',
  padding: '0.375rem 0.75rem',
  fontSize: '1rem',
  borderRadius: '0.25rem',
}

export const groupStyle: React.CSSProperties = {
  padding: '10px',
  border: '1px solid rgba(0,0,0,.125)',
  borderRadius: '0.25rem',
  flex: 1,
  marginBottom: '3px',
}

redpajama_set_name: RedPajamaGithub


text:
    <meta charset="utf-8">
    <title>Simple test page</title>
    <link rel="stylesheet" href="main.css">

Hello world!

Loading


text:
maintainer       "Intuit, Inc."
maintainer_email "brett_weaver@intuit.com"
description      "Installs sdbport gem"
version          "0.1.0"
depends          "ruby"

%w{ redhat }.each do |os|
  supports os
end


text:
<?php

namespace Antvel\Tests\Unit\Features;

use Antvel\Tests\TestCase;
use Antvel\Features\ValidationRulesParser;

class FeaturesValidationRulesParserTest extends TestCase
{
	/** @test */
	function it_parses_the_feature_input_validation_rules()
	{
		$rules = ValidationRulesParser::parse([
			'name' => 'foo',
			'required' => 1,
			'max' => 20,
			'min' => 10
		]);

		$this->assertTrue(is_string($rules->toString()));
		$this->assertEquals('required|max:20|min:10', $rules->toString());
	}

	/** @test */
	function it_retrieves_the_feature_validations_rules()
	{
		$rules = ValidationRulesParser::decode('required|max:20|min:10');

		$this->assertTrue($rules->all()->contains('required'));
		$this->assertTrue($rules->all()->contains('max:20'));
		$this->assertTrue($rules->all()->contains('min:10'));
		$this->assertEquals('required|max:20|min:10', $rules->toString());
	}

	/** @test */
	function it_is_able_to_expose_the_allowed_validation_rules()
	{
		$allowed = ValidationRulesParser::allowed();

		$this->assertTrue($allowed->contains('required'));
		$this->assertTrue($allowed->contains('max'));
		$this->assertTrue($allowed->contains('min'));
		$this->assertTrue($allowed->count() > 0);
	}

	/** @test */
	function it_can_parse_a_null_rules()
	{
	    $rules = ValidationRulesParser::parse(null);

	   	$this->assertCount(0, $rules->all());
	    $this->assertNull($rules->toString());
	}

	/** @test */
	function it_can_decode_a_false_rules()
	{
		$rules = ValidationRulesParser::decode('');

	    $this->assertNull($rules->toString());
	    $this->assertCount(0, $rules->all());
	}
}


text:
GkGameAnimationsSoundBinding::GkGameAnimationsSoundBinding(GkSoundSource *soundSource, GkSoundDevice *soundDevice, bool looping, int numAnimations, ...)
	: GkGameEntity()
{
	this->soundSource = soundSource;
	this->soundDevice = soundDevice;
	this->looping = looping;

	this->numAnimations = numAnimations;
	GkGameAnimation **parameters = (GkGameAnimation **)((int)&numAnimations + 4);
	this->animations = new GkGameAnimationsSoundBindingAnimation[numAnimations];
	for (int i = 0; i < numAnimations; i++)
	{
		this->animations[i].animation = parameters[i];
		this->animations[i].played = false;
	}
}

GkGameAnimationsSoundBinding::~GkGameAnimationsSoundBinding(void)
{
	delete[] this->animations;
}

void GkGameAnimationsSoundBinding::Process(GkGame *game)
{
	for (int i = 0; i < this->numAnimations; i++)
	{
		if (this->animations[i].animation->visible && this->animations[i].animation->active)
		{
			if (!this->animations[i].played)
			{
				if (this->looping)
					this->soundDevice->PlaySourceLooping(this->soundSource);
				else
					this->soundDevice->PlaySource(this->soundSource);

				this->animations[i].played = true;
			}
		}
		else
		{
			if (this->looping)
			{
				this->soundDevice->PauseSource(this->soundSource);
			}

			this->animations[i].played = false;
		}
	}
}


text:
package com.linkedin.pinot.transport.common;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.util.concurrent.ImmediateExecutor;

import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;

import com.linkedin.pinot.Checkable;
import com.linkedin.pinot.TestUtils;
import com.linkedin.pinot.common.response.ServerInstance;

public class ResponseFutureTest {
  protected static Logger LOGGER = LoggerFactory.getLogger(ResponseFutureTest.class);

  /*
   * Future Handle provided to the request sender to asynchronously wait for response.
   * We use guava API for implementing Futures.
   */
  public static class ResponseFuture extends AsyncResponseFuture<ServerInstance, ByteBuf> {

    public ResponseFuture(ServerInstance key) {
      super(key, "");
    }

  }

  @Test
  public void testResponseFutureOtherCases() throws Exception {
    ServerInstance s = new ServerInstance("localhost", 8080);

    // A Future is cancelled but gets a message after that. get() called before cancellation
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      f.cancel(false);
      String message = "dummy Message";
      setResponse(f, message);
      runner.waitForDone();
      Assert.assertTrue(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // A Future is cancelled but gets a message after that. get() called after cancellation
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      f.cancel(false);
      String message = "dummy Message";
      setResponse(f, message);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      runner.waitForDone();
      Assert.assertTrue(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // A Future is cancelled but gets an exception after that. get() called before cancellation
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      f.cancel(false);
      f.onError(new Exception("dummy"));
      runner.waitForDone();
      Assert.assertTrue(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // A Future is cancelled but gets an exception after that. get() called after cancellation
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      f.cancel(false);
      f.onError(new Exception("dummy"));
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      runner.waitForDone();
      Assert.assertTrue(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // Throw Exception. Then we try to cancel. Future Client calls get() before exception
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      f.cancel(false);
      runner.waitForDone();
      // Now we know listener executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertEquals(runner.getError(), expectedError, "Error");
      executor.shutdown();
    }

    // Throw Exception. Then we try to cancel. Future Client calls get() after exception
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      f.cancel(false);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      runner.waitForDone();
      // Now we know listener executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertEquals(runner.getError(), expectedError, "Error");
      executor.shutdown();
    }

    // Set Response. Then we try to cancel. Future Client calls get() before response
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      String message = "dummy Message";
      setResponse(f, message);
      f.cancel(false);
      runner.waitForDone();
      // Now we know runner executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertEquals(runner.getMessage(), message, "Response Check:");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // Set Response. Then we try to cancel. Future Client calls get() after response
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      String message = "dummy Message";
      setResponse(f, message);
      f.cancel(false);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      runner.waitForDone();
      // Now we know runner executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertEquals(runner.getMessage(), message, "Response Check:");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // Set Response. Then an exception happens. Future Client calls get() before response
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      String message = "dummy Message";
      setResponse(f, message);
      f.onError(new Exception("dummy"));
      runner.waitForDone();
      // Now we know runner executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertEquals(runner.getMessage(), message, "Response Check:");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // Set Response. then an exception gets processed. Future Client calls get() after response
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      String message = "dummy Message";
      setResponse(f, message);
      f.onError(new Exception("dummy"));
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      runner.waitForDone();
      // Now we know runner executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertEquals(runner.getMessage(), message, "Response Check:");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // Throw Exception. Then a valid response arrives. Future Client calls get() before exception
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      String message = "dummy Message";
      setResponse(f, message);
      runner.waitForDone();
      // Now we know listener executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertEquals(runner.getError(), expectedError, "Error");
      executor.shutdown();
    }

    // Throw Exception. Then a valid response arrives. Future Client calls get() after exception
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      String message = "dummy Message";
      setResponse(f, message);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      runner.waitForDone();
      // Now we know listener executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertEquals(runner.getError(), expectedError, "Error");
      executor.shutdown();
    }
  }

  @Test
  public void testResponseFuture() throws Exception {
    ServerInstance s = new ServerInstance("localhost", 8080);
    //Cancelled Future. Future Client calls get() before cancel()
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      f.cancel(false);
      runner.waitForDone();
      Assert.assertTrue(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    //Cancelled Future. Future Client calls get() after cancel()
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      f.cancel(false);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForDone();
      Assert.assertTrue(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // Throw Exception. Future Client calls get() before exception
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      runner.waitForDone();
      // Now we know listener executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertEquals(runner.getError(), expectedError, "Error");
      executor.shutdown();
    }

    // Throw Exception. Future Client calls get() after exception
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForDone();
      // Now we know listener executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertNull(runner.getMessage(), "No Reponse :");
      Assert.assertEquals(runner.getError(), expectedError, "Error");
      executor.shutdown();
    }

    // Set Response. Future Client calls get() before response
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForAboutToGet(); // No guarantees as this only ensures the thread is started but not blocking in get().
      Thread.sleep(100);
      String message = "dummy Message";
      setResponse(f, message);
      runner.waitForDone();
      // Now we know runner executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertEquals(runner.getMessage(), message, "Response Check:");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }

    // Set Response. Future Client calls get() after response
    {
      ResponseFuture f = new ResponseFuture(s);
      ResponseFutureClientRunner runner = new ResponseFutureClientRunner(f);
      String message = "dummy Message";
      setResponse(f, message);
      ThreadPoolExecutor executor =
          new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
      executor.execute(runner);
      runner.waitForDone();
      // Now we know runner executed
      Assert.assertFalse(runner.isCancelled(), "Cancelled ?");
      Assert.assertTrue(runner.isDone(), "Is Done ? ");
      Assert.assertEquals(runner.getMessage(), message, "Response Check:");
      Assert.assertNull(runner.getError(), "No Error :");
      executor.shutdown();
    }
  }

  @Test
  public void testResponseFutureListener() throws Exception {
    ServerInstance s = new ServerInstance("localhost", 8080);

    // Cancelled future. Listener added before cancelling
    {
      ResponseFuture f = new ResponseFuture(s);
      FutureListener listener = new FutureListener(f);
      WaitForCompletion w = new WaitForCompletion(listener);
      f.cancel(false);
      TestUtils.assertWithBackoff(w, 10, 1000, 2, 1, 2);
      // Now we know listener executed
      Assert.assertTrue(listener.isCancelled(), "Cancelled ?");
      Assert.assertEquals(listener.getNumRuns(), 1, "Num Runs of listener");
      Assert.assertTrue(listener.isDone(), "Is Done ? ");
      Assert.assertNull(listener.getMessage(), "No Reponse :");
      Assert.assertNull(listener.getError(), "No Error :");
      listener.close();
    }

    // Cancelled future. Listener added after cancelling
    {
      ResponseFuture f = new ResponseFuture(s);
      FutureListener listener = new FutureListener(f);
      WaitForCompletion w = new WaitForCompletion(listener);
      f.cancel(false);
      TestUtils.assertWithBackoff(w, 10, 1000, 2, 1, 2);
      // Now we know listener executed
      Assert.assertTrue(listener.isCancelled(), "Cancelled ?");
      Assert.assertEquals(listener.getNumRuns(), 1, "Num Runs of listener");
      Assert.assertTrue(listener.isDone(), "Is Done ? ");
      Assert.assertNull(listener.getMessage(), "No Reponse :");
      Assert.assertNull(listener.getError(), "No Error :");
      listener.close();
    }

    // Throw Exception. Listener added before exception
    {
      ResponseFuture f = new ResponseFuture(s);
      FutureListener listener = new FutureListener(f);
      WaitForCompletion w = new WaitForCompletion(listener);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      TestUtils.assertWithBackoff(w, 10, 1000, 2, 1, 2);
      // Now we know listener executed
      Assert.assertFalse(listener.isCancelled(), "Cancelled ?");
      Assert.assertEquals(listener.getNumRuns(), 1, "Num Runs of listener");
      Assert.assertTrue(listener.isDone(), "Is Done ? ");
      Assert.assertNull(listener.getMessage(), "No Reponse :");
      Assert.assertEquals(listener.getError(), expectedError, "Error");
    }

    // Throw Exception. Listener added after exception
    {
      ResponseFuture f = new ResponseFuture(s);
      FutureListener listener = new FutureListener(f);
      Exception expectedError = new Exception("error processing");
      f.onError(expectedError);
      WaitForCompletion w = new WaitForCompletion(listener);
      TestUtils.assertWithBackoff(w, 10, 1000, 2, 1, 2);
      // Now we know listener executed
      Assert.assertFalse(listener.isCancelled(), "Cancelled ?");
      Assert.assertEquals(listener.getNumRuns(), 1, "Num Runs of listener");
      Assert.assertTrue(listener.isDone(), "Is Done ? ");
      Assert.assertNull(listener.getMessage(), "No Reponse :");
      Assert.assertEquals(listener.getError(), expectedError, "Error");
    }

    // Set Response. Listener added before response
    {
      ResponseFuture f = new ResponseFuture(s);
      FutureListener listener = new FutureListener(f);
      WaitForCompletion w = new WaitForCompletion(listener);
      String message = "dummy Message";
      setResponse(f, message);
      TestUtils.assertWithBackoff(w, 10, 1000, 2, 1, 2);
      // Now we know listener executed
      Assert.assertFalse(listener.isCancelled(), "Cancelled ?");
      Assert.assertEquals(listener.getNumRuns(), 1, "Num Runs of listener");
      Assert.assertTrue(listener.isDone(), "Is Done ? ");
      Assert.assertEquals(listener.getMessage(), message, "Response Check:");
      Assert.assertNull(listener.getError(), "No Error :");
      listener.close();
    }

    // Set Response. Listener added after response
    {
      ResponseFuture f = new ResponseFuture(s);
      FutureListener listener = new FutureListener(f);
      String message = "dummy Message";
      setResponse(f, message);
      WaitForCompletion w = new WaitForCompletion(listener);
      TestUtils.assertWithBackoff(w, 10, 1000, 2, 1, 2);
      // Now we know listener executed
      Assert.assertFalse(listener.isCancelled(), "Cancelled ?");
      Assert.assertEquals(listener.getNumRuns(), 1, "Num Runs of listener");
      Assert.assertTrue(listener.isDone(), "Is Done ? ");
      Assert.assertEquals(listener.getMessage(), message, "Response Check:");
      Assert.assertNull(listener.getError(), "No Error :");
      listener.close();
    }
  }

  private static void setResponse(ResponseFuture f, String message) {
    ByteBuf b = Unpooled.wrappedBuffer(message.getBytes());
    f.onSuccess(b);
  }

  private static class WaitForCompletion implements Checkable {
    private final FutureListener _listener;

    public WaitForCompletion(FutureListener f) {
      _listener = f;
    }

    @Override
    public boolean runCheck() throws AssertionError {
      boolean isCalled = false;
      synchronized (_listener) {
        isCalled = _listener.isCalled();
      }
      return isCalled;
    }
  }

  private static class ResponseFutureClientRunner implements Runnable {
    private boolean _isDone;
    private boolean _isCancelled;
    private String _message;
    private Throwable _error;
    private final ResponseFuture _future;
    private final CountDownLatch _latch = new CountDownLatch(1);
    private final CountDownLatch _endLatch = new CountDownLatch(1);

    public ResponseFutureClientRunner(ResponseFuture f) {
      _future = f;
    }

    public void waitForAboutToGet() throws InterruptedException {
      _latch.await();
    }

    public void waitForDone() throws InterruptedException {
      _endLatch.await();
    }

    @Override
    public synchronized void run() {
      LOGGER.info("Running Future runner !!");

      ByteBuf message = null;

      try {
        _latch.countDown();
        message = _future.getOne();
      } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (ExecutionException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }

      _isDone = _future.isDone();
      _isCancelled = _future.isCancelled();

      if (null != message) {
        byte[] m = new byte[message.readableBytes()];
        message.readBytes(m);
        _message = new String(m);
      }

      Map<ServerInstance, Throwable> errorMap = _future.getError();
      if ((null != errorMap) && (!errorMap.isEmpty())) {
        _error = errorMap.values().iterator().next();
      }

      _endLatch.countDown();
      LOGGER.info("End Running Listener !!");
    }

    public boolean isDone() {
      return _isDone;
    }

    public boolean isCancelled() {
      return _isCancelled;
    }

    public String getMessage() {
      return _message;
    }

    public Throwable getError() {
      return _error;
    }

    public ResponseFuture getFuture() {
      return _future;
    }
  }

  private static class FutureListener implements Runnable {
    private int _numRuns = 0;
    private boolean _isCalled;
    private boolean _isDone;
    private boolean _isCancelled;
    private String _message;
    private Throwable _error;
    private final ResponseFuture _future;
    private Executor _executor = null;

    public FutureListener(ResponseFuture f) {
      _future = f;
      LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>();
      queue.add(this);
      //_executor = new ThreadPoolExecutor(1, 1, 1, TimeUnit.HOURS, queue);
      _executor = ImmediateExecutor.INSTANCE;
      f.addListener(this, _executor);
    }

    @Override
    public synchronized void run() {
      _isCalled = true;
      _numRuns++;
      LOGGER.info("Running Listener !!");
      //try{ throw new Exception("dummy"); } catch(Exception e) {e.printStackTrace();}
      _isDone = _future.isDone();
      _isCancelled = _future.isCancelled();

      ByteBuf message = null;

      try {
        message = _future.getOne();
      } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (ExecutionException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }

      if (null != message) {
        byte[] m = new byte[message.readableBytes()];
        message.readBytes(m);
        _message = new String(m);
      }
      Map<ServerInstance, Throwable> errorMap = _future.getError();

      if ((null != errorMap) && (!errorMap.isEmpty())) {
        _error = errorMap.values().iterator().next();
      }

      LOGGER.info("End Running Listener !!");
    }

    public void close() {
      //_executor.shutdown();
    }

    public boolean isCalled() {
      return _isCalled;
    }

    public boolean isDone() {
      return _isDone;
    }

    public boolean isCancelled() {
      return _isCancelled;
    }

    public String getMessage() {
      return _message;
    }

    public Throwable getError() {
      return _error;
    }

    public int getNumRuns() {
      return _numRuns;
    }
  }
}


text:
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals

import io
import pytest
from mock import Mock
from tabulator import Stream
from tabulator.parsers.csv import CSVParser
BASE_URL = 'https://raw.githubusercontent.com/okfn/tabulator-py/master/%s'

# Read

def test_stream_local_csv():
    with Stream('data/table.csv') as stream:
        assert stream.headers is None
        assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

def test_stream_local_csv_with_bom():
    with Stream('data/special/bom.csv') as stream:
        assert stream.headers is None
        assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

def test_stream_local_csv_with_bom_with_encoding():
    with Stream('data/special/bom.csv', encoding='utf-8') as stream:
        assert stream.headers is None
        assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

def test_stream_csv_excel():
    source = 'value1,value2\nvalue3,value4'
    with Stream(source, scheme='text', format='csv') as stream:
        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]

def test_stream_csv_excel_tab():
    source = 'value1\tvalue2\nvalue3\tvalue4'
    with Stream(source, scheme='text', format='csv', delimiter='\t') as stream:
        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]

def test_stream_csv_unix():
    source = '"value1","value2"\n"value3","value4"'
    with Stream(source, scheme='text', format='csv') as stream:
        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]

def test_stream_csv_escaping():
    with Stream('data/special/escaping.csv', escapechar='\\') as stream:
        assert stream.read() == [
            ['ID', 'Test'],
            ['1', 'Test line 1'],
            ['2', 'Test " line 2'],
            ['3', 'Test " line 3'],
        ]

def test_stream_csv_doublequote():
    with Stream('data/special/doublequote.csv') as stream:
        for row in  stream:
            assert len(row) == 17

def test_stream_stream_csv():
    source = io.open('data/table.csv', mode='rb')
    with Stream(source, format='csv') as stream:
        assert stream.headers is None
        assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

def test_stream_text_csv():
    source = 'text://id,name\n1,english\n2,中国人\n'
    with Stream(source, format='csv') as stream:
        assert stream.headers is None
        assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

@pytest.mark.remote
def test_stream_remote_csv():
    with Stream(BASE_URL % 'data/table.csv') as stream:
        assert stream.headers is None
        assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

@pytest.mark.remote
def test_stream_remote_csv_non_ascii_url():
    with Stream('http://data.defra.gov.uk/ops/government_procurement_card/over_£500_GPC_apr_2013.csv') as stream:
        assert stream.sample[0] == [
            'Entity',
            'Transaction Posting Date',
            'Merchant Name',
            'Amount',
            'Description']

def test_stream_csv_delimiter():
    source = '"value1";"value2"\n"value3";"value4"'
    with Stream(source, scheme='text', format='csv', delimiter=';') as stream:
        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]

def test_stream_csv_escapechar():
    source = 'value1%,value2\nvalue3%,value4'
    with Stream(source, scheme='text', format='csv', escapechar='%') as stream:
        assert stream.read() == [['value1,value2'], ['value3,value4']]

def test_stream_csv_quotechar():
    source = '%value1,value2%\n%value3,value4%'
    with Stream(source, scheme='text', format='csv', quotechar='%') as stream:
        assert stream.read() == [['value1,value2'], ['value3,value4']]

def test_stream_csv_skipinitialspace():
    source = 'value1, value2\nvalue3, value4'
    with Stream(source, scheme='text', format='csv', skipinitialspace=True) as stream:
        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]

def test_stream_csv_detect_delimiter_tab():
    source = 'a1\tb1\tc1A,c1B\na2\tb2\tc2\n'
    with Stream(source, scheme='text', format='csv') as stream:
        assert stream.read() == [['a1', 'b1', 'c1A,c1B'], ['a2', 'b2', 'c2']]

def test_stream_csv_detect_delimiter_semicolon():
    source = 'a1;b1\na2;b2\n'
    with Stream(source, scheme='text', format='csv') as stream:
        assert stream.read() == [['a1', 'b1'], ['a2', 'b2']]

def test_stream_csv_detect_delimiter_pipe():
    source = 'a1|b1\na2|b2\n'
    with Stream(source, scheme='text', format='csv') as stream:
        assert stream.read() == [['a1', 'b1'], ['a2', 'b2']]

def test_stream_csv_dialect_should_not_persist_if_sniffing_fails_issue_goodtables_228():
    source1 = 'a;b;c\n#comment'
    source2 = 'a,b,c\n#comment'
    with Stream(source1, scheme='text', format='csv', headers=1, delimiter=';') as stream:
        assert stream.headers == ['a', 'b', 'c']
    with Stream(source2, scheme='text', format='csv', headers=1) as stream:
        assert stream.headers == ['a', 'b', 'c']

def test_stream_csv_quotechar_is_empty_string():
    source = 'value1,value2",value3'
    with Stream(source, scheme='text', format='csv', quotechar='') as stream:
        stream.read() == ['value1', 'value2"', 'value3']

# Write

def test_stream_save_csv(tmpdir):
    source = 'data/table.csv'
    target = str(tmpdir.join('table.csv'))
    with Stream(source, headers=1) as stream:
        assert stream.save(target) == 2
    with Stream(target, headers=1) as stream:
        assert stream.headers == ['id', 'name']
        assert stream.read(extended=True) == [
            (2, ['id', 'name'], ['1', 'english']),
            (3, ['id', 'name'], ['2', '中国人']),
        ]

# Internal

def test_parser_csv():

    source = 'data/table.csv'
    encoding = None
    loader = Mock()
    loader.load = Mock(return_value=io.open(source, encoding='utf-8'))
    parser = CSVParser(loader)

    assert parser.closed
    parser.open(source, encoding=encoding)
    assert not parser.closed

    assert list(parser.extended_rows) == [
        (1, None, ['id', 'name']),
        (2, None, ['1', 'english']),
        (3, None, ['2', '中国人'])]

    assert len(list(parser.extended_rows)) == 0
    parser.reset()
    assert len(list(parser.extended_rows)) == 3

    parser.close()
    assert parser.closed


text:
package com.panoramagl;

import com.panoramagl.opengl.GLUES;

import javax.microedition.khronos.opengles.GL10;

public class PLCylindricalPanorama extends PLQuadricPanoramaBase {
    /**
     * member variables
     */

    private float mHeight, mHalfHeight;

    /**
     * init methods
     */

    public PLCylindricalPanorama() {
        super();
    }

    @Override
    protected void initializeValues() {
        super.initializeValues();
        this.setHeight(PLConstants.kDefaultCylinderHeight);
        this.setPreviewDivs(PLConstants.kDefaultCylinderPreviewDivs);
        this.setDivs(PLConstants.kDefaultCylinderDivs);
        this.setPitchRange(0.0f, 0.0f);
        this.setXAxisEnabled(false);
    }

    /**
     * property methods
     */

    public float getHeight() {
        return mHeight;
    }

    public void setHeight(float height) {
        if (height > 0.0f)
            this.setInternalHeight(height);
    }

    protected float getHalfHeight() {
        return mHalfHeight;
    }

    protected void setInternalHeight(float height) {
        mHeight = height;
        mHalfHeight = height * 0.5f;
    }

    @Override
    public void setImage(PLIImage image) {
        if (image != null)
            this.setTexture(new PLTexture(image));
    }

    public void setTexture(PLITexture texture) {
        this.setTexture(texture, 0);
    }

    /**
     * render methods
     */

    @Override
    protected void internalRender(GL10 gl, PLIRenderer renderer) {
        PLITexture previewTexture = this.getPreviewTextures()[0], texture = this.getTextures()[0];

        boolean textureIsValid = (texture != null && texture.getTextureId(gl) != 0);

        if (textureIsValid || (previewTexture != null && previewTexture.getTextureId(gl) != 0)) {
            gl.glEnable(GL10.GL_TEXTURE_2D);

            int divs;

            if (textureIsValid) {
                divs = this.getDivs();
                gl.glBindTexture(GL10.GL_TEXTURE_2D, texture.getTextureId(gl));
                if (previewTexture != null)
                    this.removePreviewTextureAtIndex(0, true);
            } else {
                divs = this.getPreviewDivs();
                gl.glBindTexture(GL10.GL_TEXTURE_2D, previewTexture.getTextureId(gl));
            }

            gl.glTranslatef(0.0f, 0.0f, -mHalfHeight);

            GLUES.gluCylinder(gl, this.getQuadric(), PLConstants.kPanoramaRadius, PLConstants.kPanoramaRadius, mHeight, divs, divs);

            gl.glTranslatef(0.0f, 0.0f, mHalfHeight);

            gl.glDisable(GL10.GL_TEXTURE_2D);
        }
    }
}


text:
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
const q = require("smartq");
const plugins = require("./npmts.plugins");
const paths = require("./npmts.paths");
const NpmtsConfig = require("./npmts.config");
const NpmtsMods = require("./npmts.mods");
const NpmtsWatch = require("./npmts.watch");
const NpmtsShip = require("./npmts.ship");
/**
 * smartanalytics
 * this data is fully anonymized (no Ips or any other personal information is tracked).
 * It just keeps track which of our tools are really used...
 * ... so we know where to spend our limited resources for improving them.
 * Since yarn is out and there is heavy caching going on,
 * pure download stats are just not reliable enough for us anymore
 * Feel free to dig into the smartanalytics package, if you are interested in how it works.
 * It is just an https call to our own Lossless Analytics API.
 * Our privacy policy can be found here: https://lossless.gmbh/privacy.html
 */
let npmtsAnalytics = new plugins.smartanalytics.Analytics({
    apiEndPoint: 'https://pubapi.lossless.one/analytics',
    projectId: 'gitzone',
    appName: 'npmts'
});
process.nextTick(() => __awaiter(this, void 0, void 0, function* () {
    // make the analytics call
    npmtsAnalytics
        .recordEvent('npmToolExecution', {
        executionMode: (yield NpmtsConfig.configPromise).mode,
        tsOptions: (yield NpmtsConfig.configPromise).tsOptions,
        watch: (yield NpmtsConfig.configPromise).watch,
        coverageTreshold: (yield NpmtsConfig.configPromise).coverageTreshold
    })
        .catch(err => {
        plugins.beautylog.warn('Lossless Analytics API not available...');
    });
}));
exports.run = () => __awaiter(this, void 0, void 0, function* () {
    let done = q.defer();
    plugins.beautylog.figletSync('NPMTS');
    let npmtsProjectInfo = new plugins.projectinfo.ProjectinfoNpm(paths.npmtsPackageRoot);
    // check for updates
    yield plugins.smartupdate.standardHandler.check('npmts', npmtsProjectInfo.version, 'http://gitzone.gitlab.io/npmts/changelog.html');
    plugins.beautylog.log('---------------------------------------------');
    let npmtsCli = new plugins.smartcli.Smartcli();
    // build
    npmtsCli
        .addCommand('build')
        .subscribe((argvArg) => __awaiter(this, void 0, void 0, function* () {
        let done = q.defer();
        plugins.beautylog.info('npmts version: ' + npmtsProjectInfo.version);
        const configArg = yield NpmtsConfig.run(argvArg);
        plugins.beautylog.ora.start('loading additional modules...');
        NpmtsMods.modCompile
            .load()
            .then(modCompile => {
            return modCompile.run(configArg);
        })
            .then(configArg => {
            let done = q.defer();
            NpmtsMods.modDocs
                .load()
                .then(modDocs => {
                return modDocs.run(configArg);
            })
                .then(configArg => {
                done.resolve(configArg);
            });
            return done.promise;
        })
            .then(configArg => {
            let done = q.defer();
            NpmtsMods.modTest
                .load()
                .then(modTest => {
                return modTest.run(configArg);
            })
                .then(configArg => {
                done.resolve(configArg);
            });
            return done.promise;
        })
            .then(NpmtsWatch.run)
            .then(NpmtsShip.run);
        return done.promise;
    }), err => {
        if (err instanceof Error) {
            console.log(err);
        }
    });
    // standard task
    npmtsCli.standardTask().subscribe((argvArg) => __awaiter(this, void 0, void 0, function* () {
        yield npmtsCli.trigger('build');
    }));
    // cli metadata
    npmtsCli.addVersion(npmtsProjectInfo.version);
    // start parsing
    npmtsCli.startParse();
    return yield done.promise;
});


text:
using System;
using System.IO;
using System.Text;
using System.Threading.Tasks;
using Dot.Net.DevFast.Extensions;
using Dot.Net.DevFast.Extensions.StringExt;
using Dot.Net.DevFast.Tests.TestHelpers;
using NUnit.Framework;

namespace Dot.Net.DevFast.Tests.Extensions
{
    [TestFixture]
    public class CompressionExtsTest
    {
        [Test]
        [TestCase(0, 0, true)]
        [TestCase(1, 0, true)]
        [TestCase(1, 1, true)]
        [TestCase(2, 0, true)]
        [TestCase(2, 1, true)]
        [TestCase(2, 2, true)]
        [TestCase(350, 123, true)]
        [TestCase(99, 91, true)]
        [TestCase(0, 0, false)]
        [TestCase(1, 0, false)]
        [TestCase(1, 1, false)]
        [TestCase(2, 0, false)]
        [TestCase(2, 1, false)]
        [TestCase(2, 2, false)]
        [TestCase(350, 123, false)]
        [TestCase(99, 91, false)]
        public async Task Segment_Based_CompressAsync_DecompressAsSegmentAsync_Harmonize(int arrSize, int segSize,
            bool gzip)
        {
            var randm = new Random();
            var bytes = new byte[arrSize];
            randm.NextBytes(bytes);

            var segment = new ArraySegment<byte>(bytes, 0, segSize);
            using (var mem = new MemoryStream())
            {
                await segment.CompressAsync(mem, gzip).ConfigureAwait(false);
                var compressedData = mem.ToArray();
                if (segSize == 0)
                {
#if NETFRAMEWORK
                    Assert.True(compressedData.Length == 0);
#else
                    Assert.False(compressedData.Length == 0);
#endif
                }
                else
                {
                    var invalidCompress = compressedData.Length == segSize;
                    Assert.NotNull(segment.Array);
                    for (var i = 0; i < Math.Min(segSize, compressedData.Length); i++)
                    {
                        if (invalidCompress)
                        {
                            invalidCompress = segment.Array[i] == compressedData[i];
                        }
                    }

                    Assert.False(invalidCompress);
                }

                mem.Seek(0, SeekOrigin.Begin);
                var uncompressed = await mem.DecompressAsSegmentAsync(gzip).ConfigureAwait(false);
                Assert.True(segment.Count == uncompressed.Count);
                Assert.NotNull(segment.Array);
                Assert.NotNull(uncompressed.Array);
                for (var i = 0; i < uncompressed.Count; i++)
                {
                    Assert.True(segment.Array[i].Equals(uncompressed.Array[i]));
                }
            }
        }

        [Test]
        [TestCase(0, true)]
        [TestCase(1, true)]
        [TestCase(2, true)]
        [TestCase(3, true)]
        [TestCase(350, true)]
        [TestCase(0, false)]
        [TestCase(1, false)]
        [TestCase(2, false)]
        [TestCase(3, false)]
        [TestCase(350, false)]
        public async Task ByteArray_Based_CompressAsync_DecompressAsync_Harmonize(int arrSize, bool gzip)
        {
            var randm = new Random();
            var bytes = new byte[arrSize];
            randm.NextBytes(bytes);

            using (var mem = new MemoryStream())
            {
                await bytes.CompressAsync(mem, gzip).ConfigureAwait(false);
                var compressedData = mem.ToArray();
                if (arrSize == 0)
                {
#if NETFRAMEWORK
                    Assert.True(compressedData.Length == 0);
#else
                    Assert.False(compressedData.Length == 0);
#endif
                }
                else
                {
                    var invalidCompress = compressedData.Length == arrSize;
                    for (var i = 0; i < Math.Min(arrSize, compressedData.Length); i++)
                    {
                        if (invalidCompress)
                        {
                            invalidCompress = bytes[i] == compressedData[i];
                        }
                    }
                    Assert.False(invalidCompress);
                }
                mem.Seek(0, SeekOrigin.Begin);
                var uncompressed = await mem.DecompressAsync(gzip).ConfigureAwait(false);
                Assert.True(arrSize == uncompressed.Length);
                for (var i = 0; i < uncompressed.Length; i++)
                {
                    Assert.True(bytes[i].Equals(uncompressed[i]));
                }
            }
        }

        [Test]
        [TestCase(0, true)]
        [TestCase(1, true)]
        [TestCase(2, true)]
        [TestCase(3, true)]
        [TestCase(350, true)]
        [TestCase(0, false)]
        [TestCase(1, false)]
        [TestCase(2, false)]
        [TestCase(3, false)]
        [TestCase(350, false)]
        public async Task Stream_Stream_Based_CompressAsync_DecompressAsync_Harmonize(int arrSize, bool gzip)
        {
            var randm = new Random();
            var bytes = new byte[arrSize];
            randm.NextBytes(bytes);

            using (var input = new MemoryStream(bytes, false))
            {
                using (var mem = new MemoryStream())
                {
                    await input.CompressAsync(mem, gzip).ConfigureAwait(false);
                    var compressedData = mem.ToArray();
                    if (arrSize == 0)
                    {
                        Assert.True(compressedData.Length >= 0);
                    }
                    else
                    {
                        var invalidCompress = compressedData.Length == arrSize;
                        for (var i = 0; i < Math.Min(arrSize, compressedData.Length); i++)
                        {
                            if (invalidCompress)
                            {
                                invalidCompress = bytes[i] == compressedData[i];
                            }
                        }
                        Assert.False(invalidCompress);
                    }
                    mem.Seek(0, SeekOrigin.Begin);
                    using (var output = new MemoryStream())
                    {
                        await mem.DecompressAsync(output, gzip).ConfigureAwait(false);
                        var uncompressed = output.ToArray();
                        Assert.True(arrSize == uncompressed.Length);
                        for (var i = 0; i < uncompressed.Length; i++)
                        {
                            Assert.True(bytes[i].Equals(uncompressed[i]));
                        }
                    }
                }
            }
        }

        [Test]
        [TestCase(null, true)]
        [TestCase("utf-8", true)]
        [TestCase("utf-32BE", true)]
        [TestCase("utf-32LE", true)]
        [TestCase("utf-32", true)]
        [TestCase("utf-16BE", true)]
        [TestCase("utf-16LE", true)]
        [TestCase("utf-16", true)]
        [TestCase("us-ascii", true)]
        [TestCase(null, false)]
        [TestCase("utf-8", false)]
        [TestCase("utf-32BE", false)]
        [TestCase("utf-32LE", false)]
        [TestCase("utf-32", false)]
        [TestCase("utf-16BE", false)]
        [TestCase("utf-16LE", false)]
        [TestCase("utf-16", false)]
        [TestCase("us-ascii", false)]
        public async Task StringBuilder_Based_CompressAsync_DecompressAsync_Harmonize(string enc, bool gzip)
        {
            var encIns = Encoding.GetEncoding(enc.TrimSafeOrDefault("utf-8"));
            var sb = new StringBuilder(TestValues.BigString);
            using (var mem = new MemoryStream())
            {
                await sb.CompressAsync(mem, gzip, enc: encIns).ConfigureAwait(false);
                Assert.False(encIns.GetString(mem.ToArray()).Equals(TestValues.BigString));
                mem.Seek(0, SeekOrigin.Begin);
                sb.Clear();
                await mem.DecompressAsync(sb, gzip, enc: encIns).ConfigureAwait(false);
                Assert.True(sb.ToString().Equals(TestValues.BigString));
            }
        }

        [Test]
        [TestCase(null, true)]
        [TestCase("utf-8", true)]
        [TestCase("utf-32BE", true)]
        [TestCase("utf-32LE", true)]
        [TestCase("utf-32", true)]
        [TestCase("utf-16BE", true)]
        [TestCase("utf-16LE", true)]
        [TestCase("utf-16", true)]
        [TestCase("us-ascii", true)]
        [TestCase(null, false)]
        [TestCase("utf-8", false)]
        [TestCase("utf-32BE", false)]
        [TestCase("utf-32LE", false)]
        [TestCase("utf-32", false)]
        [TestCase("utf-16BE", false)]
        [TestCase("utf-16LE", false)]
        [TestCase("utf-16", false)]
        [TestCase("us-ascii", false)]
        public async Task String_Based_CompressAsync_DecompressAsStringAsync_Harmonize(string enc, bool gzip)
        {
            var encIns = Encoding.GetEncoding(enc.TrimSafeOrDefault("utf-8"));
            using (var mem = new MemoryStream())
            {
                await TestValues.BigString.CompressAsync(mem, gzip, enc: encIns).ConfigureAwait(false);
                Assert.False(encIns.GetString(mem.ToArray()).Equals(TestValues.BigString));
                mem.Seek(0, SeekOrigin.Begin);
                Assert.True(
                    (await mem.DecompressAsStringAsync(gzip, enc: encIns).ConfigureAwait(false)).Equals(
                        TestValues.BigString));
            }
        }
    }
}


text:
namespace renderer
{

namespace
{
    class SerialTileCallback
      : public ITileCallback
    {
      public:
        explicit SerialTileCallback(SerialRendererController* controller)
          : m_controller(controller)
        {
        }

        void release() override
        {
            delete this;
        }

        void on_tiled_frame_begin(const Frame* frame) override
        {
            m_controller->add_on_tiled_frame_begin_callback(frame);
        }

        void on_tiled_frame_end(const Frame* frame) override
        {
            m_controller->add_on_tiled_frame_end_callback(frame);
        }

        void on_tile_begin(
            const Frame*    frame,
            const size_t    tile_x,
            const size_t    tile_y) override
        {
            m_controller->add_on_tile_begin_callback(frame, tile_x, tile_y);
        }

        void on_tile_end(
            const Frame*    frame,
            const size_t    tile_x,
            const size_t    tile_y) override
        {
            m_controller->add_on_tile_end_callback(frame, tile_x, tile_y);
        }

        void on_progressive_frame_update(const Frame* frame) override
        {
            m_controller->add_on_progressive_frame_update_callback(frame);
        }

      private:
        SerialRendererController* m_controller;
    };
}

//
// SerialTileCallbackFactory class implementation.
//

SerialTileCallbackFactory::SerialTileCallbackFactory(SerialRendererController* controller)
  : m_controller(controller)
{
}

void SerialTileCallbackFactory::release()
{
    delete this;
}

ITileCallback* SerialTileCallbackFactory::create()
{
    return new SerialTileCallback(m_controller);
}

}   // namespace renderer


text:
package org.apache.spark.sql.execution

import org.apache.spark._
import org.apache.spark.rdd.RDD
import org.apache.spark.shuffle.sort.SortShuffleManager
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.vectorized.ColumnarBatch

/**
 * The [[Partition]] used by [[ShuffledColumnarBatchRDD]].
 */
private final case class ShuffledColumnarBatchRDDPartition(index: Int, spec: ShufflePartitionSpec)
    extends Partition

/**
 * [[ShuffledColumnarBatchRDD]] is the columnar version of [[org.apache.spark.rdd.ShuffledRDD]].
 */
class ShuffledColumnarBatchRDD(
    var dependency: ShuffleDependency[Int, ColumnarBatch, ColumnarBatch],
    metrics: Map[String, SQLMetric],
    partitionSpecs: Array[ShufflePartitionSpec])
    extends RDD[ColumnarBatch](dependency.rdd.context, Nil) {

  def this(
      dependency: ShuffleDependency[Int, ColumnarBatch, ColumnarBatch],
      metrics: Map[String, SQLMetric]) = {
    this(
      dependency,
      metrics,
      Array.tabulate(dependency.partitioner.numPartitions)(i => CoalescedPartitionSpec(i, i + 1)))
  }

  if (SQLConf.get.fetchShuffleBlocksInBatch) {
    dependency.rdd.context
      .setLocalProperty(SortShuffleManager.FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED_KEY, "true")
  }

  override def getDependencies: Seq[Dependency[_]] = List(dependency)

  override val partitioner: Option[Partitioner] =
    if (partitionSpecs.forall(_.isInstanceOf[CoalescedPartitionSpec])) {
      val indices = partitionSpecs.map(_.asInstanceOf[CoalescedPartitionSpec].startReducerIndex)
      // TODO this check is based on assumptions of callers' behavior but is sufficient for now.
      if (indices.toSet.size == partitionSpecs.length) {
        Some(new CoalescedPartitioner(dependency.partitioner, indices))
      } else {
        None
      }
    } else {
      None
    }

  override def getPartitions: Array[Partition] = {
    Array.tabulate[Partition](partitionSpecs.length) { i =>
      ShuffledColumnarBatchRDDPartition(i, partitionSpecs(i))
    }
  }

  override def getPreferredLocations(partition: Partition): Seq[String] = {
    val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
    partition.asInstanceOf[ShuffledColumnarBatchRDDPartition].spec match {
      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
        // TODO order by partition size.
        startReducerIndex.until(endReducerIndex).flatMap { reducerIndex =>
          tracker.getPreferredLocationsForShuffle(dependency, reducerIndex)
        }

      case PartialReducerPartitionSpec(_, startMapIndex, endMapIndex) =>
        tracker.getMapLocation(dependency, startMapIndex, endMapIndex)

      case PartialMapperPartitionSpec(mapIndex, _, _) =>
        tracker.getMapLocation(dependency, mapIndex, mapIndex + 1)
    }
  }

  override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = {
    val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
    // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
    // as well as the `tempMetrics` for basic shuffle metrics.
    val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
    val reader = split.asInstanceOf[ShuffledColumnarBatchRDDPartition].spec match {
      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
        SparkEnv.get.shuffleManager.getReader(
          dependency.shuffleHandle,
          startReducerIndex,
          endReducerIndex,
          context,
          sqlMetricsReporter)

      case PartialReducerPartitionSpec(reducerIndex, startMapIndex, endMapIndex) =>
        SparkEnv.get.shuffleManager.getReaderForRange(
          dependency.shuffleHandle,
          startMapIndex,
          endMapIndex,
          reducerIndex,
          reducerIndex + 1,
          context,
          sqlMetricsReporter)

      case PartialMapperPartitionSpec(mapIndex, startReducerIndex, endReducerIndex) =>
        SparkEnv.get.shuffleManager.getReaderForRange(
          dependency.shuffleHandle,
          mapIndex,
          mapIndex + 1,
          startReducerIndex,
          endReducerIndex,
          context,
          sqlMetricsReporter)
    }
    reader.read().asInstanceOf[Iterator[Product2[Int, ColumnarBatch]]].map(_._2)
  }

  override def clearDependencies() {
    super.clearDependencies()
    dependency = null
  }
}


text:
static const int kNumFramesToPaint = 500;
static base::TimeTicks g_start_;
static base::TimeTicks g_end_;

long CalculateYUVFrameSize(FILE* file_handle, int num_frames) {
  fseek(file_handle, 0, SEEK_END);
  long file_size = (long) ftell(file_handle);
  rewind(file_handle);
  return file_size / num_frames;
}

void GetFrames(std::string file_name,
               int width, int height, int num_frames,
               std::deque<scoped_refptr<media::VideoFrame> >& out_frames) {
  FILE* file_handle = fopen(file_name.c_str(), "rb");
  if (!file_handle) {
    printf("Could not open %s\n", file_name.c_str());
    exit(1);
  }

  long frame_size = CalculateYUVFrameSize(file_handle, num_frames);

  for (int i = 0; i < num_frames; i++) {
    scoped_refptr<media::VideoFrame> video_frame;

    media::VideoFrame::CreateFrame(media::VideoFrame::YV12,
                                   width,
                                   height,
                                   base::TimeDelta(),
                                   base::TimeDelta(),
                                   &video_frame);
    long bytes_read =
        fread(video_frame->data(0), 1, frame_size, file_handle);

    if (bytes_read != frame_size) {
      printf("Could not read %s\n", file_name.c_str());
      fclose(file_handle);
      exit(1);
    }
    out_frames.push_back(video_frame);
  }

  fclose(file_handle);
}

void TestFinished() {
  g_end_ = base::TimeTicks::HighResNow();
  double time_in_seconds =
      static_cast<double>((g_end_ - g_start_).InMilliseconds()) / 1000;
  double fps = kNumFramesToPaint / time_in_seconds;
  printf("Printed %f frames per second.\n", fps);
}

void RunTest(media::Window* window, Painter* painter) {
  g_start_ = base::TimeTicks::HighResNow();
  window->Start(kNumFramesToPaint, NewRunnableFunction(&TestFinished), painter);
}

int main(int argc, char** argv) {
  // Read arguments.
  if (argc == 1) {
    printf("Usage: %s --file=FILE --wxh=DIMENSIONS --frames=NUM_FRAMES\n"
           "FILE is a raw .yuv file with 1+ frames in it\n"
           "DIMENSIONS is the width and height of the frame in pixels\n"
           "NUM_FRAMES is the number of frames in FILE\n", argv[0]);
    return 1;
  }

  // Read command line.
#if defined(OS_LINUX)
  gtk_init(&argc, &argv);
#endif
  CommandLine::Init(argc, argv);

  // Determine file name.
  std::string file_name =
      CommandLine::ForCurrentProcess()->GetSwitchValueASCII("file");

  // Determine number of frames.
  int num_frames = 0;
  std::string str_num_frames =
      CommandLine::ForCurrentProcess()->GetSwitchValueASCII("frames");
  base::StringToInt(str_num_frames, &num_frames);

  // Determine video dimensions.
  int width = 0;
  int height = 0;
  std::string dimensions =
      CommandLine::ForCurrentProcess()->GetSwitchValueASCII("wxh");
  int x_index = dimensions.find('x');
  std::string str_width = dimensions.substr(0, x_index);
  std::string str_height =
      dimensions.substr(x_index + 1, dimensions.length() - x_index - 1);
  base::StringToInt(str_width, &width);
  base::StringToInt(str_height, &height);

  // Process files.
  std::deque<scoped_refptr<media::VideoFrame> > frames;
  GetFrames(file_name, width, height, num_frames, frames);

  // Initialize window and graphics context.
  base::AtExitManager at_exit_manager;
  gfx::GLContext::InitializeOneOff();
  scoped_ptr<media::Window> window(new media::Window(width, height));
  gfx::GLContext* context =
      gfx::GLContext::CreateViewGLContext(window->PluginWindow(), false);
  context->MakeCurrent();
  // This sets D3DPRESENT_INTERVAL_IMMEDIATE on Windows.
  context->SetSwapInterval(0);

  // Initialize and name GPU painters.
  static const int kNumPainters = 3;
  static const struct {
    const char* name;
    GPUPainter* painter;
  } painters[] = {
    { "CPU CSC + GPU Render", new CPUColorPainter() },
    { "GPU CSC/Render", new GPUColorWithLuminancePainter() },
    { "GPU CSC/Render (experimental)", new GPUColorRGBALumHackPainter() },
  };

  // Run GPU painter tests.
  for (int i = 0; i < kNumPainters; i++) {
    scoped_ptr<GPUPainter> painter(painters[i].painter);
    painter->LoadFrames(&frames);
    painter->SetGLContext(context);
    painter->Initialize(width, height);
    printf("Running %s tests...", painters[i].name);
    RunTest(window.get(), painter.get());
  }

  return 0;
}


text:
//
// SecureLogString.cs
//
// Author:
// 	Jim Borden  <jim.borden@couchbase.com>
//
// Copyright (c) 2016 Couchbase, Inc All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
using System;
using System.Linq;
using System.Text;

namespace Couchbase.Lite.Util
{
    /// <summary>
    /// The sensitivity at which the logger should redact
    /// sensitive information
    /// </summary>
    public enum LogScrubSensitivity
    {
        /// <summary>
        /// No potentially insecure information shall be logged
        /// </summary>
        NoInsecure = 0,

        /// <summary>
        /// Information that might be insecure (i.e. user generated)
        /// may be logger, but access tokens, passwords, etc should still
        /// be redacted
        /// </summary>
        PotentiallyInsecureOK,

        /// <summary>
        /// All information should be logged
        /// </summary>
        AllOK
    }

    internal enum LogMessageSensitivity
    {
        PotentiallyInsecure = 1,
        Insecure
    }

    internal abstract class SecureLogItem
    {
        protected const string Redacted = "<redacted>";
        protected const int CharLimit = 100;

        private readonly LogMessageSensitivity _sensitivity;

        protected bool ShouldLog
        {
            get {
                return (int)_sensitivity <= (int)Log.ScrubSensitivity;
            }
        }

        protected SecureLogItem(LogMessageSensitivity sensitivity)
        {
            _sensitivity = sensitivity;
        }

    }

    internal sealed class SecureLogString : SecureLogItem
    {
        private string _string;
        private readonly byte[] _bytes;
        private readonly object _obj;

        private string String
        {
            get {
                if (_string == null) {
                    var str = default(string);
                    if (_bytes != null) {
                        str = Encoding.UTF8.GetString(_bytes);
                    } else if (_obj != null) {
                        str = _obj.ToString();
                    } else {
                        str = "(null)";
                    }

                    if(str.Length > 100) {
                        _string = $"{new string(str.Take(100).ToArray())}...";
                    } else {
                        _string = str;
                    }
                }

                return _string;
            }
        }

        public SecureLogString(string str, LogMessageSensitivity sensitivityLevel) : base(sensitivityLevel)
        {
            _string = str;
        }

        public SecureLogString(byte[] utf8Bytes, LogMessageSensitivity sensitivityLevel) : base(sensitivityLevel)
        {
            _bytes = utf8Bytes;
        }

        public SecureLogString(object obj, LogMessageSensitivity sensitivityLevel) : base(sensitivityLevel)
        {
            _obj = obj;
        }

        public override string ToString()
        {
            return ShouldLog ? String : Redacted;
        }
    }

    internal sealed class SecureLogJsonString : SecureLogItem
    {
        private readonly object _object;
        private string _str;

        private string String
        {
            get {
                if(_str == null) {
                    var str = Manager.GetObjectMapper().WriteValueAsString(_object);
                    if(str.Length > 100) {
                        _str = $"{new string(str.Take(100).ToArray())}...";
                    } else {
                        _str = str;
                    }
                }

                return _str;
            }
        }

        public SecureLogJsonString(object input, LogMessageSensitivity sensitivityLevel) : base(sensitivityLevel)
        {
            _object = input;
        }

        public override string ToString()
        {
            return ShouldLog ? String : Redacted;
        }
    }

    internal sealed class SecureLogUri : SecureLogItem
    {
        private readonly Uri _uri;
        private string _str;

        private string UriString
        {
            get {
                if (_str == null) {
                    _str = _uri.ToString().ReplaceAll("://.*:.*@", "://<redacted>:<redacted>@");
                }

                return _str;
            }
        }

        // Only used for stripping credentials, so always insecure
        public SecureLogUri(Uri uri) : base(LogMessageSensitivity.Insecure)
        {
            _uri = uri;
        }

        public override string ToString()
        {
            return ShouldLog ? _uri.ToString() : UriString;
        }
    }
}


text:
namespace viz {

// Uniquely identifies OverlayProposedCandidates as it will be stored in the
// OverlayCombinationCache. An overlay with the same strategy and the same
// rounded display_rect are considered identical.
struct OverlayCacheKey {
  explicit OverlayCacheKey(const OverlayProposedCandidate& candidate);

  bool operator<(const OverlayCacheKey& other) const {
    if (other.strategy != strategy) {
      return strategy < other.strategy;
    } else {
      return display_rect < other.display_rect;
    }
  }

  // The candidate's rounded display_rect.
  gfx::Rect display_rect;
  // The strategy that proposed this candidate.
  OverlayStrategy strategy;
};

// A vector of proposed candidates to try and promote to overlays, as determined
// by the OverlayCombinationCache. There is also a flag that indicates that this
// entire combination has been successfully promoted to overlays the last time
// it was tested.
struct VIZ_SERVICE_EXPORT OverlayCombinationToTest {
  OverlayCombinationToTest();
  ~OverlayCombinationToTest();
  OverlayCombinationToTest(OverlayCombinationToTest&& other);
  OverlayCombinationToTest& operator=(OverlayCombinationToTest&& other);

  // The candidates that should be promoted to overlays.
  std::vector<OverlayProposedCandidate> candidates_to_test;
  // True iff this combination of candidates were successfully promoted to
  // overlays the last time they were tested.
  bool previously_succeeded = false;
};

class CombinationIdMapper;

// Attempts to determine the optimal combination of candidates that we should
// attempt to promote to overlays each frame, based on total expected relative
// power gain, and past success/failure of these combinations being promoted.
class VIZ_SERVICE_EXPORT OverlayCombinationCache {
 public:
  // The maximum number of candidates this cache can track.
  static constexpr size_t kMaxTrackedCandidates = 8;
  // The invalid candidate id.
  static constexpr size_t kInvalidCandidateId = kMaxTrackedCandidates;
  // We track all 2^N combinations of the max N tracked overlay candidates.
  static constexpr size_t kMaxTrackedCombinations = 1 << kMaxTrackedCandidates;

  // Represents a set of CandidateIds.
  using CandidateCombination = std::bitset<kMaxTrackedCandidates>;
  // Represents a bit index used in a CandidateCombination.
  // This may also represent `kInvalidCandidateId`.
  // Valid CandidateIds are in the range [0, kMaxTrackedCandidates).
  using CandidateId = size_t;

  OverlayCombinationCache();
  ~OverlayCombinationCache();

  // Given proposed candidates that are sorted by power gain, and the maximum
  // number of overlays you'd like to try and promote, determines the best
  // combination of overlays to try promoting this frame.
  OverlayCombinationToTest GetOverlayCombinationToTest(
      const std::vector<OverlayProposedCandidate>& sorted_candidates,
      int max_overlays_considered);

  // Records the success/failure of each candidate we attempted to promote to an
  // overlay. This should be called after testing the combination returned by
  // GetOverlayCombinationToTest() each frame.
  // NOTE: The OverlayProposedCandidate.candidate.overlay_handled field should
  // be true iff that candidate was promoted this frame.
  void DeclarePromotedCandidates(
      const std::vector<OverlayProposedCandidate>& attempted_candidates);

  // Clears all caches. This includes CandidateIds and CombinationResults.
  void ClearCache();

 private:
  friend class OverlayCombinationCacheTest;

  // The result of attempting to promote a CandidateCombination to overlays, or
  // unknown.
  enum class CombinationResult : uint8_t { kUnknown, kPromoted, kFailed };

  // Get the first max_overlays_possible candidates (if there are that many)
  // that represent unique quads. If there are multiple candidates for a single
  // quad, only the first one will be used.
  std::vector<OverlayProposedCandidate> GetConsideredCandidates(
      const std::vector<OverlayProposedCandidate>& sorted_candidates,
      size_t max_overlays_possible);

  // Gets the CandidateIds for each considered candidate. Reuses cached ids if
  // they exist, or maps new ids. All ids returned will be valid.
  // Any candidates that were considered last frame but are not this frame are
  // considered stale, and will have their CandidateIds and CombinationResults
  // cleared.
  std::vector<CandidateId> GetIds(
      const std::vector<OverlayProposedCandidate>& considered_candidates);

  // Generates all (2^N - 1) possible combinations of considered_candidates
  // (ignoring the empty set), and calculates the sum of their relative power
  // gains. Returns a vector of all of these CandidateCombination and total
  // power gain pairs.
  std::vector<std::pair<CandidateCombination, int>> GetPowerSortedCombinations(
      const std::vector<OverlayProposedCandidate>& considered_candidates,
      const std::vector<CandidateId>& considered_ids);

  // Reset the cached CombinationResult to kUnknown for any combination
  // containing any of the candidates in stale_candidates.
  void RemoveStaleCombinations(const CandidateCombination& stale_candidates);

  // Maps OverlayProposedCandidates to CandidateIds.
  std::unique_ptr<CombinationIdMapper> id_mapper_;
  // The test result, if known, for every possible CandidateCombination.
  std::array<CombinationResult, kMaxTrackedCombinations> combo_results_;
};

// Manages the mapping from OverlayProposedCandidates to CandidateIds ids based
// on their OverlayCacheKey. There are a fixed number of available ids, based on
// the maximum size of a CandidateCombination.
class VIZ_SERVICE_EXPORT CombinationIdMapper {
 public:
  CombinationIdMapper();
  ~CombinationIdMapper();

  // Gets a CandidateId for a given OverlayProposedCandidate.
  // Returns the cached id for this candidate if it exists, otherwise it
  // attempts to map the candidate to new id. If the cache is full (more than
  // kMaxTrackedCandidates in the cache), then kInvalidCandidateId is returned.
  //
  // NOTE: Callers need to check for kInvalidCandidateId before using it to
  // access a CandidateCombination.
  OverlayCombinationCache::CandidateId GetCandidateId(
      const OverlayProposedCandidate& candidate);

  // Returns a CandidateCombination of all the ids currently claimed.
  OverlayCombinationCache::CandidateCombination GetClaimedIds() {
    return claimed_ids_;
  }

  // Remove all ids in stale_candidates from the cache.
  void RemoveStaleIds(
      const OverlayCombinationCache::CandidateCombination& stale_candidates);

  // Clear all cached ids.
  void ClearIds();

 private:
  // Maps up to (kMaxTrackedCandidates) OverlayCacheKeys to unique CandidateIds.
  std::map<OverlayCacheKey, OverlayCombinationCache::CandidateId>
      candidate_ids_;
  // A set containing all ids that have been claimed.
  OverlayCombinationCache::CandidateCombination claimed_ids_;
};

}  // namespace viz

#endif  // COMPONENTS_VIZ_SERVICE_DISPLAY_OVERLAY_COMBINATION_CACHE_H_


text:
"""From a dump generated by dump_process.cc dump, prints statistics about
its content.
"""

from __future__ import print_function

import argparse
import array
import collections
import hashlib
import logging
import os
import zlib

PAGE_SIZE = 1 << 12

# These are typically only populated with DCHECK() on.
FREED_PATTERNS = {
    0xcccccccc: 'V8',
    0xcdcdcdcd: 'PartitionAlloc zapped',
    0xabababab: 'PartitionAlloc uninitialized',
    0xdeadbeef: 'V8 zapped',
    0x0baddeaf: 'V8 zapped handles',
    0x0baffedf: 'V8 zapped global handles',
    0x0beefdaf: 'V8 zapped from space',
    0xbeefdeef: 'V8 zapped slots',
    0xbadbaddb: 'V8 debug zapped',
    0xfeed1eaf: 'V8 zapped freelist'
}

def _ReadPage(f):
  """Reads a page of data from a file.

  Args:
    f: (file) An opened file to read from.

  Returns:
    An array.array() of unsigned int with the page content.
  """
  result = array.array('I')
  result.fromfile(f, PAGE_SIZE / result.itemsize)
  return result

def _PrettyPrintSize(x):
  """Pretty print sizes in bytes, e.g. 123456 -> 123.45kB.

  Args:
    x: (int) size

  Returns:
    (str) Pretty printed version, 2 decimal places.
  """
  if x < 1e3:
    return '%dB' % x
  elif 1e3 <= x < 1e6:
    return '%.2fkB' % (x / 1e3)
  elif 1e6 <= x < 1e9:
    return '%.2fMB' % (x / 1e6)
  else:
    return '%.2fGB' % (x / 1e9)

class MappingStats(object):
  """Statistics about a mapping, from a dump.

  Slots:
    filename: (str) Dump filename.
    start: (int) Start address of the mapping.
    end: (int) End address of the mapping.
    pages: (int) Sizs of the mapping in pages.
    is_zero: ([bool]) For each page, whether it's a zero page.
    is_present: ([bool]) For each page, whether it's present.
    is_swapped: ([bool]) For each page, whether it has been swapped out.
    compressed_size: ([int]) If a page is not zero, its compressed size.
    hashes: ([str]) If a page is not zero, its SHA1 hash.
    freed: ({'description (str)': size (int)}) Size of freed data, per type.
  """
  __slots__ = ('filename', 'start', 'end', 'pages', 'is_zero', 'is_present',
               'is_swapped', 'compressed_size', 'hashes', 'freed')

  def __init__(self, filename, start, end):
    """Init.

    Args:
      filename: (str) Dump filename.
      start: (int) Start address.
      end: (int) End address
    """
    self.filename = filename
    self.start = start
    self.end = end
    self.pages = (end - start) / PAGE_SIZE
    self.is_zero = [False for _ in range(self.pages)]
    self.is_present = [False for _ in range(self.pages)]
    self.is_swapped = [False for _ in range(self.pages)]
    self.compressed_size = [0 for _ in range(self.pages)]
    self.hashes = [None for _ in range(self.pages)]
    self.freed = collections.defaultdict(int)

def _GetStatsFromFileDump(filename):
  """Computes per-dump statistics.

  Args:
    filename: (str) Path to the dump.

  Returns:
    MappingStats for the mapping.
  """
  # Dump integrity checks.
  metadata_filename = filename + '.metadata'
  pid_start_end = os.path.basename(filename)[:-len('.dump')]
  (_, start, end) = [int(x, 10) for x in pid_start_end.split('-')]
  file_stat = os.stat(filename)
  assert start % PAGE_SIZE == 0
  assert end % PAGE_SIZE == 0
  assert file_stat.st_size == (end - start)
  metadata_file_stat = os.stat(metadata_filename)
  result = MappingStats(filename, start, end)
  # each line is [01]{2}\n, eg '10\n', 1 line per page.
  assert metadata_file_stat.st_size == 3 * result.pages

  with open(filename, 'r') as f, open(metadata_filename, 'r') as metadata_f:
    for i in range(result.pages):
      page = _ReadPage(f)
      assert len(page) == 1024
      for x in page:
        if x in FREED_PATTERNS:
          result.freed[FREED_PATTERNS[x]] += 4
      is_zero = max(page) == 0
      present, swapped = (bool(int(x)) for x in metadata_f.readline().strip())
      # Not present, not swapped private anonymous == lazily initialized zero
      # page.
      if not present and not swapped:
        assert is_zero
      result.is_zero[i] = is_zero
      result.is_present[i] = present
      result.is_swapped[i] = swapped
      if not is_zero:
        sha1 = hashlib.sha1()
        sha1.update(page)
        page_hash = sha1.digest()
        result.hashes[i] = page_hash
        compressed = zlib.compress(page, 1)
        result.compressed_size[i] = len(compressed)
  return result

def _FindPageFromHash(mappings, page_hash):
  """Returns a page with a given hash from a list of mappings.

  Args:
    mappings: ([MappingStats]) List of mappings.
    page_hash: (str) Page hash to look for,

  Returns:
    array.array(uint32_t) with the page content
  """
  for mapping in mappings:
    for i in range(mapping.pages):
      if mapping.hashes[i] == page_hash:
        with open(mapping.filename, 'r') as f:
          f.seek(i * PAGE_SIZE)
          page = _ReadPage(f)
          sha1 = hashlib.sha1()
          sha1.update(page)
          assert page_hash == sha1.digest()
          return page

def _PrintPage(page):
  """Prints the content of a page."""
  for i, x in enumerate(page):
    print('{:08x}'.format(x), end=' ')
    if i % 16 == 15:
      print()

AggregateStats = collections.namedtuple(
    'AggregateStats', ('content_to_count', 'pages', 'zero_pages',
                       'compressed_size', 'swapped_pages',
                       'not_present_pages', 'present_zero_pages', 'freed'))

def _AggregateStats(dump_stats):
  """Aggreates statistics across dumps.

  Args:
    dump_stats: ([MappingStats]) Stats from all mappings.

  Returns:
    An instance of AggregateStats.
  """
  content_to_count = collections.defaultdict(int)
  total_pages = sum(stats.pages for stats in dump_stats)
  total_zero_pages = sum(sum(stats.is_zero) for stats in dump_stats)
  total_compressed_size = sum(sum(stats.compressed_size)
                              for stats in dump_stats)
  total_swapped_pages = sum(sum(stats.is_swapped) for stats in dump_stats)
  total_not_present_pages = sum(stats.pages - sum(stats.is_present)
                                for stats in dump_stats)
  total_present_zero_pages = sum(
      sum(x == (True, True) for x in zip(stats.is_zero, stats.is_present))
      for stats in dump_stats)
  total_freed_space = {x: 0 for x in FREED_PATTERNS.values()}
  for dump in dump_stats:
    for (freed_data_type, value) in dump.freed.items():
      total_freed_space[freed_data_type] += value

  content_to_count = collections.defaultdict(int)
  for stats in dump_stats:
    for page_hash in stats.hashes:
      if page_hash:
        content_to_count[page_hash] += 1

  return AggregateStats(
      content_to_count=content_to_count, pages=total_pages,
      zero_pages=total_zero_pages, compressed_size=total_compressed_size,
      swapped_pages=total_swapped_pages,
      not_present_pages=total_not_present_pages,
      present_zero_pages=total_present_zero_pages,
      freed=total_freed_space)

def PrintStats(dumps, verbose):
  """Logs statistics about a process mappings dump.

  Args:
    dumps: ([str]) List of dumps.
    verbose: (bool) Verbose output.
  """
  dump_stats = [_GetStatsFromFileDump(filename) for filename in dumps]
  total = _AggregateStats(dump_stats)
  duplicated_pages = sum(x - 1 for x in total.content_to_count.values())
  count_and_hashes = sorted(((v, k) for k, v in total.content_to_count.items()),
                            reverse=True)
  max_common_pages = count_and_hashes[0][0] - 1
  total_size_non_zero_pages = (total.pages - total.zero_pages) * PAGE_SIZE

  print('Total pages = %d (%s)' % (total.pages,
                                   _PrettyPrintSize(total.pages * PAGE_SIZE)))
  print('Total zero pages = %d (%.02f%%)' %
        (total.zero_pages, (100. * total.zero_pages) / total.pages))
  print('Total present zero pages = %d (%s)' %
        (total.present_zero_pages,
         _PrettyPrintSize(total.present_zero_pages * PAGE_SIZE)))
  print(
      'Total size of non-zero pages = %d (%s)' %
      (total_size_non_zero_pages, _PrettyPrintSize(total_size_non_zero_pages)))
  print('Total compressed size = %d (%.02f%%)' %
        (total.compressed_size,
         (100. * total.compressed_size) / total_size_non_zero_pages))
  print('Duplicated non-zero pages = %d' % duplicated_pages)
  print('Max non-zero pages with the same content = %d' % max_common_pages)
  print(
      'Swapped pages = %d (%s)' %
      (total.swapped_pages, _PrettyPrintSize(total.swapped_pages * PAGE_SIZE)))
  print('Non-present pages = %d (%s)' %
        (total.not_present_pages,
         _PrettyPrintSize(total.not_present_pages * PAGE_SIZE)))
  print('Freed: ')
  for k in total.freed:
    print('  %s = %d (%s)' % (k, total.freed[k], _PrettyPrintSize(
        total.freed[k])))

  if verbose:
    print('Top Duplicated Pages:')
    for i in range(10):
      count, page_hash = count_and_hashes[i]
      print('%d common pages' % count)
      page = _FindPageFromHash(dump_stats, page_hash)
      _PrintPage(page)
      print()

def _CreateArgumentParser():
  parser = argparse.ArgumentParser()
  parser.add_argument('--directory', type=str, required=True,
                      help='Dumps directory')
  parser.add_argument('--verbose', action='store_true', help='Dumps directory')
  return parser

def main():
  logging.basicConfig(level=logging.INFO)
  parser = _CreateArgumentParser()
  args = parser.parse_args()

  dumps = []
  for f in os.listdir(args.directory):
    if f.endswith('.dump'):
      dumps.append(os.path.join(args.directory, f))

  PrintStats(dumps, args.verbose)

if __name__ == '__main__':
  main()