text:
package Paws::SSM::DescribeEffectivePatchesForPatchBaselineResult;
  use Moose;
  has EffectivePatches => (is => 'ro', isa => 'ArrayRef[Paws::SSM::EffectivePatch]');
  has NextToken => (is => 'ro', isa => 'Str');

  has _request_id => (is => 'ro', isa => 'Str');

### main pod documentation begin ###

=head1 NAME

Paws::SSM::DescribeEffectivePatchesForPatchBaselineResult

=head1 ATTRIBUTES

=head2 EffectivePatches => ArrayRef[L<Paws::SSM::EffectivePatch>]

An array of patches and patch status.

=head2 NextToken => Str

The token to use when requesting the next set of items. If there are no
additional items to return, the string is empty.

=head2 _request_id => Str

=cut

1;

redpajama_set_name: RedPajamaGithub


text:
require "spec_helper"

describe Jigsaw::Request do

  include CompanySearchWebmockHelpers

  context 'Listing companies' do

    before(:each) do
      stub_jigsaw_for_request
    end

    it 'should retrieve a list of companies' do
      response = Jigsaw::Request.search_company(:token => api_key)
      response['companies'].should_not be_empty
    end

    it 'should retrieve the number of matching companies' do
      response = Jigsaw::Request.search_company(:token => api_key)
      response['totalHits'].should_not be_nil
    end
  end

end


text:
class Comment < ActiveRecord::Base
	belongs_to :feed_entry
end


text:
package com.ytdl.conversion;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.UUID;

import com.ytdl.YtdlConstants;
import com.ytdl.api.SessionHandler;

public class VideoDownloader {
	private static boolean debug = true;

	/**
	 *
	 * @param url
	 *            YT video url
	 * @param id
	 *            id of session
	 * @param i
	 *            element of session
	 * @return
	 */
	public static boolean download(String url, UUID id, int i) {
		boolean downloadSuccess = false;
		InputStream stdout = null;
		InputStream stderr = null;
		InputStreamReader isr = null;
		InputStreamReader isrE = null;
		BufferedReader br = null;
		BufferedReader brE = null;
		try {
			ProcessBuilder pb = new ProcessBuilder("youtube-dl", url, "-x", "--audio-format=mp3", "--no-playlist", "-o",
					YtdlConstants.content_directory + "%(title)s-%(id)s.%(ext)s");

			Process proc = pb.start();

			stdout = proc.getInputStream();
			stderr = proc.getInputStream();
			isr = new InputStreamReader(stdout);
			isrE = new InputStreamReader(stderr);
			br = new BufferedReader(isr);
			brE = new BufferedReader(isrE);
			String outLine = null;
			String errLine = null;
			while ((outLine = br.readLine()) != null || (errLine = brE.readLine()) != null) {
				if (outLine.startsWith("[ffm")) {
					SessionHandler.updateRequest(id, i, SessionHandler.STATUS_CONVERTING);
					// break;
				} else if (outLine.startsWith("Deleting")) {
					downloadSuccess = true;
					// break;
				}
				if (debug) {
					if (outLine != null)
						System.out.println("[Debug] " + outLine);
					if (errLine != null)
						System.err.println("<Error>" + errLine + "<Error>");
				}
			}

		} catch (Throwable t) {
			t.printStackTrace();
		} finally {
			try {
				stdout.close();
				stderr.close();
				isr.close();
				isrE.close();
				br.close();
				brE.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		return downloadSuccess;
	}

	private VideoDownloader() {
	}
}


text:
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models

class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'List'
        db.create_table(u'list_list', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('title', self.gf('django.db.models.fields.CharField')(max_length=80)),
            ('slug', self.gf('django.db.models.fields.SlugField')(max_length=255)),
            ('description', self.gf('django.db.models.fields.TextField')()),
            ('md_content', self.gf('django.db.models.fields.TextField')()),
            ('html_content', self.gf('django.db.models.fields.TextField')()),
            ('url', self.gf('django.db.models.fields.URLField')(max_length=200, null=True, blank=True)),
            ('owner', self.gf('django.db.models.fields.related.ForeignKey')(related_name='list_owner', to=orm['auth.User'])),
            ('is_public', self.gf('django.db.models.fields.BooleanField')(default=True)),
            ('pubdate', self.gf('django.db.models.fields.DateField')(auto_now_add=True, blank=True)),
            ('views', self.gf('django.db.models.fields.IntegerField')(default=0)),
        ))
        db.send_create_signal(u'list', ['List'])

        # Adding model 'ListLinks'
        db.create_table(u'list_listlinks', (
            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('alist', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['list.List'])),
            ('links', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['link.Link'])),
            ('adddate', self.gf('django.db.models.fields.DateField')(auto_now_add=True, blank=True)),
        ))
        db.send_create_signal(u'list', ['ListLinks'])

    def backwards(self, orm):
        # Deleting model 'List'
        db.delete_table(u'list_list')

        # Deleting model 'ListLinks'
        db.delete_table(u'list_listlinks')

    models = {
        u'auth.group': {
            'Meta': {'object_name': 'Group'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        u'auth.permission': {
            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        u'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        u'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        u'language.language': {
            'Meta': {'object_name': 'Language'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'language': ('django.db.models.fields.CharField', [], {'max_length': '7'})
        },
        u'link.link': {
            'Meta': {'object_name': 'Link'},
            'author': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
            'hash2': ('django.db.models.fields.CharField', [], {'max_length': '11'}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'language': ('django.db.models.fields.related.ForeignKey', [], {'default': '1', 'to': u"orm['language.Language']"}),
            'level': ('django.db.models.fields.CharField', [], {'max_length': '30', 'null': 'True', 'blank': 'True'}),
            'pubdate': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '255'}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '120'}),
            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
        },
        u'list.list': {
            'Meta': {'object_name': 'List'},
            'description': ('django.db.models.fields.TextField', [], {}),
            'html_content': ('django.db.models.fields.TextField', [], {}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'links': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['link.Link']", 'through': u"orm['list.ListLinks']", 'symmetrical': 'False'}),
            'md_content': ('django.db.models.fields.TextField', [], {}),
            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'list_owner'", 'to': u"orm['auth.User']"}),
            'pubdate': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'slug': ('django.db.models.fields.SlugField', [], {'max_length': '255'}),
            'title': ('django.db.models.fields.CharField', [], {'max_length': '80'}),
            'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
            'views': ('django.db.models.fields.IntegerField', [], {'default': '0'})
        },
        u'list.listlinks': {
            'Meta': {'object_name': 'ListLinks'},
            'adddate': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'alist': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['list.List']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'links': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['link.Link']"})
        },
        u'taggit.tag': {
            'Meta': {'object_name': 'Tag'},
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '100'}),
            'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'})
        },
        u'taggit.taggeditem': {
            'Meta': {'object_name': 'TaggedItem'},
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_tagged_items'", 'to': u"orm['contenttypes.ContentType']"}),
            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}),
            'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "u'taggit_taggeditem_items'", 'to': u"orm['taggit.Tag']"})
        }
    }

    complete_apps = ['list']


text:
@interface CHRJListToolBarContentModel : NSObject
@property (nonatomic,strong)NSNumber * t;
@property (nonatomic,copy)NSString * p;
@property (nonatomic,copy)NSString * pnew;
@property (nonatomic,copy)NSString * title;
@property (nonatomic,copy)NSString * d;
@property (nonatomic,copy)NSString * color_size;
@property (nonatomic,copy)NSString * ipnew;
@property (nonatomic,copy)NSString * ip;

@end


text:
require "xfabricator/version"
require 'xcodeproj'
require 'xfabricator/x_code_file_template'
require 'fileutils'
require 'xfabricator/xcodeproj_additions'

module Xfabricator

  class Fabricator

    def initialize(config, search_path)
      @config = config
      @search_path = search_path
    end

    def fabricate(template, variables, target_name)
      group = find_current_group

      unless group
        raise ArgumentError, "couldn't find current group"
      end

      fabricate_in_group(template, variables, target_name, group)
    end

    def fabricate_in_group(template, variables, target_name, group)
      unless group
        raise ArgumentError, "group not specified"
      end

      unless target_name
        raise ArgumentError, "target_name not specified"
      end

      project = @config.project
      target = project.targets.find { |target| target.name == target_name }

      unless target
        raise ArgumentError, "Couldn't find target with name: #{target_name}"
      end

      template.template_files.each do |template_file|
        template_file_path = "#{@config.template_location}/#{template_file.template_file}"

        unless Pathname.new(template_file_path).exist?
          template_file_path = template_file.template_file
        end

        if Pathname.new(template_file_path).exist?
          name_template = Mustache.new
          output_name = name_template.render template_file.output_name_template, variables

          current_group = group

          if template_file.sub_path.length > 0
            groups = group.xfab_create_subpath(template_file.sub_path)
            current_group = groups.last
          end

          output_file = fabricate_file(output_name, template_file_path, variables, current_group.real_path)

          ref = current_group.new_reference output_file
          target.add_file_references [ref]
        else
          raise RuntimeError, "template file was not found: #{template_file_path}"
        end

      end

      project.save

    end

    def fabricate_file(output_name, template_file_path, variables, output_dir)
      FileUtils.mkdir_p output_dir

      template = XCodeFileTemplate.new @config.project, variables
      template.template_file = template_file_path
      rendered_content = template.render

      output_file = "#{output_dir}/#{output_name}"
      File.open(output_file, 'w') { |file| file.write(rendered_content) }
      output_file
    end

    def find_current_group
      project = @config.project
      main_group = project.main_group

      if main_group.real_path == @search_path.to_s
        return main_group
      end

      find_group_for_directory(main_group, @search_path, project)
    end

    def find_group_for_directory(current_group, directory, project)
      group_found = current_group.groups.find { |ref|
        full_path = "#{project.path.parent.to_s}#{ref.hierarchy_path}"
        full_path == directory.to_s
      }

      unless group_found
        current_group.groups.each do |group|
          full_path = "#{project.path.parent.to_s}#{group.hierarchy_path}"
          return find_group_for_directory(group, directory, project) if directory.to_s.start_with? full_path
        end
      end

      group_found
    end

  end

end


text:
'use strict';

const stampit = require('stampit');
const debug = require('debug')('buspirate:buspirate');
const SerialPort = require('serialport').SerialPort;
const EventEmittable = require('./eventemittable');
const Promise = require('bluebird');

const BusPirate = stampit({
  refs: {
    baud: 115200
  },
  init() {
    const port = this.port = new SerialPort(this.device, this.baud);
    debug('Initializing port...');

    return new Promise((resolve, reject) => {
      port.on('open', resolve)
        .on('error', reject);
    })
      .tap(() => {
        debug('Port initialized');
        this.emit('ready', port);
      })
      .return(this);
  }
})
  .compose(EventEmittable);

module.exports = BusPirate;


text:
/*
* <auto-generated>
*     This code was generated by a tool.
*
*     Changes to this file may cause incorrect behavior and will be lost if
*     the code is regenerated.
* </auto-generated>
*/

#import <Foundation/Foundation.h>
#import "MOZUClient.h"
#import "MOZUReturnItemCollection.h"
#import "MOZUReturnItem.h"
#import "MOZUReturn.h"
#import "MOZUReturnCollection.h"
#import "MOZUPaymentAction.h"
#import "MOZUReasonCollection.h"
#import "MOZUPaymentCollection.h"
#import "MOZUPayment.h"
#import "MOZUReturnAction.h"

@interface MOZUReturnClient : NSObject

//
#pragma mark -
#pragma mark Get Operations
#pragma mark -
//

/**
Retrieves a list of all returns according to any filter and sort criteria.
@param filter A set of expressions that consist of a field, operator, and value and represent search parameter syntax when filtering results of a query. Valid operators include equals (eq), does not equal (ne), greater than (gt), less than (lt), greater than or equal to (ge), less than or equal to (le), starts with (sw), or contains (cont). For example - "filter=IsDisplayed+eq+true"
@param pageSize The number of results to display on each page when creating paged results from a query. The maximum value is 200.
@param responseFields Use this field to include those fields which are not included by default.
@param sortBy The property by which to sort results and whether the results appear in ascending (a-z) order, represented by ASC or in descending (z-a) order, represented by DESC. The sortBy parameter follows an available property. For example: "sortBy=productCode+asc"
@param startIndex When creating paged results from a query, this value indicates the zero-based offset in the complete result set where the returned entities begin. For example, with a PageSize of 25, to get the 51st through the 75th items, use startIndex=3.
*/

+ (MOZUClient *)clientForGetReturnsOperationWithStartIndex:(NSNumber *)startIndex pageSize:(NSNumber *)pageSize sortBy:(NSString *)sortBy filter:(NSString *)filter responseFields:(NSString *)responseFields;

/**
Retrieves a list of the actions available to perform for the specified return based on its current state.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForGetAvailableReturnActionsOperationWithReturnId:(NSString *)returnId;

/**
Retrieves the details of a single return item.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
@param returnItemId Unique identifier of the return item whose details you want to get.
*/

+ (MOZUClient *)clientForGetReturnItemOperationWithReturnId:(NSString *)returnId returnItemId:(NSString *)returnItemId responseFields:(NSString *)responseFields;

/**
Retrieves the details of all return items in an order.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForGetReturnItemsOperationWithReturnId:(NSString *)returnId responseFields:(NSString *)responseFields;

/**
Retrieves a list of the payment actions available to perform for the specified return when a return results in a refund to the customer.
@param paymentId Unique identifier of the payment for which to perform the action.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForGetAvailablePaymentActionsForReturnOperationWithReturnId:(NSString *)returnId paymentId:(NSString *)paymentId;

/**
Retrieves the details of a payment submitted as part of a refund associated with a customer return.
@param paymentId Unique identifier of the payment for which to perform the action.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForGetPaymentOperationWithReturnId:(NSString *)returnId paymentId:(NSString *)paymentId responseFields:(NSString *)responseFields;

/**
Retrieves a list of all payments submitted as part of a refund associated with a customer return.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForGetPaymentsOperationWithReturnId:(NSString *)returnId responseFields:(NSString *)responseFields;

/**
Retrieves a list of properties for the specified return.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForGetReturnOperationWithReturnId:(NSString *)returnId responseFields:(NSString *)responseFields;

/**
commerce-returns Get GetReasons description DOCUMENT_HERE
@param responseFields A list or array of fields returned for a call. These fields may be customized and may be used for various types of data calls in Mozu. For example, responseFields are returned for retrieving or updating attributes, carts, and messages in Mozu.
*/

+ (MOZUClient *)clientForGetReasonsOperationWithResponseFields:(NSString *)responseFields;

//
#pragma mark -
#pragma mark Post Operations
#pragma mark -
//

/**
Creates a return for previously fulfilled items. Each return must either be associated with an original order or a product definition to represent each returned item.
@param body Properties of a return of one or more previously fulfilled items.
@param responseFields Use this field to include those fields which are not included by default.
*/

+ (MOZUClient *)clientForCreateReturnOperationWithBody:(MOZUReturn *)body responseFields:(NSString *)responseFields;

/**
Adds a return item to the return.
@param body Properties of a previously fulfilled item associated with a return.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForCreateReturnItemOperationWithBody:(MOZUReturnItem *)body returnId:(NSString *)returnId responseFields:(NSString *)responseFields;

/**
Updates a refund payment associated with a customer return by performing the specified action.
@param body Properties of the payment action performed for an order.
@param paymentId Unique identifier of the payment for which to perform the action.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForPerformPaymentActionForReturnOperationWithBody:(MOZUPaymentAction *)body returnId:(NSString *)returnId paymentId:(NSString *)paymentId responseFields:(NSString *)responseFields;

/**
Creates a new payment for a return that results in a refund to the customer.
@param body Properties of the payment action performed for an order.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForCreatePaymentActionForReturnOperationWithBody:(MOZUPaymentAction *)body returnId:(NSString *)returnId responseFields:(NSString *)responseFields;

/**
Updates the return by performing the action specified in the request.
@param body Properties of an action a user can perform for a return.
@param responseFields Use this field to include those fields which are not included by default.
*/

+ (MOZUClient *)clientForPerformReturnActionsOperationWithBody:(MOZUReturnAction *)body responseFields:(NSString *)responseFields;

//
#pragma mark -
#pragma mark Put Operations
#pragma mark -
//

/**
Updates one or more properties of a return for items previously shipped in a completed order.
@param body Properties of a return of one or more previously fulfilled items.
@param responseFields Use this field to include those fields which are not included by default.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForUpdateReturnOperationWithBody:(MOZUReturn *)body returnId:(NSString *)returnId responseFields:(NSString *)responseFields;

/**
commerce-returns Put ResendReturnEmail description DOCUMENT_HERE
@param body Properties of an action a user can perform for a return.
*/

+ (MOZUClient *)clientForResendReturnEmailOperationWithBody:(MOZUReturnAction *)body;

//
#pragma mark -
#pragma mark Delete Operations
#pragma mark -
//

/**
Removes a particular order item from the order of the current shopper.
@param returnId Unique identifier of the return whose items you want to get.
@param returnItemId Unique identifier of the return item whose details you want to get.
*/

+ (MOZUClient *)clientForDeleteOrderItemOperationWithReturnId:(NSString *)returnId returnItemId:(NSString *)returnItemId;

/**
Deletes the return specified in the request.
@param returnId Unique identifier of the return whose items you want to get.
*/

+ (MOZUClient *)clientForDeleteReturnOperationWithReturnId:(NSString *)returnId;

@end


text:
void Overlay_BuildBasisOrigin( doverlay_t *pOverlay );

// Overlay list.
CUtlVector<mapoverlay_t> g_aMapOverlays;
CUtlVector<mapoverlay_t> g_aMapWaterOverlays;

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void Overlay_GetFromEntity( entity_t *pMapEnt )
{
	// Allocate the new overlay.
	int iOverlay = g_aMapOverlays.AddToTail();
	mapoverlay_t *pMapOverlay = &g_aMapOverlays[iOverlay];

	// Get the overlay data.
	pMapOverlay->nId = g_aMapOverlays.Count() - 1;

	pMapOverlay->flU[0] = FloatForKey( pMapEnt, "StartU" );
	pMapOverlay->flU[1] = FloatForKey( pMapEnt, "EndU" );
	pMapOverlay->flV[0] = FloatForKey( pMapEnt, "StartV" );
	pMapOverlay->flV[1] = FloatForKey( pMapEnt, "EndV" );

	GetVectorForKey( pMapEnt, "BasisOrigin", pMapOverlay->vecOrigin );

	pMapOverlay->m_nRenderOrder = IntForKey( pMapEnt, "RenderOrder" );
	if ( pMapOverlay->m_nRenderOrder < 0 || pMapOverlay->m_nRenderOrder >= OVERLAY_NUM_RENDER_ORDERS )
		Error( "Overlay at %f %f %f has invalid render order (%d).\n", pMapOverlay->vecOrigin );

	GetVectorForKey( pMapEnt, "uv0", pMapOverlay->vecUVPoints[0] );
	GetVectorForKey( pMapEnt, "uv1", pMapOverlay->vecUVPoints[1] );
	GetVectorForKey( pMapEnt, "uv2", pMapOverlay->vecUVPoints[2] );
	GetVectorForKey( pMapEnt, "uv3", pMapOverlay->vecUVPoints[3] );

	GetVectorForKey( pMapEnt, "BasisU", pMapOverlay->vecBasis[0] );
	GetVectorForKey( pMapEnt, "BasisV", pMapOverlay->vecBasis[1] );
	GetVectorForKey( pMapEnt, "BasisNormal", pMapOverlay->vecBasis[2] );

	const char *pMaterialName = ValueForKey( pMapEnt, "material" );
	Assert( strlen( pMaterialName ) < OVERLAY_MAP_STRLEN );
	if ( strlen( pMaterialName ) >= OVERLAY_MAP_STRLEN )
	{
		Error( "Overlay Material Name > OVERLAY_MAP_STRLEN" );
		return;
	}
	strcpy( pMapOverlay->szMaterialName, pMaterialName );

	// Convert the sidelist to side id(s).
	const char *pSideList = ValueForKey( pMapEnt, "sides" );
	char *pTmpList = ( char* )_alloca( strlen( pSideList ) + 1 );
	strcpy( pTmpList, pSideList );
	const char *pScan = strtok( pTmpList, " " );
	if ( !pScan )
		return;

	pMapOverlay->aSideList.Purge();
	pMapOverlay->aFaceList.Purge();

	do
	{
		int nSideId;
		if ( sscanf( pScan, "%d", &nSideId ) == 1 )
		{
			pMapOverlay->aSideList.AddToTail( nSideId );
		}
	} while ( ( pScan = strtok( NULL, " " ) ) );
}

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
side_t *GetSide( int nSideId )
{
	for( int iSide = 0; iSide < nummapbrushsides; ++iSide )
	{
		if ( brushsides[iSide].id == nSideId )
			return &brushsides[iSide];
	}

	return NULL;
}

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void Overlay_UpdateSideLists( void )
{
	int nMapOverlayCount = g_aMapOverlays.Count();
	for( int iMapOverlay = 0; iMapOverlay < nMapOverlayCount; ++iMapOverlay )
	{
		mapoverlay_t *pMapOverlay = &g_aMapOverlays.Element( iMapOverlay );
		if ( pMapOverlay )
		{
			int nSideCount = pMapOverlay->aSideList.Count();
			for( int iSide = 0; iSide < nSideCount; ++iSide )
			{
				side_t *pSide = GetSide( pMapOverlay->aSideList[iSide] );
				if ( pSide )
				{
					if ( pSide->aOverlayIds.Find( pMapOverlay->nId ) == -1 )
					{
						pSide->aOverlayIds.AddToTail( pMapOverlay->nId );
					}
				}
			}
		}
	}
}

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void OverlayTransition_UpdateSideLists( void )
{
	int nOverlayCount = g_aMapWaterOverlays.Count();
	for( int iOverlay = 0; iOverlay < nOverlayCount; ++iOverlay )
	{
		mapoverlay_t *pOverlay = &g_aMapWaterOverlays.Element( iOverlay );
		if ( pOverlay )
		{
			int nSideCount = pOverlay->aSideList.Count();
			for( int iSide = 0; iSide < nSideCount; ++iSide )
			{
				side_t *pSide = GetSide( pOverlay->aSideList[iSide] );
				if ( pSide )
				{
					if ( pSide->aWaterOverlayIds.Find( pOverlay->nId ) == -1 )
					{
						pSide->aWaterOverlayIds.AddToTail( pOverlay->nId );
					}
				}
			}
		}
	}
}

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void Overlay_AddFaceToLists( int iFace, side_t *pSide )
{
	int nOverlayIdCount = pSide->aOverlayIds.Count();
	for( int iOverlayId = 0; iOverlayId < nOverlayIdCount; ++iOverlayId )
	{
		mapoverlay_t *pMapOverlay = &g_aMapOverlays.Element( pSide->aOverlayIds[iOverlayId] );
		if ( pMapOverlay )
		{
			if( pMapOverlay->aFaceList.Find( iFace ) == -1 )
			{
				pMapOverlay->aFaceList.AddToTail( iFace );
			}
		}
	}
}

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void OverlayTransition_AddFaceToLists( int iFace, side_t *pSide )
{
	int nOverlayIdCount = pSide->aWaterOverlayIds.Count();
	for( int iOverlayId = 0; iOverlayId < nOverlayIdCount; ++iOverlayId )
	{
		mapoverlay_t *pMapOverlay = &g_aMapWaterOverlays.Element( pSide->aWaterOverlayIds[iOverlayId] - ( MAX_MAP_OVERLAYS + 1 ) );
		if ( pMapOverlay )
		{
			if( pMapOverlay->aFaceList.Find( iFace ) == -1 )
			{
				pMapOverlay->aFaceList.AddToTail( iFace );
			}
		}
	}
}

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void Overlay_EmitOverlayFace( mapoverlay_t *pMapOverlay )
{
	Assert( g_nOverlayCount < MAX_MAP_OVERLAYS );
	if ( g_nOverlayCount >= MAX_MAP_OVERLAYS )
	{
		Error ( "g_nOverlayCount >= MAX_MAP_OVERLAYS" );
		return;
	}

	doverlay_t *pOverlay = &g_Overlays[g_nOverlayCount];
	g_nOverlayCount++;

	// Conver the map overlay into a .bsp overlay (doverlay_t).
	if ( pOverlay )
	{
		pOverlay->nId = pMapOverlay->nId;

		pOverlay->flU[0] = pMapOverlay->flU[0];
		pOverlay->flU[1] = pMapOverlay->flU[1];
		pOverlay->flV[0] = pMapOverlay->flV[0];
		pOverlay->flV[1] = pMapOverlay->flV[1];

		VectorCopy( pMapOverlay->vecUVPoints[0], pOverlay->vecUVPoints[0] );
		VectorCopy( pMapOverlay->vecUVPoints[1], pOverlay->vecUVPoints[1] );
		VectorCopy( pMapOverlay->vecUVPoints[2], pOverlay->vecUVPoints[2] );
		VectorCopy( pMapOverlay->vecUVPoints[3], pOverlay->vecUVPoints[3] );

		VectorCopy( pMapOverlay->vecOrigin, pOverlay->vecOrigin );

		VectorCopy( pMapOverlay->vecBasis[2], pOverlay->vecBasisNormal );

		pOverlay->SetRenderOrder( pMapOverlay->m_nRenderOrder );

		// Encode the BasisU into the unused z component of the vecUVPoints 0, 1, 2
		pOverlay->vecUVPoints[0].z = pMapOverlay->vecBasis[0].x;
		pOverlay->vecUVPoints[1].z = pMapOverlay->vecBasis[0].y;
		pOverlay->vecUVPoints[2].z = pMapOverlay->vecBasis[0].z;

		// Encode whether or not the v axis should be flipped.
		Vector vecCross = pMapOverlay->vecBasis[2].Cross( pMapOverlay->vecBasis[0] );
		if ( vecCross.Dot( pMapOverlay->vecBasis[1] ) < 0.0f )
		{
			pOverlay->vecUVPoints[3].z = 1.0f;
		}

		// Texinfo.
		texinfo_t texInfo;
		texInfo.flags = 0;
		texInfo.texdata = FindOrCreateTexData( pMapOverlay->szMaterialName );
		for( int iVec = 0; iVec < 2; ++iVec )
		{
			for( int iAxis = 0; iAxis < 3; ++iAxis )
			{
				texInfo.lightmapVecsLuxelsPerWorldUnits[iVec][iAxis] = 0.0f;
				texInfo.textureVecsTexelsPerWorldUnits[iVec][iAxis] = 0.0f;
			}

			texInfo.lightmapVecsLuxelsPerWorldUnits[iVec][3] = -99999.0f;
			texInfo.textureVecsTexelsPerWorldUnits[iVec][3] = -99999.0f;
		}
		pOverlay->nTexInfo = FindOrCreateTexInfo( texInfo );

		// Face List
		int nFaceCount = pMapOverlay->aFaceList.Count();
		Assert( nFaceCount < OVERLAY_BSP_FACE_COUNT );
		if ( nFaceCount >= OVERLAY_BSP_FACE_COUNT )
		{
			Error( "Face List Count >= OVERLAY_BSP_FACE_COUNT" );
			return;
		}

		pOverlay->SetFaceCount( nFaceCount );
		for( int iFace = 0; iFace < nFaceCount; ++iFace )
		{
			pOverlay->aFaces[iFace] = pMapOverlay->aFaceList.Element( iFace );
		}
	}
}

//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
void OverlayTransition_EmitOverlayFace( mapoverlay_t *pMapOverlay )
{
	Assert( g_nWaterOverlayCount < MAX_MAP_WATEROVERLAYS );
	if ( g_nWaterOverlayCount >= MAX_MAP_WATEROVERLAYS )
	{
		Error ( "g_nWaterOverlayCount >= MAX_MAP_WATEROVERLAYS" );
		return;
	}

	dwateroverlay_t *pOverlay = &g_WaterOverlays[g_nWaterOverlayCount];
	g_nWaterOverlayCount++;

	// Conver the map overlay into a .bsp overlay (doverlay_t).
	if ( pOverlay )
	{
		pOverlay->nId = pMapOverlay->nId;

		pOverlay->flU[0] = pMapOverlay->flU[0];
		pOverlay->flU[1] = pMapOverlay->flU[1];
		pOverlay->flV[0] = pMapOverlay->flV[0];
		pOverlay->flV[1] = pMapOverlay->flV[1];

		VectorCopy( pMapOverlay->vecUVPoints[0], pOverlay->vecUVPoints[0] );
		VectorCopy( pMapOverlay->vecUVPoints[1], pOverlay->vecUVPoints[1] );
		VectorCopy( pMapOverlay->vecUVPoints[2], pOverlay->vecUVPoints[2] );
		VectorCopy( pMapOverlay->vecUVPoints[3], pOverlay->vecUVPoints[3] );

		VectorCopy( pMapOverlay->vecOrigin, pOverlay->vecOrigin );

		VectorCopy( pMapOverlay->vecBasis[2], pOverlay->vecBasisNormal );

		pOverlay->SetRenderOrder( pMapOverlay->m_nRenderOrder );

		// Encode the BasisU into the unused z component of the vecUVPoints 0, 1, 2
		pOverlay->vecUVPoints[0].z = pMapOverlay->vecBasis[0].x;
		pOverlay->vecUVPoints[1].z = pMapOverlay->vecBasis[0].y;
		pOverlay->vecUVPoints[2].z = pMapOverlay->vecBasis[0].z;

		// Encode whether or not the v axis should be flipped.
		Vector vecCross = pMapOverlay->vecBasis[2].Cross( pMapOverlay->vecBasis[0] );
		if ( vecCross.Dot( pMapOverlay->vecBasis[1] ) < 0.0f )
		{
			pOverlay->vecUVPoints[3].z = 1.0f;
		}

		// Texinfo.
		texinfo_t texInfo;
		texInfo.flags = 0;
		texInfo.texdata = FindOrCreateTexData( pMapOverlay->szMaterialName );
		for( int iVec = 0; iVec < 2; ++iVec )
		{
			for( int iAxis = 0; iAxis < 3; ++iAxis )
			{
				texInfo.lightmapVecsLuxelsPerWorldUnits[iVec][iAxis] = 0.0f;
				texInfo.textureVecsTexelsPerWorldUnits[iVec][iAxis] = 0.0f;
			}

			texInfo.lightmapVecsLuxelsPerWorldUnits[iVec][3] = -99999.0f;
			texInfo.textureVecsTexelsPerWorldUnits[iVec][3] = -99999.0f;
		}
		pOverlay->nTexInfo = FindOrCreateTexInfo( texInfo );

		// Face List
		int nFaceCount = pMapOverlay->aFaceList.Count();
		Assert( nFaceCount < WATEROVERLAY_BSP_FACE_COUNT );
		if ( nFaceCount >= WATEROVERLAY_BSP_FACE_COUNT )
		{
			Error( "Water Face List Count >= WATEROVERLAY_BSP_FACE_COUNT" );
			return;
		}

		pOverlay->SetFaceCount( nFaceCount );
		for( int iFace = 0; iFace < nFaceCount; ++iFace )
		{
			pOverlay->aFaces[iFace] = pMapOverlay->aFaceList.Element( iFace );
		}
	}
}

//-----------------------------------------------------------------------------
// Purpose:
//-----------------------------------------------------------------------------
void Overlay_EmitOverlayFaces( void )
{
	int nMapOverlayCount = g_aMapOverlays.Count();
	for( int iMapOverlay = 0; iMapOverlay < nMapOverlayCount; ++iMapOverlay )
	{
		Overlay_EmitOverlayFace( &g_aMapOverlays.Element( iMapOverlay ) );
	}
}

//-----------------------------------------------------------------------------
// Purpose:
//-----------------------------------------------------------------------------
void OverlayTransition_EmitOverlayFaces( void )
{
	int nMapOverlayCount = g_aMapWaterOverlays.Count();
	for( int iMapOverlay = 0; iMapOverlay < nMapOverlayCount; ++iMapOverlay )
	{
		OverlayTransition_EmitOverlayFace( &g_aMapWaterOverlays.Element( iMapOverlay ) );
	}
}


text:
from optparse import OptionParser, OptionGroup
import glob
import h5py
import json
import pdb
import os
import shutil
import sys

import numpy as np
import pandas as pd

import slurm
import util

from basenji_test_folds import stat_tests

"""
basenji_bench_phylop_folds.py

Benchmark Basenji model replicates on BED PhyloP task.
"""

################################################################################
# main
################################################################################
def main():
  usage = 'usage: %prog [options] <params_file> <data_dir>'
  parser = OptionParser(usage)

  # sad options
  sad_options = OptionGroup(parser, 'basenji_sad.py options')
  sad_options.add_option('-f', dest='genome_fasta',
      default='%s/data/hg38.fa' % os.environ['BASENJIDIR'],
      help='Genome FASTA for sequences [Default: %default]')
  sad_options.add_option('-n', dest='norm_file',
      default=None,
      help='Normalize SAD scores')
  sad_options.add_option('-o',dest='out_dir',
      default='gtex',
      help='Output directory for tables and plots [Default: %default]')
  sad_options.add_option('--pseudo', dest='log_pseudo',
      default=1, type='float',
      help='Log2 pseudocount [Default: %default]')
  sad_options.add_option('--rc', dest='rc',
      default=False, action='store_true',
      help='Average forward and reverse complement predictions [Default: %default]')
  sad_options.add_option('--shifts', dest='shifts',
      default='0', type='str',
      help='Ensemble prediction shifts [Default: %default]')
  sad_options.add_option('--stats', dest='sad_stats',
      default='SAD',
      help='Comma-separated list of stats to save. [Default: %default]')
  sad_options.add_option('-t', dest='targets_file',
      default=None, type='str',
      help='File specifying target indexes and labels in table format')
  sad_options.add_option('--ti', dest='track_indexes',
      default=None, type='str',
      help='Comma-separated list of target indexes to output BigWig tracks')
  sad_options.add_option('--threads', dest='threads',
      default=False, action='store_true',
      help='Run CPU math and output in a separate thread [Default: %default]')
  sad_options.add_option('-u', dest='penultimate',
      default=False, action='store_true',
      help='Compute SED in the penultimate layer [Default: %default]')
  parser.add_option_group(sad_options)

  # classify
  class_options = OptionGroup(parser, 'basenji_bench_classify.py options')
  class_options.add_option('--msl', dest='msl',
      default=1, type='int',
      help='Random forest min_samples_leaf [Default: %default]')
  parser.add_option_group(class_options)

  # cross-fold
  fold_options = OptionGroup(parser, 'cross-fold options')
  # fold_options.add_option('-a', '--alt', dest='alternative',
  #     default='two-sided', help='Statistical test alternative [Default: %default]')
  fold_options.add_option('-c', dest='crosses',
      default=1, type='int',
      help='Number of cross-fold rounds [Default:%default]')
  fold_options.add_option('-d', dest='data_head',
      default=None, type='int',
      help='Index for dataset/head [Default: %default]')
  fold_options.add_option('-e', dest='conda_env',
      default='tf28',
      help='Anaconda environment [Default: %default]')
  fold_options.add_option('-g', dest='gtex_vcf_dir',
      default='/home/drk/seqnn/data/gtex_fine/susie_pip90')
  # fold_options.add_option('--label_exp', dest='label_exp',
  #     default='Experiment', help='Experiment label [Default: %default]')
  # fold_options.add_option('--label_ref', dest='label_ref',
  #     default='Reference', help='Reference label [Default: %default]')
  fold_options.add_option('--name', dest='name',
      default='gtex', help='SLURM name prefix [Default: %default]')
  fold_options.add_option('--max_proc', dest='max_proc',
      default=None, type='int',
      help='Maximum concurrent processes [Default: %default]')
  fold_options.add_option('-p', dest='processes',
      default=None, type='int',
      help='Number of processes, passed by multi script. \
            (Unused, but needs to appear as dummy.)')
  fold_options.add_option('-q', dest='queue',
      default='geforce',
      help='SLURM queue on which to run the jobs [Default: %default]')
  fold_options.add_option('-r', dest='restart',
      default=False, action='store_true',
      help='Restart a partially completed job [Default: %default]')
  parser.add_option_group(fold_options)

  (options, args) = parser.parse_args()

  if len(args) != 2:
    parser.error('Must provide parameters file and cross-fold directory')
  else:
    params_file = args[0]
    exp_dir = args[1]

  #######################################################
  # prep work

  # count folds
  num_folds = 0
  fold0_dir = '%s/f%dc0' % (exp_dir, num_folds)
  model_file = '%s/train/model_best.h5' % fold0_dir
  if options.data_head is not None:
    model_file = '%s/train/model%d_best.h5' % (fold0_dir, options.data_head)
  while os.path.isfile(model_file):
    num_folds += 1
    fold0_dir = '%s/f%dc0' % (exp_dir, num_folds)
    model_file = '%s/train/model_best.h5' % fold0_dir
    if options.data_head is not None:
      model_file = '%s/train/model%d_best.h5' % (fold0_dir, options.data_head)
  print('Found %d folds' % num_folds)
  if num_folds == 0:
    exit(1)

  ################################################################
  # SAD

  jobs = []

  for ci in range(options.crosses):
    for fi in range(num_folds):
      it_dir = '%s/f%dc%d' % (exp_dir, fi, ci)
      name = '%s-f%dc%d' % (options.name, fi, ci)

      # update output directory
      it_out_dir = '%s/%s' % (it_dir, options.out_dir)
      os.makedirs(it_out_dir, exist_ok=True)

      # SAD command base
      cmd_base = '. /home/drk/anaconda3/etc/profile.d/conda.sh;'
      cmd_base += ' conda activate %s;' % options.conda_env
      cmd_base += ' echo $HOSTNAME;'

      model_file = '%s/train/model_best.h5' % it_dir
      if options.data_head is not None:
        model_file = '%s/train/model%d_best.h5' % (it_dir, options.data_head)

      cmd_base += ' time basenji_sad.py %s %s' % (params_file, model_file)

      for gtex_pos_vcf in glob.glob('%s/*_pos.vcf' % options.gtex_vcf_dir):
        # positive job
        job_base = os.path.splitext(os.path.split(gtex_pos_vcf)[1])[0]
        sad_out_dir = '%s/%s' % (it_out_dir, job_base)
        if not options.restart or not complete_h5('%s/sad.h5'%sad_out_dir):
          cmd_sad = '%s %s' % (cmd_base, gtex_pos_vcf)
          cmd_sad += ' %s' % options_string(options, sad_options, sad_out_dir)
          name = '%s_%s' % (options.name, job_base)
          j = slurm.Job(cmd_sad, name,
              '%s.out'%sad_out_dir, '%s.err'%sad_out_dir,
              queue=options.queue, gpu=1,
              mem=22000, time='2-0:0:0')
          jobs.append(j)

        # negative job
        gtex_neg_vcf = gtex_pos_vcf.replace('_pos.','_neg.')
        job_base = os.path.splitext(os.path.split(gtex_neg_vcf)[1])[0]
        sad_out_dir = '%s/%s' % (it_out_dir, job_base)
        if not options.restart or not complete_h5('%s/sad.h5'%sad_out_dir):
          cmd_sad = '%s %s' % (cmd_base, gtex_neg_vcf)
          cmd_sad += ' %s' % options_string(options, sad_options, sad_out_dir)
          name = '%s_%s' % (options.name, job_base)
          j = slurm.Job(cmd_sad, name,
              '%s.out'%sad_out_dir, '%s.err'%sad_out_dir,
              queue=options.queue, gpu=1,
              mem=22000, time='2-0:0:0')
          jobs.append(j)

  slurm.multi_run(jobs, max_proc=options.max_proc, verbose=True,
                  launch_sleep=10, update_sleep=60)

  ################################################################
  # ensemble
  ################################################################
  ensemble_dir = '%s/ensemble' % exp_dir
  if not os.path.isdir(ensemble_dir):
    os.mkdir(ensemble_dir)

  gtex_dir = '%s/%s' % (ensemble_dir, options.out_dir)
  if not os.path.isdir(gtex_dir):
    os.mkdir(gtex_dir)

  for gtex_pos_vcf in glob.glob('%s/*_pos.vcf' % options.gtex_vcf_dir):
    gtex_neg_vcf = gtex_pos_vcf.replace('_pos.','_neg.')
    pos_base = os.path.splitext(os.path.split(gtex_pos_vcf)[1])[0]
    neg_base = os.path.splitext(os.path.split(gtex_neg_vcf)[1])[0]

    # collect SAD files
    sad_pos_files = []
    sad_neg_files = []
    for ci in range(options.crosses):
      for fi in range(num_folds):
        it_dir = '%s/f%dc%d' % (exp_dir, fi, ci)
        it_out_dir = '%s/%s' % (it_dir, options.out_dir)

        sad_pos_file = '%s/%s/sad.h5' % (it_out_dir, pos_base)
        sad_pos_files.append(sad_pos_file)

        sad_neg_file = '%s/%s/sad.h5' % (it_out_dir, neg_base)
        sad_neg_files.append(sad_neg_file)

    # ensemble
    ens_pos_dir = '%s/%s' % (gtex_dir, pos_base)
    os.makedirs(ens_pos_dir, exist_ok=True)
    ens_pos_file = '%s/sad.h5' % (ens_pos_dir)
    if not os.path.isfile(ens_pos_file):
      ensemble_sad_h5(ens_pos_file, sad_pos_files)

    ens_neg_dir = '%s/%s' % (gtex_dir, neg_base)
    os.makedirs(ens_neg_dir, exist_ok=True)
    ens_neg_file = '%s/sad.h5' % (ens_neg_dir)
    if not os.path.isfile(ens_neg_file):
      ensemble_sad_h5(ens_neg_file, sad_neg_files)

  ################################################################
  # fit classifiers
  ################################################################

  cmd_base = 'basenji_bench_classify.py -i 100 -p 2 -r 44 -s'
  cmd_base += ' --msl %d' % options.msl

  jobs = []
  for ci in range(options.crosses):
    for fi in range(num_folds):
      it_dir = '%s/f%dc%d' % (exp_dir, fi, ci)
      it_out_dir = '%s/%s' % (it_dir, options.out_dir)

      for gtex_pos_vcf in glob.glob('%s/*_pos.vcf' % options.gtex_vcf_dir):
        tissue = os.path.splitext(os.path.split(gtex_pos_vcf)[1])[0][:-4]
        sad_pos = '%s/%s_pos/sad.h5' % (it_out_dir, tissue)
        sad_neg = '%s/%s_neg/sad.h5' % (it_out_dir, tissue)
        class_out_dir = '%s/%s_class' % (it_out_dir, tissue)

        if not options.restart or not os.path.isfile('%s/stats.txt' % class_out_dir):
          cmd_class = '%s -o %s %s %s' % (cmd_base, class_out_dir, sad_pos, sad_neg)
          j = slurm.Job(cmd_class, tissue,
              '%s.out'%class_out_dir, '%s.err'%class_out_dir,
              queue='standard', cpu=2,
              mem=22000, time='1-0:0:0')
          jobs.append(j)

  # ensemble
  for gtex_pos_vcf in glob.glob('%s/*_pos.vcf' % options.gtex_vcf_dir):
    tissue = os.path.splitext(os.path.split(gtex_pos_vcf)[1])[0][:-4]
    sad_pos = '%s/%s_pos/sad.h5' % (gtex_dir, tissue)
    sad_neg = '%s/%s_neg/sad.h5' % (gtex_dir, tissue)
    class_out_dir = '%s/%s_class' % (gtex_dir, tissue)

    if not options.restart or not os.path.isfile('%s/stats.txt' % class_out_dir):
      cmd_class = '%s -o %s %s %s' % (cmd_base, class_out_dir, sad_pos, sad_neg)
      j = slurm.Job(cmd_class, tissue,
          '%s.out'%class_out_dir, '%s.err'%class_out_dir,
          queue='standard', cpu=2,
          mem=22000, time='1-0:0:0')
      jobs.append(j)

  slurm.multi_run(jobs, verbose=True)

def complete_h5(h5_file):
  if os.path.isfile(h5_file):
    try:
      h5_open = h5py.File(h5_file, 'r')
      sad = h5_open['SAD'][:]
      if (sad != 0).sum() > 0:
        return True
      else:
        return False
      h5_open.close()
      return True
    except:
      return False
  else:
    return

def ensemble_sad_h5(ensemble_h5_file, scores_files):
  # open ensemble
  ensemble_h5 = h5py.File(ensemble_h5_file, 'w')

  # transfer base
  base_keys = ['alt_allele','chr','pos','ref_allele','snp','target_ids','target_labels']
  sad_stats = []
  sad_shapes = []
  scores0_h5 = h5py.File(scores_files[0], 'r')
  for key in scores0_h5.keys():
    if key in base_keys:
      ensemble_h5.create_dataset(key, data=scores0_h5[key])
    else:
      sad_stats.append(key)
      sad_shapes.append(scores0_h5[key].shape)
  scores0_h5.close()

  # average stats
  num_folds = len(scores_files)
  for si, sad_stat in enumerate(sad_stats):
    # initialize ensemble array
    sad_values = np.zeros(shape=sad_shapes[si], dtype='float32')

    # read and add folds
    for scores_file in scores_files:
      with h5py.File(scores_file, 'r') as scores_h5:
        sad_values += scores_h5[sad_stat][:].astype('float32')

    # normalize and downcast
    sad_values /= num_folds
    sad_values = sad_values.astype('float16')

    # save
    ensemble_h5.create_dataset(sad_stat, data=sad_values)

  ensemble_h5.close()

def options_string(options, group_options, rep_dir):
  options_str = ''

  for opt in group_options.option_list:
    opt_str = opt.get_opt_string()
    opt_value = options.__dict__[opt.dest]

    # wrap askeriks in ""
    if type(opt_value) == str and opt_value.find('*') != -1:
      opt_value = '"%s"' % opt_value

    # no value for bools
    elif type(opt_value) == bool:
      if not opt_value:
        opt_str = ''
      opt_value = ''

    # skip Nones
    elif opt_value is None:
      opt_str = ''
      opt_value = ''

    # modify
    elif opt.dest == 'out_dir':
      opt_value = rep_dir

    options_str += ' %s %s' % (opt_str, opt_value)

  return options_str

################################################################################
# __main__
################################################################################
if __name__ == '__main__':
  main()


text:
import { NgModule } from '@angular/core'
import { CommonModule } from '@angular/common'
import { RouterModule } from '@angular/router';
import { rootRouterConfig } from './app.routes';
import { AppComponent } from './app.component';
import { FormsModule, ReactiveFormsModule } from '@angular/forms';
import { BrowserModule } from '@angular/platform-browser';
import { HttpModule } from '@angular/http';

import { ChatComponent } from './chat/chat.component';
import { TimeAgoPipe } from './chat/timeAgo.pipe';
import { LocationStrategy, HashLocationStrategy } from '@angular/common';

@NgModule({
  declarations: [
    AppComponent,
    ChatComponent,
    TimeAgoPipe
  ],
  imports: [
    BrowserModule,
    FormsModule,
    CommonModule,
    ReactiveFormsModule,
    HttpModule,
    RouterModule.forRoot(rootRouterConfig, { useHash: true })
  ],
  bootstrap: [ AppComponent ]
})
export class AppModule {

}


text:
<?xml version="1.0" encoding="UTF-8"?>

			Het deel van het scherm links boven geeft uw geselecteerde GoldCoin adres weer, deze hebben andere mensen nodig om u GoldCoins te kunnen sturen.
			Tik aan om het adres te wijzigen.

			De QR-code rechts bevat uw GoldCoin adres en kan gebruikt worden om in te scannen door anderen.
			Tik op de QR-code om een grotere versie weer te geven.

			In het midden van het scherm, wordt uw saldo weergegeven.
			De waarde van uw wallet is ook weergegeven een geselecteerde valuta.
			Tik op het saldo om een lijst met alle mogelijke valut weer te geven, tik één van deze aan om deze als standaard te selecteren.

			Het onderste deel van het scherm wordt gebruikt voor uw transactie lijst, in het begin zal deze leeg zijn.
			Transacties in afwachting van bevestiging zullen grijs zijn totdat ze bevestigd worden.
			Tik op een item in de lijst om de afzender of ontvangers aan het adresboek toe te voegen.
			Schuif naar links of rechts om filters toe te passen.

			U kunt GoldCoins verzenden of verzoeken door de knoppen in de actie-balk aan te klikken.
			Meer opties zijn beschikbaar in het opties-menu.


text:
title: Backpressuring in Streams
layout: docs.hbs
---

# Backpressuring in Streams

There is a general problem that occurs during data handling called
[`backpressure`][] and describes a buildup of data behind a buffer during data
transfer. When the receiving end of the transfer has complex operations, or is
slower for whatever reason, there is a tendency for data from the incoming
source to accumulate, like a clog.

To solve this problem, there must be a delegation system in place to ensure a
smooth flow of data from one source to another. Different communities have
resolved this issue uniquely to their programs, Unix pipes and TCP sockets are
good examples of this, and is often times referred to as _flow control_. In
Node.js, streams have been the adopted solution.

The purpose of this guide is to further detail what backpressure is, and how
exactly streams address this in Node.js' source code. The second part of
the guide will introduce suggested best practices to ensure your application's
code is safe and optimized when implementing streams.

We assume a little familiarity with the general definition of
[`backpressure`][], [`Buffer`][], and [`EventEmitters`][] in Node.js, as well as
some experience with [`Stream`][]. If you haven't read through those docs,
it's not a bad idea to take a look at the API documentation first, as it will
help expand your understanding while reading this guide.

## The Problem with Data Handling

In a computer system, data is transferred from one process to another through
pipes, sockets, and signals. In Node.js, we find a similar mechanism called
[`Stream`][]. Streams are great! They do so much for Node.js and almost every
part of the internal codebase utilizes that module. As a developer, you
are more than encouraged to use them too!

```javascript
const readline = require('readline');

// process.stdin and process.stdout are both instances of Streams.
const rl = readline.createInterface({
  input: process.stdin,
  output: process.stdout
});

rl.question('Why should you use streams? ', (answer) => {
  console.log(`Maybe it's ${answer}, maybe it's because they are awesome! :)`);

  rl.close();
});
```

A good example of why the backpressure mechanism implemented through streams is
a great optimization can be demonstrated by comparing the internal system tools
from Node.js' [`Stream`][] implementation.

In one scenario, we will take a large file (approximately ~9gb) and compress it
using the familiar [`zip(1)`][] tool.

```
zip The.Matrix.1080p.mkv
```

While that will take a few minutes to complete, in another shell we may run
a script that takes Node.js' module [`zlib`][], that wraps around another
compression tool, [`gzip(1)`][].

```javascript
const gzip = require('zlib').createGzip();
const fs = require('fs');

const inp = fs.createReadStream('The.Matrix.1080p.mkv');
const out = fs.createWriteStream('The.Matrix.1080p.mkv.gz');

inp.pipe(gzip).pipe(out);
```

To test the results, try opening each compressed file. The file compressed by
the [`zip(1)`][] tool will notify you the file is corrupt, whereas the
compression finished by [`Stream`][] will decompress without error.

> In this example, we use `.pipe()` to get the data source from one end
> to the other. However, notice there are no proper error handlers attached. If
> a chunk of data were to fail to be properly received, the `Readable` source or
> `gzip` stream will not be destroyed. [`pump`][] is a utility tool that would
> properly destroy all the streams in a pipeline if one of them fails or closes,
> and is a must have in this case!

[`pump`][] is only necessary for Node.js 8.x or earlier, as for Node.js 10.x
or later version, [`pipeline`][] is introduced to replace for [`pump`][].
This is a module method to pipe between streams forwarding errors and properly
cleaning up and provide a callback when the pipeline is complete.

Here is an example of using pipeline:

```javascript
const { pipeline } = require('stream');
const fs = require('fs');
const zlib = require('zlib');

// Use the pipeline API to easily pipe a series of streams
// together and get notified when the pipeline is fully done.
// A pipeline to gzip a potentially huge video file efficiently:

pipeline(
  fs.createReadStream('The.Matrix.1080p.mkv'),
  zlib.createGzip(),
  fs.createWriteStream('The.Matrix.1080p.mkv.gz'),
  (err) => {
    if (err) {
      console.error('Pipeline failed', err);
    } else {
      console.log('Pipeline succeeded');
    }
  }
);
```

You can also call [`promisify`][] on pipeline to use it with `async` / `await`:

```javascript
const stream = require('stream');
const fs = require('fs');
const zlib = require('zlib');
const util = require('util');

const pipeline = util.promisify(stream.pipeline);

async function run() {
  try {
    await pipeline(
      fs.createReadStream('The.Matrix.1080p.mkv'),
      zlib.createGzip(),
      fs.createWriteStream('The.Matrix.1080p.mkv.gz'),
    );
    console.log('Pipeline succeeded');
  } catch (err) {
    console.error('Pipeline failed', err);
  }
}
```

## Too Much Data, Too Quickly

There are instances where a [`Readable`][] stream might give data to the
[`Writable`][] much too quickly — much more than the consumer can handle!

When that occurs, the consumer will begin to queue all the chunks of data for
later consumption. The write queue will get longer and longer, and because of
this more data must be kept in memory until the entire process has completed.

Writing to a disk is a lot slower than reading from a disk, thus, when we are
trying to compress a file and write it to our hard disk, backpressure will
occur because the write disk will not be able to keep up with the speed from
the read.

```javascript
// Secretly the stream is saying: "whoa, whoa! hang on, this is way too much!"
// Data will begin to build up on the read-side of the data buffer as
// `write` tries to keep up with the incoming data flow.
inp.pipe(gzip).pipe(outputFile);
```

This is why a backpressure mechanism is important. If a backpressure system was
not present, the process would use up your system's memory, effectively slowing
down other processes, and monopolizing a large part of your system until
completion.

This results in a few things:

* Slowing down all other current processes
* A very overworked garbage collector
* Memory exhaustion

In the following examples we will take out the [return value][] of the
`.write()` function and change it to `true`, which effectively disables
backpressure support in Node.js core. In any reference to 'modified' binary,
we are talking about running the `node` binary without the `return ret;` line,
and instead with the replaced `return true;`.

## Excess Drag on Garbage Collection

Let's take a look at a quick benchmark. Using the same example from above, we
ran a few time trials to get a median time for both binaries.

```
   trial (#)  | `node` binary (ms) | modified `node` binary (ms)
=================================================================
      1       |      56924         |           55011
      2       |      52686         |           55869
      3       |      59479         |           54043
      4       |      54473         |           55229
      5       |      52933         |           59723
=================================================================
average time: |      55299         |           55975
```

Both take around a minute to run, so there's not much of a difference at all,
but let's take a closer look to confirm whether our suspicions are correct. We
use the Linux tool [`dtrace`][] to evaluate what's happening with the V8 garbage
collector.

The GC (garbage collector) measured time indicates the intervals of a full cycle
of a single sweep done by the garbage collector:

```
approx. time (ms) | GC (ms) | modified GC (ms)
=================================================
          0       |    0    |      0
          1       |    0    |      0
         40       |    0    |      2
        170       |    3    |      1
        300       |    3    |      1

         *             *           *
         *             *           *
         *             *           *

      39000       |    6    |     26
      42000       |    6    |     21
      47000       |    5    |     32
      50000       |    8    |     28
      54000       |    6    |     35
```

While the two processes start off the same and seem to work the GC at the same
rate, it becomes evident that after a few seconds with a properly working
backpressure system in place, it spreads the GC load across consistent
intervals of 4-8 milliseconds until the end of the data transfer.

However, when a backpressure system is not in place, the V8 garbage collection
starts to drag out. The normal binary called the GC approximately **75**
times in a minute, whereas, the modified binary fires only **36** times.

This is the slow and gradual debt accumulating from growing memory usage. As
data gets transferred, without a backpressure system in place, more memory is
being used for each chunk transfer.

The more memory that is being allocated, the more the GC has to take care of in
one sweep. The bigger the sweep, the more the GC needs to decide what can be
freed up, and scanning for detached pointers in a larger memory space will
consume more computing power.

## Memory Exhaustion

To determine the memory consumption of each binary, we've clocked each process
with `/usr/bin/time -lp sudo ./node ./backpressure-example/zlib.js`
individually.

This is the output on the normal binary:

```
Respecting the return value of .write()
=============================================
real        58.88
user        56.79
sys          8.79
  87810048  maximum resident set size
         0  average shared memory size
         0  average unshared data size
         0  average unshared stack size
     19427  page reclaims
      3134  page faults
         0  swaps
         5  block input operations
       194  block output operations
         0  messages sent
         0  messages received
         1  signals received
        12  voluntary context switches
    666037  involuntary context switches
```

The maximum byte size occupied by virtual memory turns out to be approximately
87.81 mb.

And now changing the [return value][] of the [`.write()`][] function, we get:

```
Without respecting the return value of .write():
==================================================
real        54.48
user        53.15
sys          7.43
1524965376  maximum resident set size
         0  average shared memory size
         0  average unshared data size
         0  average unshared stack size
    373617  page reclaims
      3139  page faults
         0  swaps
        18  block input operations
       199  block output operations
         0  messages sent
         0  messages received
         1  signals received
        25  voluntary context switches
    629566  involuntary context switches
```

The maximum byte size occupied by virtual memory turns out to be approximately
1.52 gb.

Without streams in place to delegate the backpressure, there is an order of
magnitude greater of memory space being allocated - a huge margin of
difference between the same process!

This experiment shows how optimized and cost-effective Node.js' backpressure
mechanism is for your computing system. Now, let's do a break down on how it
works!

## How Does Backpressure Resolve These Issues?

There are different functions to transfer data from one process to another. In
Node.js, there is an internal built-in function called [`.pipe()`][]. There are
[other packages][] out there you can use too! Ultimately though, at the basic
level of this process, we have two separate components: the _source_ of the
data and the _consumer_.

When [`.pipe()`][] is called from the source, it signals to the consumer that
there is data to be transferred. The pipe function helps to set up the
appropriate backpressure closures for the event triggers.

In Node.js the source is a [`Readable`][] stream and the consumer is the
[`Writable`][] stream (both of these may be interchanged with a [`Duplex`][] or
a [`Transform`][] stream, but that is out-of-scope for this guide).

The moment that backpressure is triggered can be narrowed exactly to the return
value of a [`Writable`][]'s [`.write()`][] function. This return value is
determined by a few conditions, of course.

In any scenario where the data buffer has exceeded the [`highWaterMark`][] or
the write queue is currently busy, [`.write()`][] will return `false`.

When a `false` value is returned, the backpressure system kicks in. It will
pause the incoming [`Readable`][] stream from sending any data and wait until
the consumer is ready again. Once the data buffer is emptied, a [`'drain'`][]
event will be emitted and resume the incoming data flow.

Once the queue is finished, backpressure will allow data to be sent again.
The space in memory that was being used will free itself up and prepare for the
next batch of data.

This effectively allows a fixed amount of memory to be used at any given
time for a [`.pipe()`][] function. There will be no memory leakage, no
infinite buffering, and the garbage collector will only have to deal with
one area in memory!

So, if backpressure is so important, why have you (probably) not heard of it?
Well the answer is simple: Node.js does all of this automatically for you.

That's so great! But also not so great when we are trying to understand how to
implement our own custom streams.

> In most machines, there is a byte size that determines when a buffer
> is full (which will vary across different machines). Node.js allows you to set
> your own custom [`highWaterMark`][], but commonly, the default is set to 16kb
> (16384, or 16 for objectMode streams). In instances where you might
> want to raise that value, go for it, but do so with caution!

## Lifecycle of `.pipe()`

To achieve a better understanding of backpressure, here is a flow-chart on the
lifecycle of a [`Readable`][] stream being [piped][] into a [`Writable`][]
stream:

```
                                                     +===================+
                         x-->  Piping functions   +-->   src.pipe(dest)  |
                         x     are set up during     |===================|
                         x     the .pipe method.     |  Event callbacks  |
  +===============+      x                           |-------------------|
  |   Your Data   |      x     They exist outside    | .on('close', cb)  |
  +=======+=======+      x     the data flow, but    | .on('data', cb)   |
          |              x     importantly attach    | .on('drain', cb)  |
          |              x     events, and their     | .on('unpipe', cb) |
+---------v---------+    x     respective callbacks. | .on('error', cb)  |
|  Readable Stream  +----+                           | .on('finish', cb) |
+-^-------^-------^-+    |                           | .on('end', cb)    |
  ^       |       ^      |                           +-------------------+
  |       |       |      |
  |       ^       |      |
  ^       ^       ^      |    +-------------------+         +=================+
  ^       |       ^      +---->  Writable Stream  +--------->  .write(chunk)  |
  |       |       |           +-------------------+         +=======+=========+
  |       |       |                                                 |
  |       ^       |                              +------------------v---------+
  ^       |       +-> if (!chunk)                |    Is this chunk too big?  |
  ^       |       |     emit .end();             |    Is the queue busy?      |
  |       |       +-> else                       +-------+----------------+---+
  |       ^       |     emit .write();                   |                |
  |       ^       ^                                   +--v---+        +---v---+
  |       |       ^-----------------------------------<  No  |        |  Yes  |
  ^       |                                           +------+        +---v---+
  ^       |                                                               |
  |       ^               emit .pause();          +=================+     |
  |       ^---------------^-----------------------+  return false;  <-----+---+
  |                                               +=================+         |
  |                                                                           |
  ^            when queue is empty     +============+                         |
  ^------------^-----------------------<  Buffering |                         |
               |                       |============|                         |
               +> emit .drain();       |  ^Buffer^  |                         |
               +> emit .resume();      +------------+                         |
                                       |  ^Buffer^  |                         |
                                       +------------+   add chunk to queue    |
                                       |            <---^---------------------<
                                       +============+
```

> If you are setting up a pipeline to chain together a few streams to
> manipulate your data, you will most likely be implementing [`Transform`][]
> stream.

In this case, your output from your [`Readable`][] stream will enter in the
[`Transform`][] and will pipe into the [`Writable`][].

```javascript
Readable.pipe(Transformable).pipe(Writable);
```

Backpressure will be automatically applied, but note that both the incoming and
outgoing `highWaterMark` of the [`Transform`][] stream may be manipulated and
will effect the backpressure system.

## Backpressure Guidelines

Since [Node.js v0.10][], the [`Stream`][] class has offered the ability to
modify the behavior of the [`.read()`][] or [`.write()`][] by using the
underscore version of these respective functions ([`._read()`][] and
[`._write()`][]).

There are guidelines documented for [implementing Readable streams][] and
[implementing Writable streams][]. We will assume you've read these over, and
the next section will go a little bit more in-depth.

## Rules to Abide By When Implementing Custom Streams

The golden rule of streams is **to always respect backpressure**. What
constitutes as best practice is non-contradictory practice. So long as you are
careful to avoid behaviors that conflict with internal backpressure support,
you can be sure you're following good practice.

In general,

1. Never `.push()` if you are not asked.
2. Never call `.write()` after it returns false but wait for 'drain' instead.
3. Streams changes between different Node.js versions, and the library you use.
Be careful and test things.

> In regards to point 3, an incredibly useful package for building
> browser streams is [`readable-stream`][]. Rodd Vagg has written a
> [great blog post][] describing the utility of this library. In short, it
> provides a type of automated graceful degradation for [`Readable`][] streams,
> and supports older versions of browsers and Node.js.

## Rules specific to Readable Streams

So far, we have taken a look at how [`.write()`][] affects backpressure and have
focused much on the [`Writable`][] stream. Because of Node.js' functionality,
data is technically flowing downstream from [`Readable`][] to [`Writable`][].
However, as we can observe in any transmission of data, matter, or energy, the
source is just as important as the destination and the [`Readable`][] stream
is vital to how backpressure is handled.

Both these processes rely on one another to communicate effectively, if
the [`Readable`][] ignores when the [`Writable`][] stream asks for it to stop
sending in data, it can be just as problematic to when the [`.write()`][]'s return
value is incorrect.

So, as well with respecting the [`.write()`][] return, we must also respect the
return value of [`.push()`][] used in the [`._read()`][] method. If
[`.push()`][] returns a `false` value, the stream will stop reading from the
source. Otherwise, it will continue without pause.

Here is an example of bad practice using [`.push()`][]:

```javascript
// This is problematic as it completely ignores return value from push
// which may be a signal for backpressure from the destination stream!
class MyReadable extends Readable {
  _read(size) {
    let chunk;
    while (null !== (chunk = getNextChunk())) {
      this.push(chunk);
    }
  }
}
```

Additionally, from outside the custom stream, there are pitfalls for ignoring
backpressure. In this counter-example of good practice, the application's code
forces data through whenever it is available (signaled by the
[`'data'` event][]):

```javascript
// This ignores the backpressure mechanisms Node.js has set in place,
// and unconditionally pushes through data, regardless if the
// destination stream is ready for it or not.
readable.on('data', (data) =>
  writable.write(data)
);
```

## Rules specific to Writable Streams

Recall that a [`.write()`][] may return true or false dependent on some
conditions. Luckily for us, when building our own [`Writable`][] stream,
the [`stream state machine`][] will handle our callbacks and determine when to
handle backpressure and optimize the flow of data for us.

However, when we want to use a [`Writable`][] directly, we must respect the
[`.write()`][] return value and pay close attention to these conditions:

* If the write queue is busy, [`.write()`][] will return false.
* If the data chunk is too large, [`.write()`][] will return false (the limit
is indicated by the variable, [`highWaterMark`][]).

<!-- eslint-disable indent -->
```javascript
// This writable is invalid because of the async nature of JavaScript callbacks.
// Without a return statement for each callback prior to the last,
// there is a great chance multiple callbacks will be called.
class MyWritable extends Writable {
  _write(chunk, encoding, callback) {
    if (chunk.toString().indexOf('a') >= 0)
      callback();
    else if (chunk.toString().indexOf('b') >= 0)
      callback();
    callback();
  }
}

// The proper way to write this would be:
    if (chunk.contains('a'))
      return callback();
    if (chunk.contains('b'))
      return callback();
    callback();
```

There are also some things to look out for when implementing [`._writev()`][].
The function is coupled with [`.cork()`][], but there is a common mistake when
writing:

```javascript
// Using .uncork() twice here makes two calls on the C++ layer, rendering the
// cork/uncork technique useless.
ws.cork();
ws.write('hello ');
ws.write('world ');
ws.uncork();

ws.cork();
ws.write('from ');
ws.write('Matteo');
ws.uncork();

// The correct way to write this is to utilize process.nextTick(), which fires
// on the next event loop.
ws.cork();
ws.write('hello ');
ws.write('world ');
process.nextTick(doUncork, ws);

ws.cork();
ws.write('from ');
ws.write('Matteo');
process.nextTick(doUncork, ws);

// As a global function.
function doUncork(stream) {
  stream.uncork();
}
```

[`.cork()`][] can be called as many times we want, we just need to be careful to
call [`.uncork()`][] the same amount of times to make it flow again.

## Conclusion

Streams are an often used module in Node.js. They are important to the internal
structure, and for developers, to expand and connect across the Node.js modules
ecosystem.

Hopefully, you will now be able to troubleshoot, safely code your own
[`Writable`][] and [`Readable`][] streams with backpressure in mind, and share
your knowledge with colleagues and friends.

Be sure to read up more on [`Stream`][] for other API functions to help
improve and unleash your streaming capabilities when building an application with
Node.js.

[`Stream`]: https://nodejs.org/api/stream.html
[`Buffer`]: https://nodejs.org/api/buffer.html
[`EventEmitters`]: https://nodejs.org/api/events.html
[`Writable`]: https://nodejs.org/api/stream.html#stream_writable_streams
[`Readable`]: https://nodejs.org/api/stream.html#stream_readable_streams
[`Duplex`]: https://nodejs.org/api/stream.html#stream_duplex_and_transform_streams
[`Transform`]: https://nodejs.org/api/stream.html#stream_duplex_and_transform_streams
[`zlib`]: https://nodejs.org/api/zlib.html
[`'drain'`]: https://nodejs.org/api/stream.html#stream_event_drain
[`'data'` event]: https://nodejs.org/api/stream.html#stream_event_data
[`.read()`]: https://nodejs.org/docs/latest/api/stream.html#stream_readable_read_size
[`.write()`]: https://nodejs.org/api/stream.html#stream_writable_write_chunk_encoding_callback
[`._read()`]: https://nodejs.org/docs/latest/api/stream.html#stream_readable_read_size_1
[`._write()`]: https://nodejs.org/docs/latest/api/stream.html#stream_writable_write_chunk_encoding_callback_1
[`._writev()`]: https://nodejs.org/api/stream.html#stream_writable_writev_chunks_callback
[`.cork()`]: https://nodejs.org/api/stream.html#stream_writable_cork
[`.uncork()`]: https://nodejs.org/api/stream.html#stream_writable_uncork

[`.push()`]: https://nodejs.org/docs/latest/api/stream.html#stream_readable_push_chunk_encoding

[implementing Writable streams]: https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_writable_stream
[implementing Readable streams]: https://nodejs.org/docs/latest/api/stream.html#stream_implementing_a_readable_stream

[other packages]: https://github.com/sindresorhus/awesome-nodejs#streams
[`backpressure`]: https://en.wikipedia.org/wiki/Backpressure_routing
[Node.js v0.10]: https://nodejs.org/docs/v0.10.0/
[`highWaterMark`]: https://nodejs.org/api/stream.html#stream_buffering
[return value]: https://github.com/nodejs/node/blob/55c42bc6e5602e5a47fb774009cfe9289cb88e71/lib/_stream_writable.js#L239

[`readable-stream`]: https://github.com/nodejs/readable-stream
[great blog post]:https://r.va.gg/2014/06/why-i-dont-use-nodes-core-stream-module.html

[`dtrace`]: http://dtrace.org/blogs/about/
[`zip(1)`]: https://linux.die.net/man/1/zip
[`gzip(1)`]: https://linux.die.net/man/1/gzip
[`stream state machine`]: https://en.wikipedia.org/wiki/Finite-state_machine

[`.pipe()`]: https://nodejs.org/docs/latest/api/stream.html#stream_readable_pipe_destination_options
[piped]: https://nodejs.org/docs/latest/api/stream.html#stream_readable_pipe_destination_options
[`pump`]: https://github.com/mafintosh/pump
[`pipeline`]: https://nodejs.org/api/stream.html#stream_stream_pipeline_streams_callback
[`promisify`]: https://nodejs.org/api/util.html#util_util_promisify_original


text:
title: DBTable::indices — Core Library
package: lib
packageTitle: Core Library
layout: default
className: DBTable
type: method
---

# DBTable::indices

## Synopsis

public function DBTable::indices()


text:
/* Evangelos Georganas (Intel Corp.)
******************************************************************************/
#ifndef GENERATOR_MATEQUATION_SCRATCH_AARCH64_H
#define GENERATOR_MATEQUATION_SCRATCH_AARCH64_H
#include "generator_common.h"
#include "libxsmm_main.h"

LIBXSMM_API_INTERN
void libxsmm_generator_matequation_set_input_in_stack_param_struct_aarch64( libxsmm_generated_code*   io_generated_code,
    libxsmm_matequation_kernel_config*                  i_micro_kernel_config,
    libxsmm_matequation_gp_reg_mapping*                 i_gp_reg_mapping,
    libxsmm_matrix_eqn_elem*                            cur_node,
    unsigned int                                        temp_reg,
    unsigned int                                        ptr_id );

LIBXSMM_API_INTERN
void libxsmm_generator_matequation_set_output_in_stack_param_struct_aarch64(libxsmm_generated_code*   io_generated_code,
    libxsmm_matequation_kernel_config*                  i_micro_kernel_config,
    libxsmm_matequation_gp_reg_mapping*                 i_gp_reg_mapping,
    libxsmm_matrix_eqn_elem*                            cur_node,
    unsigned int                                        temp_reg,
    unsigned int                                        is_last_op );

LIBXSMM_API_INTERN
void libxsmm_generator_matequation_tmp_stack_scratch_aarch64_kernel( libxsmm_generated_code* io_generated_code,
    const libxsmm_meqn_descriptor*          i_mateqn_desc,
    libxsmm_matequation_gp_reg_mapping*     i_gp_reg_mapping,
    libxsmm_matequation_kernel_config*      i_micro_kernel_config,
    libxsmm_loop_label_tracker*             io_loop_label_tracker,
    libxsmm_matrix_eqn*                     eqn );

#endif


text:
[![Latest Stable Version](https://poser.pugx.org/vivomedia/fusion-cachevisualisation/v/stable)](https://packagist.org/packages/vivomedia/fusion-cachevisualisation)
[![Total Downloads](https://poser.pugx.org/vivomedia/fusion-cachevisualisation/downloads)](https://packagist.org/packages/vivomedia/fusion-cachevisualisation)
[![License](https://poser.pugx.org/vivomedia/fusion-cachevisualisation/license)](https://packagist.org/packages/vivomedia/fusion-cachevisualisation)
[![Maintainability](https://api.codeclimate.com/v1/badges/7cf5fd14d617b5e8b136/maintainability)](https://codeclimate.com/github/vivomedia-de/fusion-cachevisualisation/maintainability)

# Fusion Cache Visualisation
Helps to debug cache configurations in [Neos Fusion](https://github.com/neos/typoscript/).

This is just a debugging tool and highly recommended to use in development context only.

## What it provides
This plugin adds a wrapper around all cached and uncached segments to visualise them. Also it provides additional information for each segement:

Cached Segments
* Fusion path
* Cache tags
* Lifetime

Uncached Segments
* Fusion path
* Context variable names

# Install
## Composer
Install via composer as a dev package
```bash
php composer.phar require --dev "vivomedia/fusion-cachevisualisation"
```

## Flow
After install clear the content cache of your flow/neos application.
```bash
./flow flow:cache:flush --force
```

## jQuery
The plugin uses jQuery. Please ensure that it's loaded early enough.

# Configuration
You can enable and disable the plugin within your `Settings.yaml`

```yaml
VIVOMEDIA:
  Fusion:
    CacheVisualisation:
      enabled: true # Or false
```

It is enabled by default for the development context.

Please keep in mind to clear the cache after enabling/disabling the plugin.

# Screenshots
![Cached segment](/Docs/screenshot_cached.png?raw=true "Cached segment")
![Unached segment](/Docs/screenshot_uncached.png?raw=true "Uncached segment")


text:
package com.opengamma.util.map;

import java.util.Map;

import com.opengamma.util.tuple.Triple;

/**
 * Variant of {@link Map} that allows three keys to be used, equivalent to a map that has a composite key of three elements.
 *
 * @param <K1> key 1 type
 * @param <K2> key 2 type
 * @param <K3> key 3 type
 * @param <V> value type
 */
public interface Map3<K1, K2, K3, V> extends Map<Triple<K1, K2, K3>, V> {

  /**
   * Returns the element referenced by the given keys.
   *
   * @param key1 the first key
   * @param key2 the second key
   * @param key3 the third key
   * @return the value or null if the key-triple is not in the map
   */
  V get(K1 key1, K2 key2, K3 key3);

  /**
   * Stores a new element in the map.
   *
   * @param key1 the first key
   * @param key2 the second key
   * @param key3 the third key
   * @param value the value to store
   * @return the previously stored value
   */
  V put(K1 key1, K2 key2, K3 key3, V value);

  /**
   * Stores a new element in the map if there is not already one for that key triple.
   *
   * @param key1 the first key
   * @param key2 the second key
   * @param key3 the third key
   * @param value the value to store
   * @return the previously stored value
   */
  V putIfAbsent(K1 key1, K2 key2, K3 key3, V value);

  /**
   * Removes an element from the map.
   *
   * @param key1 the first key
   * @param key2 the second key
   * @param key3 the third key
   * @return the value stored, or null if the key-triple is not in the map
   */
  V remove(K1 key1, K2 key2, K3 key3);

  /**
   * Tests if a key-triple is present in the map.
   *
   * @param key1 the first key
   * @param key2 the second key
   * @param key3 the third key
   * @return true if the key-triple is present, false otherwise
   */
  boolean containsKey(K1 key1, K2 key2, K3 key3);

}