text:
{% extends "admin/layout.html" %}

{% block title %}{%endblock%}

{% block body %}

Dashboard

{% endblock %}


text:
<!doctype html>

    <meta charset="UTF-8">
    <title>Document</title>

    <input type=button value="Back" style="font-size:25px" onclick="{window.history.back()}">

redpajama_set_name: RedPajamaGithub


text:
/*global describe:false, it:false, expect:false, CodeMirror */
describe('codemirror', function(){
    var namespace = CodeMirror;

    it('should have a namespace', function(){
        expect(namespace).toBeDefined();
    });
});


text:
/***
|''Name''|RevisionsCommandPlugin|
|''Description''|provides access to tiddler revisions|
|''Author''|FND|
|''Contributors''|Martin Budden|
|''Version''|0.3.3|
|''Status''|@@beta@@|
|''Source''|http://svn.tiddlywiki.org/Trunk/association/plugins/RevisionsCommandPlugin.js|
|''CodeRepository''|http://svn.tiddlywiki.org/Trunk/association/plugins/|
|''License''|[[BSD|http://www.opensource.org/licenses/bsd-license.php]]|
|''CoreVersion''|2.6.0|
|''Keywords''|serverSide|
!Usage
Extend [[ToolbarCommands]] with {{{revisions}}}.
!Revision History
!!v0.1 (2009-07-23)
* initial release (renamed from experimental ServerCommandsPlugin)
!!v0.2 (2010-03-04)
* suppressed wikification in diff view
!!v0.3 (2010-04-07)
* restored wikification in diff view
* added link to side-by-side diff view
!To Do
* strip server.* fields from revision tiddlers
* resolve naming conflicts
* i18n, l10n
* code sanitizing
* documentation
!Code
***/
//{{{
(function($) {

jQuery.twStylesheet(".diff { white-space: pre, font-family: monospace }",
	{ id: "diff" });

var cmd = config.commands.revisions = {
	type: "popup",
	hideShadow: true,
	text: "revisions",
	tooltip: "display tiddler revisions",
	revTooltip: "", // TODO: populate dynamically?
	loadLabel: "loading...",
	loadTooltip: "loading revision list",
	selectLabel: "select",
	selectTooltip: "select revision for comparison",
	selectedLabel: "selected",
	compareLabel: "compare",
	linkLabel: "side-by-side view",
	revSuffix: " [rev. #%0]",
	diffSuffix: " [diff: #%0 #%1]",
	dateFormat: "YYYY-0MM-0DD 0hh:0mm",
	listError: "revisions could not be retrieved",

	handlePopup: function(popup, title) {
		title = this.stripSuffix("rev", title);
		title = this.stripSuffix("diff", title);
		var tiddler = store.getTiddler(title);
		var type = _getField("server.type", tiddler);
		var adaptor = new config.adaptors[type]();
		var limit = null; // TODO: customizable
		var context = {
			host: _getField("server.host", tiddler),
			workspace: _getField("server.workspace", tiddler)
		};
		var loading = createTiddlyButton(popup, cmd.loadLabel, cmd.loadTooltip);
		var params = { popup: popup, loading: loading, origin: title };
		adaptor.getTiddlerRevisionList(title, limit, context, params, this.displayRevisions);
	},

	displayRevisions: function(context, userParams) {
		removeNode(userParams.loading);
		if(context.status) {
			var callback = function(ev) {
				var e = ev || window.event;
				var revision = resolveTarget(e).getAttribute("revision");
				context.adaptor.getTiddlerRevision(tiddler.title, revision, context,
					userParams, cmd.displayTiddlerRevision);
			};
			var table = createTiddlyElement(userParams.popup, "table");
			for(var i = 0; i < context.revisions.length; i++) {
				var tiddler = context.revisions[i];
				var row = createTiddlyElement(table, "tr");
				var timestamp = tiddler.modified.formatString(cmd.dateFormat);
				var revision = tiddler.fields["server.page.revision"];
				var cell = createTiddlyElement(row, "td");
				createTiddlyButton(cell, timestamp, cmd.revTooltip, callback, null,
					null, null, { revision: revision });
				cell = createTiddlyElement(row, "td", null, null, tiddler.modifier);
				cell = createTiddlyElement(row, "td");
				createTiddlyButton(cell, cmd.selectLabel, cmd.selectTooltip,
					cmd.revisionSelected, null, null, null,
					{ index:i, revision: revision, col: 2 });
				cmd.context = context; // XXX: unsafe (singleton)!?
			}
		} else {
			$("<li />").text(cmd.listError).appendTo(userParams.popup);
		}
	},

	revisionSelected: function(ev) {
		var e = ev || window.event;
		e.cancelBubble = true;
		if(e.stopPropagation) {
			e.stopPropagation();
		}
		var n = resolveTarget(e);
		var index = n.getAttribute("index");
		var col = n.getAttribute("col");
		while(!index || !col) {
			n = n.parentNode;
			index = n.getAttribute("index");
			col = n.getAttribute("col");
		}
		cmd.revision = n.getAttribute("revision");
		var table = n.parentNode.parentNode.parentNode;
		var rows = table.childNodes;
		for(var i = 0; i < rows.length; i++) {
			var c = rows[i].childNodes[col].firstChild;
			if(i == index) {
				if(c.textContent) {
					c.textContent = cmd.selectedLabel;
				} else {
					c.text = cmd.selectedLabel;
				}
			} else {
				if(c.textContent) {
					c.textContent = cmd.compareLabel;
				} else {
					c.text = cmd.compareLabel;
				}
				c.onclick = cmd.compareSelected;
			}
		}
	},

	compareSelected: function(ev) {
		var e = ev || window.event;
		var n = resolveTarget(e);
		var context = cmd.context;
		context.rev1 = n.getAttribute("revision");
		context.rev2 = cmd.revision;
		context.tiddler = context.revisions[n.getAttribute("index")];
		context.format = "unified";
		context.adaptor.getTiddlerDiff(context.tiddler.title, context,
			context.userParams, cmd.displayTiddlerDiffs);
	},

	displayTiddlerDiffs: function(context, userParams) {
		var tiddler = context.tiddler;
		tiddler.title += cmd.diffSuffix.format([context.rev1, context.rev2]);
		tiddler.text = "{{diff{\n" + context.diff + "\n}}}";
		tiddler.tags = ["diff"];
		tiddler.fields.doNotSave = "true"; // XXX: correct?
		if(!store.getTiddler(tiddler.title)) {
			store.addTiddler(tiddler);
		}
		var src = story.getTiddler(userParams.origin);
		var tiddlerEl = story.displayTiddler(src, tiddler);
		var uri = context.uri.replace("format=unified", "format=horizontal");
		var link = $('<a target="_blank" />').attr("href", uri).text(cmd.linkLabel);
		$(".viewer", tiddlerEl).prepend(link);
	},

	displayTiddlerRevision: function(context, userParams) {
		var tiddler = context.tiddler;
		tiddler.title += cmd.revSuffix.format([tiddler.fields["server.page.revision"]]);
		tiddler.fields.doNotSave = "true"; // XXX: correct?
		if(!store.getTiddler(tiddler.title)) {
			store.addTiddler(tiddler);
		}
		var src = story.getTiddler(userParams.origin);
		story.displayTiddler(src, tiddler);
	},

	stripSuffix: function(type, title) {
		var str = cmd[type + "Suffix"];
		var i = str.indexOf("%0");
		i = title.indexOf(str.substr(0, i));
		if(i != -1) {
			title = title.substr(0, i);
		}
		return title;
	}
};

var _getField = function(name, tiddler) {
	return tiddler.fields[name] || config.defaultCustomFields[name];
};

})(jQuery);
//}}}


text:
extern "C" {
#endif

#define PHASTAIO_OPENTIME(cmd) cmd
#define PHASTAIO_CLOSETIME(cmd) cmd
#define PHASTAIO_READTIME(cmd,ignored) cmd
#define PHASTAIO_WRITETIME(cmd,ignored) cmd

enum phastaio_file { GEOMBC_READ, RESTART_READ, RESTART_WRITE };

typedef int phastaioTime;
struct phastaio_stats;
void phastaio_time(phastaioTime*);
size_t phastaio_time_diff(phastaioTime*, phastaioTime*);
void phastaio_addReadBytes(size_t);
void phastaio_addWriteBytes(size_t);
void phastaio_addReadTime(size_t);
void phastaio_addWriteTime(size_t);
void phastaio_setfile(int);
void phastaio_addOpenTime(size_t);
void phastaio_addCloseTime(size_t);
void phastaio_printStats();
void phastaio_initStats();

#ifdef __cplusplus
}
#endif

#endif


text:
@interface ViewController ()

@end

@implementation ViewController

- (void)viewDidLoad {
    [super viewDidLoad];
    // Do any additional setup after loading the view, typically from a nib.
}

- (void)didReceiveMemoryWarning {
    [super didReceiveMemoryWarning];
    // Dispose of any resources that can be recreated.
}

-(IBAction)onPush:(id)sender {
    NSURL * url = [NSURL URLWithString:@"http://github.com"];
    LTWebViewController *webVC = [[LTWebViewController alloc] initWithURL:url type:LTWebViewTypeWKWebView title:@"常用电话"];

    [self.navigationController pushViewController:webVC animated:YES];
}

@end


text:
<#############################################################################
The DoPx module provides a rich set of commands that extend the automation
capabilities of the DigitalOcean (DO) cloud service. These commands make it
easier to manage your DigitalOcean environment from Windows PowerShell. When
used with the LinuxPx module, you can manage your entire DigitalOcean
environment from one shell.

Copyright 2014 Kirk Munro

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
#############################################################################>

<#
.SYNOPSIS
    Removes one or more domains that have been added to your DigitalOcean environment
.DESCRIPTION
    The Remove-DoPxDomain command removes one or more domains that have been added to your DigitalOcean environment

    You can use Remove-DoPxDomain command to remove specific domains by passing the domain names to the Name parameter.
.INPUTS
    digitalocean.domain
.OUTPUTS
    None
.NOTES
    This command sends an HTTP DELETE request that includes your access token to a DigitalOcean REST API v2 endpoint. To learn more about the DigitalOcean REST API, consult the DigitalOcean API documentation online at https://developers.digitalocean.com.
.EXAMPLE
    PS C:\> $accessToken = ConvertTo-SecureString -String a91a22c7d3c572306e9d6ebfce5f1f697bd7fe8d910d9497ca0f75de2bb37a32 -AsPlainText -Force
    PS C:\> Set-DoPxDefaultAccessToken -AccessToken $accessToken
    PS C:\> Remove-DoPxDomain -Name example.com

    This command removes the domain with name "example.com" from your DigitalOcean environment
.EXAMPLE
    PS C:\> $accessToken = ConvertTo-SecureString -String a91a22c7d3c572306e9d6ebfce5f1f697bd7fe8d910d9497ca0f75de2bb37a32 -AsPlainText -Force
    PS C:\> Set-DoPxDefaultAccessToken -AccessToken $accessToken
    PS C:\> Get-DoPxDomain | Remove-DoPxDomain

    This command removes all domains from your DigitalOcean environment
.LINK
    Add-DoPxDomain
.LINK
    Get-DoPxDomain
.LINK
    Rename-DoPxDomain
#>
function Remove-DoPxDomain {
    [CmdletBinding(SupportsShouldProcess=$true)]
    [OutputType([System.Void])]
    param(
        # The name of the domain.
        [Parameter(Position=0, Mandatory=$true, ValueFromPipelineByPropertyName=$true)]
        [ValidateNotNullOrEmpty()]
        [System.String[]]
        $Name,

        # The access token for your DigitalOcean account, in secure string format.
        [Parameter()]
        [ValidateNotNull()]
        [System.Security.SecureString]
        $AccessToken
    )
    process {
        Invoke-Snippet -Name ProxyFunction.Begin -Parameters @{
            CommandName = 'Remove-DoPxObject'
            CommandType = 'Function'
            PreProcessScriptBlock = {
                #region Set up the ID properly.

                $PSCmdlet.MyInvocation.BoundParameters['Id'] = $Name
                $PSCmdlet.MyInvocation.BoundParameters.Remove('Name') > $null

                #endregion

                #region Add additional required parameters to the BoundParameters hashtable.

                $PSCmdlet.MyInvocation.BoundParameters['RelativeUri'] = 'domains'

                #endregion
            }
        }
        Invoke-Snippet -Name ProxyFunction.Process.NoPipeline
        Invoke-Snippet -Name ProxyFunction.End
    }
}

Export-ModuleMember -Function Remove-DoPxDomain


text:
package me.zmzhang.utils;

import com.google.zxing.WriterException;
import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException;
import com.sun.org.apache.xml.internal.security.utils.Base64;
import org.apache.commons.io.IOUtils;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

/**
 * Created by zmzhang2 on 8/7/17.
 */
public class ZipUtil {

    public static byte[] compress(String data) throws IOException {
        ByteArrayOutputStream bos = new ByteArrayOutputStream(data.length());
        GZIPOutputStream gzip = new GZIPOutputStream(bos);
        gzip.write(data.getBytes());
        gzip.close();
        byte[] compressed = bos.toByteArray();
        bos.close();
        return compressed;
    }

    public static String decompress(final byte[] compressed) throws IOException {
        ByteArrayInputStream bis = new ByteArrayInputStream(compressed);
        GZIPInputStream gis = new GZIPInputStream(bis);
        byte[] bytes = IOUtils.toByteArray(gis);
        return new String(bytes, "UTF-8");
    }

    public static String encode(String data) throws IOException {
        byte[] compressed = compress(data);
        String encoded = Base64.encode(compressed);
        return  encoded;
    }

    public static String decode(String data) throws IOException {
        byte[] compressed = new byte[0];
        try {
            compressed = Base64.decode(data);
        } catch (Base64DecodingException e) {
            e.printStackTrace();
        }
        String decode = decompress(compressed);
        return  decode;
    }

    public static void main(String[] args) throws IOException, WriterException {
        FileUtil fileUtil = new FileUtil();
        File file = new File("large.csv");
        String qrCodeData = FileUtil.getFileContent(new File("large.csv"));
        String encoded = ZipUtil.encode(qrCodeData);
        String decode = ZipUtil.decode(encoded);
        System.out.print(qrCodeData.getBytes().length);
        System.out.print("\n");
        System.out.print(encoded.getBytes().length);
        System.out.print("\n");
        System.out.print(decode.getBytes().length);
    }
}


text:
import math
import random
import numpy
from colorsys import hsv_to_rgb, rgb_to_hsv

from effectlayer import *

class MonoLayer(EffectLayer):

    # nb - 16 degrees is the hue for giants orange. Looks a little red to me.
    def __init__(self, hue=30/float(360)):
        self.hue = hue

    def render(self, model, params, frame):
        len = frame.size/3
        for i in range(0,len):
            pixel = frame[i]
            hsv = numpy.array(rgb_to_hsv(pixel[0], pixel[1], pixel[2]))
            frame[i] = numpy.array(hsv_to_rgb(self.hue, (hsv[1]+0.5) / 1.5, hsv[2]))


text:
package net.ricecode.similarity;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

/**
 * An implementation of StringSimilarityService.
 *
 * @author Ralph Allan Rice <ralph.rice@gmail.com>
 * @see StringSimilarityService
 */
public class StringSimilarityServiceImpl implements StringSimilarityService {

    private SimilarityStrategy strategy;

    /**
     * Creates a similarity calculator instance.
     *
     * @param strategy
     *         The similarity strategy to use when calculating similarity scores.
     */
    public StringSimilarityServiceImpl(SimilarityStrategy strategy) {
        this.strategy = strategy;
    }

    /**
     * Calculates all similarity scores for a given set of features.
     *
     * @param features
     *         The list of features.
     * @param target
     *         The target string to compare against the features.
     *
     * @return A list of similarity scores.
     */
    public List<SimilarityScore> scoreAll(List<String> features, String target) {
        ArrayList<SimilarityScore> scores = new ArrayList<SimilarityScore>();

        for (String feature : features) {
            double score = strategy.score(feature, target);
            scores.add(new SimilarityScore(feature, score));
        }

        return scores;
    }

    /**
     * Calculates the similarity score of a single feature.
     *
     * @param feature
     *         The feature string to compare.
     * @param target
     *         The target string to compare against the feature.
     *
     * @return The similarity score between the feature and target.
     */
    public double score(String feature, String target) {
        return strategy.score(feature, target);
    }

    /**
     * Finds the feature within a set of given features that best match the target string.
     *
     * @param features
     *         A list of strings containing the features to compare.
     * @param target
     *         The target string to compare against the features.
     *
     * @return The similarity score with the highest value.
     */
    public SimilarityScore findTop(List<String> features, String target) {
        return findTop(features, target, new DescendingSimilarityScoreComparator());
    }

    /**
     * Finds the feature within a set of given features that best match the target string.
     *
     * @param features
     *         A list of strings containing the features to compare.
     * @param target
     *         The target string to compare against the features.
     * @param comparator
     *         A comparator that is used sort the scores.
     *
     * @return A SimilarityScore that has the top value amongst the features, according to the comparator.
     */
    public SimilarityScore findTop(List<String> features, String target, Comparator<SimilarityScore> comparator) {
        if (features.size() == 0) {
            return null;
        }
        List<SimilarityScore> scores = scoreAll(features, target);
        Collections.sort(scores, comparator);
        return scores.get(0);
    }
}


text:
"use strict";
var __extends = (this && this.__extends) || function (d, b) {
    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
    function __() { this.constructor = d; }
    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
var Lint = require("../lint");
var Rule = (function (_super) {
    __extends(Rule, _super);
    function Rule() {
        _super.apply(this, arguments);
    }
    Rule.prototype.apply = function (sourceFile) {
        var newParensWalker = new NewParensWalker(sourceFile, this.getOptions());
        return this.applyWithWalker(newParensWalker);
    };
    Rule.metadata = {
        ruleName: "new-parens",
        description: "Requires parentheses when invoking a constructor via the `new` keyword.",
        rationale: "Maintains stylistic consistency with other function calls.",
        optionsDescription: "Not configurable.",
        options: null,
        optionExamples: ["true"],
        type: "style",
    };
    Rule.FAILURE_STRING = "Parentheses are required when invoking a constructor";
    return Rule;
}(Lint.Rules.AbstractRule));
exports.Rule = Rule;
var NewParensWalker = (function (_super) {
    __extends(NewParensWalker, _super);
    function NewParensWalker() {
        _super.apply(this, arguments);
    }
    NewParensWalker.prototype.visitNewExpression = function (node) {
        if (node.arguments === undefined) {
            this.addFailure(this.createFailure(node.getStart(), node.getWidth(), Rule.FAILURE_STRING));
        }
        _super.prototype.visitNewExpression.call(this, node);
    };
    return NewParensWalker;
}(Lint.RuleWalker));


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<title>com.plexobject.dp.domain Class Hierarchy (dp-framework API)</title>
<meta name="date" content="2016-02-13">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">

JavaScript is disabled on your browser.

Overview

Package

Class

Tree

Deprecated

Index

Help

Prev

Next

Frames

No Frames

All Classes

Hierarchy For Package com.plexobject.dp.domain

Package Hierarchies:

All Packages

Class Hierarchy

java.lang.Object

com.plexobject.dp.domain.DataRequest

com.plexobject.dp.domain.DataResponse

com.plexobject.dp.domain.DataRow

com.plexobject.dp.domain.DataRowSet

com.plexobject.dp.domain.InitialValue

com.plexobject.dp.domain.Metadata

com.plexobject.dp.domain.MetaField (implements java.lang.Comparable<T>)

com.plexobject.dp.domain.MetaFieldFactory

com.plexobject.dp.domain.NullObject

com.plexobject.dp.domain.QueryConfiguration

Enum Hierarchy

java.lang.Object

java.lang.Enum<E> (implements java.lang.Comparable<T>, java.io.Serializable)

com.plexobject.dp.domain.MetaFieldType

Overview

Package

Class

Tree

Deprecated

Index

Help

Prev

Next

Frames

No Frames

All Classes


text:
*****
About
*****

Introduction
============

The **Python Record Linkage Toolkit** is a library to link records in or
between data sources. The toolkit provides most of the tools needed for
record linkage and deduplication. The package contains indexing methods,
functions to compare records and classifiers. The package is developed
for research and the linking of small or medium sized files.

The project is inspired by the `Freely Extensible Biomedical Record Linkage
(FEBRL) <https://sourceforge.net/projects/febrl/>`__ project, which is a great
project. In contrast with FEBRL, the recordlinkage project makes extensive use
of data manipulation tools like `pandas <http://pandas.pydata.org/>`__ and
`numpy <http://www.numpy.org/>`__. The use of *pandas*, a flexible and
powerful data analysis and manipulation library for Python, makes the record
linkage process much easier and faster. The extensive *pandas* library can be
used to integrate your record linkage directly into existing data manipulation
projects.

One of the aims of this project is to make an extensible record linkage
framework. It is easy to include your own indexing algorithms,
comparison/similarity measures and classifiers. The main features of the
Python Record Linkage Toolkit are:

-  Clean and standardise data with easy to use tools
-  Make pairs of records with smart indexing methods such as
   **blocking** and **sorted neighbourhood indexing**
-  Compare records with a large number of comparison and similarity measures
   for different types of variables such as strings, numbers and dates.
-  Several classifications algorithms, both supervised and unsupervised
   algorithms.
-  Common record linkage evaluation tools
-  Several built-in datasets.

What is record linkage?
=======================

The term record linkage is used to indicate the procedure of bringing together
information from two or more records that are believed to belong to the same
entity. Record linkage is used to link data from multiple data sources or to
find duplicates in a single data source. In computer science, record linkage
is also known as data matching or deduplication (in case of search duplicate
records within a single file).

In record linkage, the attributes of the entity (stored in a record) are used
to link two or more records. Attributes can be unique entity identifiers (SSN,
license plate number), but also attributes like (sur)name, date of birth and
car model/colour. The record linkage procedure can be represented as a
workflow [Christen, 2012]. The steps are: cleaning, indexing, comparing,
classifying and evaluation. If needed, the classified record pairs flow back
to improve the previous step. The Python Record Linkage Toolkit follows this
workflow.

.. seealso::

    *Christen, Peter. 2012. Data matching: concepts and techniques for record
    linkage, entity resolution, and duplicate detection. Springer Science &
    Business Media.*

    *Fellegi, Ivan P and Alan B Sunter. 1969. "A theory for record linkage."
    Journal of the American Statistical Association 64(328):1183–1210.*

    *Dunn, Halbert L. 1946. "Record linkage." American Journal of Public
    Health and the Nations Health 36(12):1412–1416.*

    *Herzog, Thomas N, Fritz J Scheuren and William E Winkler. 2007. Data
    quality and record linkage techniques. Vol. 1 Springer.*

How to link records?
====================

Import the ``recordlinkage`` module with all important tools for record
linkage and import the data manipulation framework **pandas**.

.. code:: python

    import recordlinkage
    import pandas

Consider that you try to link two datasets with personal information
like name, sex and date of birth. Load these datasets into a pandas
``DataFrame``.

.. code:: python

    df_a = pandas.DataFrame(YOUR_FIRST_DATASET)
    df_b = pandas.DataFrame(YOUR_SECOND_DATASET)

Comparing all record can be computationally intensive. Therefore, we
make smart set of candidate links with one of the built-in indexing
techniques like **blocking**. Only records pairs agreeing on the
surname are included.

.. code:: python

    indexer = recordlinkage.Index()
    indexer.block('surname')
    candidate_links = indexer.index(df_a, df_b)

Each ``candidate_link`` needs to be compared on the comparable attributes.
This can be done easily with the Compare class and the available comparison
and similarity measures.

.. code:: python

    compare = recordlinkage.Compare()

    compare.string('name', 'name', method='jarowinkler', threshold=0.85)
    compare.exact('sex', 'gender')
    compare.exact('dob', 'date_of_birth')
    compare.string('streetname', 'streetname', method='damerau_levenshtein', threshold=0.7)
    compare.exact('place', 'placename')
    compare.exact('haircolor', 'haircolor', missing_value=9)

    # The comparison vectors
    compare_vectors = compare.compute(candidate_links, df_a, df_b)

This record linkage package contains several classification algorithms.
Plenty of the algorithms need trainings data (supervised learning) while
some others are unsupervised. An example of supervised learning:

.. code:: python

    true_linkage = pandas.Series(YOUR_GOLDEN_DATA, index=pandas.MultiIndex(YOUR_MULTI_INDEX))

    logrg = recordlinkage.LogisticRegressionClassifier()
    logrg.fit(compare_vectors[true_linkage.index], true_linkage)

    logrg.predict(compare_vectors)

and an example of unsupervised learning (the well known ECM-algorithm):

.. code:: python

    ecm = recordlinkage.BernoulliEMClassifier()
    ecm.fit_predict(compare_vectors)


text:
package org.bboxdb.network.client.future.client;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import org.bboxdb.network.client.BBoxDBClient;
import org.bboxdb.network.client.future.client.helper.ThreadedJoinedTupleListFutureIterator;
import org.bboxdb.network.client.future.network.NetworkOperationFuture;
import org.bboxdb.storage.entity.MultiTuple;
import org.bboxdb.storage.util.TimeBasedEntityDuplicateTracker;

public class JoinedTupleListFuture extends AbstractListFuture<MultiTuple>{

	public JoinedTupleListFuture(final Supplier<List<NetworkOperationFuture>> futures) {
		super(futures);
	}

	@Override
	protected Iterator<MultiTuple> createThreadedIterator() {
		return new ThreadedJoinedTupleListFutureIterator(this);
	}

	@Override
	protected Iterator<MultiTuple> createSimpleIterator() {
		final List<MultiTuple> allTuples = getListWithAllResults();
		final TimeBasedEntityDuplicateTracker entityDuplicateTracker = new TimeBasedEntityDuplicateTracker();

		final Iterator<MultiTuple> iterator = allTuples.iterator();
		while(iterator.hasNext()) {
			final MultiTuple nextElement = iterator.next();

			if(entityDuplicateTracker.isElementAlreadySeen(nextElement)) {
				iterator.remove();
			}
		}

		runShutdownCallbacks();

		return allTuples.iterator();
	}

	/**
	 * Get all used connections
	 * @return
	 */
	public Map<BBoxDBClient, List<Short>> getAllConnections() {

		final Map<BBoxDBClient, List<Short>> result = new HashMap<>();

		for(final NetworkOperationFuture future : futures) {
			final BBoxDBClient key = future.getConnection().getBboxDBClient();
			result.computeIfAbsent(key, k -> new ArrayList<Short>());
			result.get(key).add(future.getTransmittedPackage().getSequenceNumber());
		}

		return result;
	}

	/**
	 * Add further result tuples to process
	 * @param list
	 */
	public void addFuture(List<NetworkOperationFuture> list) {
		futures.addAll(list);
	}
}


text:
.. _tune-faq:

Ray Tune FAQ
------------

Here we try to answer questions that come up often.
If you still have questions after reading this FAQ, let us know!

.. contents::
    :local:
    :depth: 1

What are Hyperparameters?
~~~~~~~~~~~~~~~~~~~~~~~~~

What are *hyperparameters?* And how are they different from *model parameters*?

In supervised learning, we train a model with labeled data so the model can properly identify new data values.
Everything about the model is defined by a set of parameters, such as the weights in a linear regression. These
are *model parameters*; they are learned during training.

.. image:: /images/hyper-model-parameters.png

In contrast, the *hyperparameters* define structural details about the kind of model itself, like whether or not
we are using a linear regression or classification, what architecture is best for a neural network,
how many layers, what kind of filters, etc. They are defined before training, not learned.

.. image:: /images/hyper-network-params.png

Other quantities considered *hyperparameters* include learning rates, discount rates, etc. If we want our training
process and resulting model to work well, we first need to determine the optimal or near-optimal set of *hyperparameters*.

How do we determine the optimal *hyperparameters*? The most direct approach is to perform a loop where we pick
a candidate set of values from some reasonably inclusive list of possible values, train a model, compare the results
achieved with previous loop iterations, and pick the set that performed best. This process is called
*Hyperparameter Tuning* or *Optimization* (HPO). And *hyperparameters* are specified over a configured and confined
search space, collectively defined for each *hyperparameter* in a ``config`` dictionary.

.. TODO: We *really* need to improve this section.

Which search algorithm/scheduler should I choose?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Ray Tune offers :ref:`many different search algorithms <tune-search-alg>`
and :ref:`schedulers <tune-schedulers>`.
Deciding on which to use mostly depends on your problem:

* Is it a small or large problem (how long does it take to train? How costly
  are the resources, like GPUs)? Can you run many trials in parallel?
* How many hyperparameters would you like to tune?
* What values are valid for hyperparameters?

**If your model returns incremental results** (eg. results per epoch in deep learning,
results per each added tree in GBDTs, etc.) using early stopping usually allows for sampling
more configurations, as unpromising trials are pruned before they run their full course.
Please note that not all search algorithms can use information from pruned trials.
Early stopping cannot be used without incremental results - in case of the functional API,
that means that ``session.report()`` has to be called more than once - usually in a loop.

**If your model is small**, you can usually try to run many different configurations.
A **random search** can be used to generate configurations. You can also grid search
over some values. You should probably still use
:ref:`ASHA for early termination of bad trials <tune-scheduler-hyperband>` (if your problem
supports early stopping).

**If your model is large**, you can try to either use
**Bayesian Optimization-based search algorithms** like :ref:`BayesOpt <bayesopt>` or
:ref:`Dragonfly <Dragonfly>` to get good parameter configurations after few
trials. :ref:`Ax <tune-ax>` is similar but more robust to noisy data.
Please note that these algorithms only work well with **a small number of hyperparameters**.
Alternatively, you can use :ref:`Population Based Training <tune-scheduler-pbt>` which
works well with few trials, e.g. 8 or even 4. However, this will output a hyperparameter *schedule* rather
than one fixed set of hyperparameters.

**If you have a small number of hyperparameters**, Bayesian Optimization methods
work well. Take a look at :ref:`BOHB <tune-scheduler-bohb>` or :ref:`Optuna <tune-optuna>`
with the :ref:`ASHA <tune-scheduler-hyperband>` scheduler to combine the
benefits of Bayesian Optimization with early stopping.

**If you only have continuous values for hyperparameters** this will work well
with most Bayesian Optimization methods. Discrete or categorical variables still
work, but less good with an increasing number of categories.

**If you have many categorical values for hyperparameters**, consider using random search,
or a TPE-based Bayesian Optimization algorithm such as :ref:`Optuna <tune-optuna>` or
:ref:`HyperOpt <tune-hyperopt>`.

**Our go-to solution** is usually to use **random search** with
:ref:`ASHA for early stopping <tune-scheduler-hyperband>` for smaller problems.
Use :ref:`BOHB <tune-scheduler-bohb>` for **larger problems** with a **small number of hyperparameters**
and :ref:`Population Based Training <tune-scheduler-pbt>` for **larger problems** with a
**large number of hyperparameters** if a learning schedule is acceptable.

How do I choose hyperparameter ranges?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A good start is to look at the papers that introduced the algorithms, and also
to see what other people are using.

Most algorithms also have sensible defaults for some of their parameters.
For instance, `XGBoost's parameter overview <https://xgboost.readthedocs.io/en/latest/parameter.html>`_
reports to use ``max_depth=6`` for the maximum decision tree depth. Here, anything
between 2 and 10 might make sense (though that naturally depends on your problem).

For **learning rates**, we suggest using a **loguniform distribution** between
**1e-5** and **1e-1**: ``tune.loguniform(1e-5, 1e-1)``.

For **batch sizes**, we suggest trying **powers of 2**, for instance, 2, 4, 8,
16, 32, 64, 128, 256, etc. The magnitude depends on your problem. For easy
problems with lots of data, use higher batch sizes, for harder problems with
not so much data, use lower batch sizes.

For **layer sizes** we also suggest trying **powers of 2**. For small problems
(e.g. Cartpole), use smaller layer sizes. For larger problems, try larger ones.

For **discount factors** in reinforcement learning we suggest sampling uniformly
between 0.9 and 1.0. Depending on the problem, a much stricter range above 0.97
or oeven above 0.99 can make sense (e.g. for Atari).

How can I use nested/conditional search spaces?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Sometimes you might need to define parameters whose value depend on the value
of other parameters. Ray Tune offers some methods to define these.

Nested spaces
'''''''''''''
You can nest hyperparameter definition in sub dictionaries:

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __basic_config_start__
    :end-before: __basic_config_end__

The trial config will be nested exactly like the input config.

Conditional spaces
''''''''''''''''''
:ref:`Custom and conditional search spaces are explained in detail here <tune_custom-search>`.
In short, you can pass custom functions to ``tune.sample_from()`` that can
return values that depend on other values:

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __conditional_spaces_start__
    :end-before: __conditional_spaces_end__

Conditional grid search
'''''''''''''''''''''''
If you would like to grid search over two parameters that depend on each other,
this might not work out of the box. For instance say that *a* should be a value
between 5 and 10 and *b* should be a value between 0 and a. In this case, we
cannot use ``tune.sample_from`` because it doesn't support grid searching.

The solution here is to create a list of valid *tuples* with the help of a
helper function, like this:

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __iter_start__
    :end-before: __iter_end__

Your trainable then can do something like ``a, b = config["ab"]`` to split
the a and b variables and use them afterwards.

How does early termination (e.g. Hyperband/ASHA) work?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Early termination algorithms look at the intermediately reported values,
e.g. what is reported to them via ``session.report()`` after each training
epoch. After a certain number of steps, they then remove the worst
performing trials and keep only the best performing trials. Goodness of a trial
is determined by ordering them by the objective metric, for instance accuracy
or loss.

In ASHA, you can decide how many trials are early terminated.
``reduction_factor=4`` means that only 25% of all trials are kept each
time they are reduced. With ``grace_period=n`` you can force ASHA to
train each trial at least for ``n`` epochs.

Why are all my trials returning "1" iteration?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

**This is most likely applicable for the Tune function API.**

Ray Tune counts iterations internally every time ``session.report()`` is
called. If you only call ``session.report()`` once at the end of the training,
the counter has only been incremented once. If you're using the class API,
the counter is increased after calling ``step()``.

Note that it might make sense to report metrics more often than once. For
instance, if you train your algorithm for 1000 timesteps, consider reporting
intermediate performance values every 100 steps. That way, schedulers
like Hyperband/ASHA can terminate bad performing trials early.

What are all these extra outputs?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

You'll notice that Ray Tune not only reports hyperparameters (from the
``config``) or metrics (passed to ``session.report()``), but also some other
outputs.

.. code-block:: bash

    Result for easy_objective_c64c9112:
      date: 2020-10-07_13-29-18
      done: false
      experiment_id: 6edc31257b564bf8985afeec1df618ee
      experiment_tag: 7_activation=tanh,height=-53.116,steps=100,width=13.885
      hostname: ubuntu
      iterations: 0
      iterations_since_restore: 1
      mean_loss: 4.688385317424468
      neg_mean_loss: -4.688385317424468
      node_ip: 192.168.1.115
      pid: 5973
      time_since_restore: 7.605552673339844e-05
      time_this_iter_s: 7.605552673339844e-05
      time_total_s: 7.605552673339844e-05
      timestamp: 1602102558
      timesteps_since_restore: 0
      training_iteration: 1
      trial_id: c64c9112

See the :ref:`tune-autofilled-metrics` section for a glossary.

How do I set resources?
~~~~~~~~~~~~~~~~~~~~~~~
If you want to allocate specific resources to a trial, you can use the
``tune.with_resources`` and wrap it around you trainable together with
a dict or a :class:`PlacementGroupFactory <ray.tune.execution.placement_groups.PlacementGroupFactory>` object:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __resources_start__
    :end-before: __resources_end__

The example above showcases three things:

1. The `cpu` and `gpu` options set how many CPUs and GPUs are available for
   each trial, respectively. **Trials cannot request more resources** than these
   (exception: see 3).
2. It is possible to request **fractional GPUs**. A value of 0.5 means that
   half of the memory of the GPU is made available to the trial. You will have
   to make sure yourself that your model still fits on the fractional memory.
3. You can request custom resources you supplied to Ray when starting the cluster.
   Trials will only be scheduled on single nodes that can provide all resources you
   requested.

One important thing to keep in mind is that each Ray worker (and thus each
Ray Tune Trial) will only be scheduled on **one machine**. That means if
you for instance request 2 GPUs for your trial, but your cluster consists
of 4 machines with 1 GPU each, the trial will never be scheduled.

In other words, you will have to make sure that your Ray cluster
has machines that can actually fulfill your resource requests.

In some cases your trainable might want to start other remote actors, for instance if you're
leveraging distributed training via Ray Train. In these cases, you can use
:ref:`placement groups <ray-placement-group-doc-ref>` to request additional resources:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __resources_pgf_start__
    :end-before: __resources_pgf_end__

Here, you're requesting 2 additional CPUs for remote tasks. These two additional
actors do not necessarily have to live on the same node as your main trainable.
In fact, you can control this via the ``strategy`` parameter. In this example, ``PACK``
will try to schedule the actors on the same node, but allows them to be scheduled
on other nodes as well. Please refer to the
:ref:`placement groups documentation <ray-placement-group-doc-ref>` to learn more
about these placement strategies.

You can also use the :ref:`ScalingConfig <train-config>` to achieve the same results:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __resources_scalingconfig_start__
    :end-before: __resources_scalingconfig_end__

You can also allocate specific resources to a trial based on a custom rule via lambda functions.
For instance, if you want to allocate GPU resources to trials based on a setting in your param space:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __resources_lambda_start__
    :end-before: __resources_lambda_end__

Why is my training stuck and Ray reporting that pending actor or tasks cannot be scheduled?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

This is usually caused by Ray actors or tasks being started by the
trainable without the trainable resources accounting for them, leading to a deadlock.
This can also be "stealthly" caused by using other libraries in the trainable that are
based on Ray, such as Modin. In order to fix the issue, request additional resources for
the trial using :ref:`placement groups <ray-placement-group-doc-ref>`, as outlined in
the section above.

For example, if your trainable is using Modin dataframes, operations on those will spawn
Ray tasks. By allocating an additional CPU bundle to the trial, those tasks will be able
to run without being starved of resources.

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __modin_start__
    :end-before: __modin_end__

How can I pass further parameter values to my trainable?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Ray Tune expects your trainable functions to accept only up to two parameters,
``config`` and ``checkpoint_dir``. But sometimes there are cases where
you want to pass constant arguments, like the number of epochs to run,
or a dataset to train on. Ray Tune offers a wrapper function to achieve
just that, called :func:`tune.with_parameters() <ray.tune.with_parameters>`:

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __huge_data_start__
    :end-before: __huge_data_end__

This function works similarly to ``functools.partial``, but it stores
the parameters directly in the Ray object store. This means that you
can pass even huge objects like datasets, and Ray makes sure that these
are efficiently stored and retrieved on your cluster machines.

:func:`tune.with_parameters() <ray.tune.with_parameters>`
also works with class trainables. Please see
:ref:`here for further details <tune-with-parameters>` and examples.

How can I reproduce experiments?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Reproducing experiments and experiment results means that you get the exact same
results when running an experiment again and again. To achieve this, the
conditions have to be exactly the same each time you run the exeriment.
In terms of ML training and tuning, this mostly concerns
the random number generators that are used for sampling in various places of the
training and tuning lifecycle.

Random number generators are used to create randomness, for instance to sample a hyperparameter
value for a parameter you defined. There is no true randomness in computing, rather
there are sophisticated algorithms that generate numbers that *seem* to be random and
fulfill all properties of a random distribution. These algorithms can be *seeded* with
an initial state, after which the generated random numbers are always the same.

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __seeded_1_start__
    :end-before: __seeded_1_end__

The most commonly used random number generators from Python libraries are those in the
native ``random`` submodule and the ``numpy.random`` module.

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __seeded_2_start__
    :end-before: __seeded_2_end__

In your tuning and training run, there are several places where randomness occurs, and
at all these places we will have to introduce seeds to make sure we get the same behavior.

* **Search algorithm**: Search algorithms have to be seeded to generate the same
  hyperparameter configurations in each run. Some search algorithms can be explicitly instantiated with a
  random seed (look for a ``seed`` parameter in the constructor). For others, try to use
  the above code block.
* **Schedulers**: Schedulers like Population Based Training rely on resampling some
  of the parameters, requiring randomness. Use the code block above to set the initial
  seeds.
* **Training function**: In addition to initializing the configurations, the training
  functions themselves have to use seeds. This could concern e.g. the data splitting.
  You should make sure to set the seed at the start of your training function.

PyTorch and TensorFlow use their own RNGs, which have to be initialized, too:

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __torch_tf_seeds_start__
    :end-before: __torch_tf_seeds_end__

You should thus seed both Ray Tune's schedulers and search algorithms, and the
training code. The schedulers and search algorithms should always be seeded with the
same seed. This is also true for the training code, but often it is beneficial that
the seeds differ *between different training runs*.

Here's a blueprint on how to do all this in your training code:

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __torch_seed_example_start__
    :end-before: __torch_seed_example_end__

**Please note** that it is not always possible to control all sources of non-determinism.
For instance, if you use schedulers like ASHA or PBT, some trials might finish earlier
than other trials, affecting the behavior of the schedulers. Which trials finish first
can however depend on the current system load, network communication, or other factors
in the envrionment that we cannot control with random seeds. This is also true for search
algorithms such as Bayesian Optimization, which take previous results into account when
sampling new configurations. This can be tackled by
using the **synchronous modes** of PBT and Hyperband, where the schedulers wait for all trials to
finish an epoch before deciding which trials to promote.

We strongly advise to try reproduction on smaller toy problems first before relying
on it for larger experiments.

.. _tune-bottlenecks:

How can I avoid bottlenecks?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Sometimes you might run into a message like this:

.. code-block::

    The `experiment_checkpoint` operation took 2.43 seconds to complete, which may be a performance bottleneck

Most commonly, the ``experiment_checkpoint`` operation is throwing this warning, but it might be something else,
like ``process_trial_result``.

These operations should usually take less than 500ms to complete. When it consistently takes longer, this might
indicate a problem or inefficiencies. To get rid of this message, it is important to understand where it comes
from.

These are the main reasons this problem comes up:

**The Trial config is very large**

This is the case if you e.g. try to pass a dataset or other large object via the ``config`` parameter.
If this is the case, the dataset is serialized and written to disk repeatedly during experiment
checkpointing, which takes a long time.

**Solution**: Use :func:`tune.with_parameters <ray.tune.with_parameters>` to pass large objects to
function trainables via the objects store. For class trainables you can do this manually via ``ray.put()``
and ``ray.get()``. If you need to pass a class definition, consider passing an
indicator (e.g. a string) instead and let the trainable select the class instead. Generally, your config
dictionary should only contain primitive types, like numbers or strings.

**The Trial result is very large**

This is the case if you return objects, data, or other large objects via the return value of ``step()`` in
your class trainable or to ``session.report()`` in your function trainable. The effect is the same as above:
The results are repeatedly serialized and written to disk, and this can take a long time.

**Solution**: Use checkpoint by writing data to the trainable's current working directory instead. There are various ways
to do that depending on whether you are using class or functional Trainable API.

**You are training a large number of trials on a cluster, or you are saving huge checkpoints**

Checkpoints and logs are synced between nodes
- usually at least to the driver on the head node, but sometimes between worker nodes if needed (e.g. when
using :ref:`Population Based Training <tune-scheduler-pbt>`). If these checkpoints are very large (e.g. for
NLP models), or if you are training a large number of trials, this syncing can take a long time.

If nothing else is specified, syncing happens via SSH, which can lead to network overhead as connections are
not kept open by Ray Tune.

**Solution**: There are multiple solutions, depending on your needs:

1. You can disable syncing to the driver in the :class:`tune.SyncConfig <ray.tune.SyncConfig>`. In this case,
   logs and checkpoints will not be synced to the driver, so if you need to access them later, you will have to
   transfer them where you need them manually.

2. You can use :ref:`cloud checkpointing <tune-cloud-checkpointing>` to save logs and checkpoints to a specified `upload_dir`.
   This is the preferred way to deal with this. All syncing will be taken care of automatically, as all nodes
   are able to access the cloud storage. Additionally, your results will be safe, so even when you're working on
   pre-emptible instances, you won't lose any of your data.

**You are reporting results too often**

Each result is processed by the search algorithm, trial scheduler, and callbacks (including loggers and the
trial syncer). If you're reporting a large number of results per trial (e.g. multiple results per second),
this can take a long time.

**Solution**: The solution here is obvious: Just don't report results that often. In class trainables, ``step()``
should maybe process a larger chunk of data. In function trainables, you can report only every n-th iteration
of the training loop. Try to balance the number of results you really need to make scheduling or searching
decisions. If you need more fine grained metrics for logging or tracking, consider using a separate logging
mechanism for this instead of the Ray Tune-provided progress logging of results.

How can I develop and test Tune locally?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

First, follow the instructions in :ref:`python-develop` to develop Tune without compiling Ray.
After Ray is set up, run ``pip install -r ray/python/ray/tune/requirements-dev.txt`` to install all packages
required for Tune development. Now, to run all Tune tests simply run:

.. code-block:: shell

    pytest ray/python/ray/tune/tests/

If you plan to submit a pull request, we recommend you to run unit tests locally beforehand to speed up the review process.
Even though we have hooks to run unit tests automatically for each pull request, it's usually quicker to run them
on your machine first to avoid any obvious mistakes.

How can I get started contributing to Tune?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

We use Github to track issues, feature requests, and bugs. Take a look at the
ones labeled `"good first issue" <https://github.com/ray-project/ray/issues?utf8=%E2%9C%93&q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22>`__ and `"help wanted" <https://github.com/ray-project/ray/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22>`__ for a place to start.
Look for issues with "[tune]" in the title.

.. note::

    If raising a new issue or PR related to Tune, be sure to include "[tune]" in the title and add a ``tune`` label.

For project organization, Tune maintains a relatively up-to-date organization of
issues on the `Tune Github Project Board <https://github.com/ray-project/ray/projects/4>`__.
Here, you can track and identify how issues are organized.

.. _tune-reproducible:

How can I make my Tune experiments reproducible?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Exact reproducibility of machine learning runs is hard to achieve. This
is even more true in a distributed setting, as more non-determinism is
introduced. For instance, if two trials finish at the same time, the
convergence of the search algorithm might be influenced by which trial
result is processed first. This depends on the searcher - for random search,
this shouldn't make a difference, but for most other searchers it will.

If you try to achieve some amount of reproducibility, there are two
places where you'll have to set random seeds:

1. On the driver program, e.g. for the search algorithm. This will ensure
   that at least the initial configurations suggested by the search
   algorithms are the same.

2. In the trainable (if required). Neural networks are usually initialized
   with random numbers, and many classical ML algorithms, like GBDTs, make use of
   randomness. Thus you'll want to make sure to set a seed here
   so that the initialization is always the same.

Here is an example that will always produce the same result (except for trial
runtimes).

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __reproducible_start__
    :end-before: __reproducible_end__

Some searchers use their own random states to sample new configurations.
These searchers usually accept a ``seed`` parameter that can be passed on
initialization. Other searchers use Numpy's ``np.random`` interface -
these seeds can be then set with ``np.random.seed()``. We don't offer an
interface to do this in the searcher classes as setting a random seed
globally could have side effects. For instance, it could influence the
way your dataset is split. Thus, we leave it up to the user to make
these global configuration changes.

How can I use large datasets in Tune?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

You often will want to compute a large object (e.g., training data, model weights) on the driver and use that
object within each trial.

Tune provides a wrapper function ``tune.with_parameters()`` that allows you to broadcast large objects to your trainable.
Objects passed with this wrapper will be stored on the :ref:`Ray object store <objects-in-ray>` and will
be automatically fetched and passed to your trainable as a parameter.

.. tip:: If the objects are small in size or already exist in the :ref:`Ray Object Store <objects-in-ray>`, there's no need to use ``tune.with_parameters()``. You can use `partials <https://docs.python.org/3/library/functools.html#functools.partial>`__ or pass in directly to ``config`` instead.

.. literalinclude:: doc_code/faq.py
    :language: python
    :start-after: __large_data_start__
    :end-before: __large_data_end__

How can I upload my Tune results to cloud storage?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

If an upload directory is provided, Tune will automatically sync results from the ``local_dir`` to the given directory,
natively supporting standard URIs for systems like S3, gsutil or HDFS. You can add more filesystems by installing
`fs-spec <https://filesystem-spec.readthedocs.io/en/latest/>`_-compatible filesystems e.g. using pip.

Here is an example of uploading to S3, using a bucket called ``my-log-dir``:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __log_1_start__
    :end-before: __log_1_end__

You can customize synchronization behavior by implementing your own Syncer:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __log_2_start__
    :end-before: __log_2_end__

By default, syncing occurs whenever one of the following conditions are met:

* if you have used a :py:class:`~ray.air.config.CheckpointConfig` with ``num_to_keep`` and a trial has checkpointed more than ``num_to_keep`` times since last sync,
* a ``sync_period`` of seconds (default 300) has passed since last sync.

To change the frequency of syncing, set the ``sync_period`` attribute of the sync config to the desired syncing period.

Note that uploading only happens when global experiment state is collected, and the frequency of this is
determined by the sync period. So the true upload period is given by ``max(sync period, TUNE_GLOBAL_CHECKPOINT_S)``.

Make sure that worker nodes have the write access to the cloud storage.
Failing to do so would cause error messages like ``Error message (1): fatal error: Unable to locate credentials``.
For AWS set up, this involves adding an IamInstanceProfile configuration for worker nodes.
Please :ref:`see here for more tips <aws-cluster-s3>`.

How can I use the awscli or gsutil command line commands for syncing?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Some users reported to run into problems with the default pyarrow-based syncing.
In this case, a custom syncer that invokes the respective command line tools
for transferring files between nodes and cloud storage can be implemented.

Here is an example for a syncer that uses string templates that will be run
as a command:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __custom_command_syncer_start__
    :end-before: __custom_command_syncer_end__

For different cloud services, these are example templates you can use with this syncer:

AWS S3
''''''

.. code-block::

    sync_up_template="aws s3 sync {source} {target} --exact-timestamps --only-show-errors"
    sync_down_template="aws s3 sync {source} {target} --exact-timestamps --only-show-errors"
    delete_template="aws s3 rm {target} --recursive --only-show-errors"

Google cloud storage
''''''''''''''''''''

.. code-block::

    sync_up_template="gsutil rsync -r {source} {target}"
    sync_down_template="down": "gsutil rsync -r {source} {target}"
    delete_template="delete": "gsutil rm -r {target}"

HDFS
''''

.. code-block::

    sync_up_template="hdfs dfs -put -f {source} {target}"
    sync_down_template="down": "hdfs dfs -get -f {source} {target}"
    delete_template="delete": "hdfs dfs -rm -r {target}"

.. _tune-docker:

How can I use Tune with Docker?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Tune automatically syncs files and checkpoints between different remote
containers as needed.

.. _tune-kubernetes:

How can I use Tune with Kubernetes?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Ray Tune automatically synchronizes files and checkpoints between different remote nodes as needed.
This usually happens via the Ray object store, but this can be a :ref:`performance bottleneck <tune-bottlenecks>`,
especially when running many trials in parallel.

Instead you should use shared storage for checkpoints so that no additional synchronization across nodes
is necessary. There are two main options.

First, you can use the :ref:`SyncConfig <tune-sync-config>` to store your
logs and checkpoints on cloud storage, such as AWS S3 or Google Cloud Storage:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __s3_start__
    :end-before: __s3_end__

Second, you can set up a shared file system like NFS. If you do this, disable automatic trial syncing:

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __sync_config_start__
    :end-before: __sync_config_end__

Please note that we strongly encourage you to use one of these two options, as they will
result in less overhead and provide naturally durable checkpoint storage.

.. _tune-default-search-space:

How do I configure search spaces?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

You can specify a grid search or sampling distribution via the dict passed into ``Tuner(param_space=...)``.

.. literalinclude:: doc_code/faq.py
    :dedent:
    :language: python
    :start-after: __grid_search_start__
    :end-before: __grid_search_end__

By default, each random variable and grid search point is sampled once.
To take multiple random samples, add ``num_samples: N`` to the experiment config.
If `grid_search` is provided as an argument, the grid will be repeated ``num_samples`` of times.

.. literalinclude:: doc_code/faq.py
    :emphasize-lines: 16
    :language: python
    :start-after: __grid_search_2_start__
    :end-before: __grid_search_2_end__

Note that search spaces may not be interoperable across different search algorithms.
For example, for many search algorithms, you will not be able to use a ``grid_search`` or ``sample_from`` parameters.
Read about this in the :ref:`Search Space API <tune-search-space>` page.

.. _tune-working-dir:

How do I access relative filepaths in my Tune training function?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Let's say you launch a Tune experiment from ``~/code/my_script.py``. By default, Tune
changes the working directory of each worker from ``~/code`` to its corresponding trial
directory (e.g. ``~/ray_results/exp_name/trial_0000x``). This default
guarantees separate working directories for each worker process, avoiding conflicts when
saving trial-specific outputs.

You can configure this by setting `chdir_to_trial_dir=False` in `tune.TuneConfig`.
This explicitly tells Tune to not change the working directory
to the trial directory, giving access to paths relative to the original working directory.
One caveat is that the working directory is now shared between workers, so the
:meth:`session.get_trial_dir() <ray.air.session.get_trial_dir>`
API should be used to get the path for saving trial-specific outputs.

.. literalinclude:: doc_code/faq.py
    :dedent:
    :emphasize-lines: 3, 10, 11, 12, 16
    :language: python
    :start-after: __no_chdir_start__
    :end-before: __no_chdir_end__

.. warning::

    The `TUNE_ORIG_WORKING_DIR` environment variable was the original workaround for
    accessing paths relative to the original working directory. This environment
    variable is deprecated, and the `chdir_to_trial_dir` flag described above should be
    used instead.


text:
namespace Mql2Fdk.Translator.Semantic.Fixes.ReferenceParameterTracking
{
    public class ReferenceParamaterTable
    {
        readonly ParameterKind[] _refBools;

        public ReferenceParamaterTable(int length)
        {
            _refBools = new ParameterKind[length];
        }

        public ParameterKind[] RefBools
        {
            get { return _refBools; }
        }

        public override string ToString()
        {
            return string.Format("({0})", string.Join(", ", RefBools));
        }

        public ParameterKind this[int indexParam]
        {
            set { RefBools[indexParam] = value; }
            get { return RefBools[indexParam]; }
        }
    }
}


text:
namespace {

double factorial(int i) {
  if (i < 0) return -factorial(-i);
  if (i < 1) return 1;
  return i*factorial(i - 1);
}

}

//---

bool
CExprIntegerValue::
getStringValue(std::string &s) const
{
  s = CStrUtil::toString(integer_);

  return true;
}

CExprValuePtr
CExprIntegerValue::
execUnaryOp(CExpr *expr, CExprOpType op) const
{
  switch (op) {
    case CExprOpType::UNARY_PLUS:
      return expr->createIntegerValue(integer_);
    case CExprOpType::UNARY_MINUS:
      return expr->createIntegerValue(-integer_);
    case CExprOpType::BIT_NOT:
      return expr->createIntegerValue(~integer_);
    case CExprOpType::FACTORIAL:
      return expr->createRealValue(factorial(int(integer_)));
    default:
      return CExprValuePtr();
  }
}

CExprValuePtr
CExprIntegerValue::
execBinaryOp(CExpr *expr, CExprValuePtr rhs, CExprOpType op) const
{
  long irhs;

  if (! rhs->getIntegerValue(irhs))
    return CExprValuePtr();

  switch (op) {
    case CExprOpType::POWER: {
      int error_code;

      long integer = integerPower(integer_, irhs, &error_code);

      if (error_code != 0)
        return CExprValuePtr();
      else
        return expr->createIntegerValue(integer);
    }
    case CExprOpType::TIMES:
      return expr->createIntegerValue(integer_ * irhs);
    case CExprOpType::DIVIDE: {
      // divide by zero
      if (irhs == 0) return CExprValuePtr();

      return expr->createIntegerValue(integer_ / irhs);
    }
    case CExprOpType::MODULUS: {
      // divide by zero
      if (irhs == 0) return CExprValuePtr();

      return expr->createIntegerValue(integer_ % irhs);
    }
    case CExprOpType::PLUS:
      return expr->createIntegerValue(integer_ + irhs);
    case CExprOpType::MINUS:
      return expr->createIntegerValue(integer_ - irhs);
    case CExprOpType::BIT_LSHIFT:
      return expr->createIntegerValue(integer_ << irhs);
    case CExprOpType::BIT_RSHIFT:
      return expr->createIntegerValue(integer_ >> irhs);
    case CExprOpType::LESS:
      return expr->createBooleanValue(integer_ < irhs);
    case CExprOpType::LESS_EQUAL:
      return expr->createBooleanValue(integer_ <= irhs);
    case CExprOpType::GREATER:
      return expr->createBooleanValue(integer_ > irhs);
    case CExprOpType::GREATER_EQUAL:
      return expr->createBooleanValue(integer_ >= irhs);
    case CExprOpType::EQUAL:
      return expr->createBooleanValue(integer_ == irhs);
    case CExprOpType::NOT_EQUAL:
      return expr->createBooleanValue(integer_ != irhs);
    case CExprOpType::BIT_AND:
      return expr->createIntegerValue(integer_ & irhs);
    case CExprOpType::BIT_XOR:
      return expr->createIntegerValue(integer_ ^ irhs);
    case CExprOpType::BIT_OR:
      return expr->createIntegerValue(integer_ | irhs);
    default:
      return CExprValuePtr();
  }
}

long
CExprIntegerValue::
integerPower(long integer1, long integer2, int *error_code) const
{
  *error_code = 0;

  if (integer1 == 0 && integer2 < 0) {
    *error_code = int(CExprErrorType::ZERO_TO_NEG_POWER_UNDEF);
    return 0;
  }

  double real;

  errno = 0;

  if (integer2 < 0)
    real = 1.0/pow(double(integer1), double(-integer2));
  else
    real = pow(double(integer1), double(integer2));

  if (errno != 0) {
    *error_code = int(CExprErrorType::POWER_FAILED);
    return 0;
  }

  long integer = realToInteger(real, error_code);

  if (*error_code != 0)
    return 0;

  return integer;
}

long
CExprIntegerValue::
realToInteger(double real, int *error_code) const
{
  long integer = long(real);

  double real1 = double(integer);

  if (fabs(real1 - real) >= 1.0)
    *error_code = int(CExprErrorType::REAL_TOO_BIG_FOR_INTEGER);

  return integer;
}


text:
module Example
  module Endpoints
    class Users < Grape::API
      resource :users do
        desc 'List all users'

        get do
          obj = [
            Example::Mockmodels::User.new('Kristjan', 'Puusepp', 'kristjan@example.com'),
            Example::Mockmodels::User.new('Silver', 'Liivamägi', 'silver@example.com'),
            Example::Mockmodels::User.new('Markus', 'Peterson', 'markus@example.com')
          ]
          present obj, with: Example::Entities::User
        end

      end
    end
  end
end


text:
require_relative "test_helper.rb"
describe Party do
  let(:p) {Party.new("Roberta")}
  it "accepts a birth year" do
    p.birth = Date.new(1990)
    p.birth.must_equal Date.new(1990)
  end
  it "accepts a birth year" do
    p.death = Date.new(1990)
    p.death.must_equal Date.new(1990)
  end
  it "generates proper strings for birth and death" do
    p.birth = Date.new(1980)
    p.death = Date.new(1990)
    p.name_with_birth_death.must_equal "Roberta [1980-1990]"
  end
  it "generates proper strings without birth and death" do
    p.name_with_birth_death.must_equal "Roberta"
  end
  it "generates proper strings with death" do
    p.death = Date.new(1990)
    p.name_with_birth_death.must_equal "Roberta [-1990]"
  end
  it "generates proper strings with birth" do
    p.birth = Date.new(1980)
    p.name_with_birth_death.must_equal "Roberta [1980-]"
  end

end


text:
'use strict';
require('dotenv').load();
if(process.env.NODE_ENV === 'production')
    require('newrelic');
var express = require('express');
var app     = express();
var fs = require('fs');
var API = require('./api.js');
var logger = require('morgan');
var session = require('express-session');
var bodyParser = require('body-parser');
var cookieParser = require('cookie-parser');
var methodOverride = require('method-override');
var verifier = require('./verifier');

var ExpressBrute = require('express-brute');
var MongoStore = require('express-brute-mongo');
var MongoClient = require('mongodb').MongoClient;
var store = new MongoStore(function (ready) {
  MongoClient.connect(process.env.MONGO_URI, function(err, db) {
    if (err) throw err;
    ready(db.collection('bruteforce-store'));
  });
});

var verifyBruteForce = new ExpressBrute(store, {
    freeRetries: process.env.NODE_ENV === 'production' ? 0 : 100,
    minWait: 60 * 1000,
    proxyDepth: process.env.NODE_ENV === 'production' ? 1 : 0
});

var globalBruteForce = new ExpressBrute(store, {
    freeRetries: 5,
    proxyDepth: process.env.NODE_ENV === 'production' ? 1 : 0
});

function authenticate(req, res, next){
    var ionicId = req.query.ionicId;
    var username = req.query.username;
    verifier.verifiedModel.findOne({username: username, ionicId: ionicId}, function(err, user){
        if(err || !user)
            return res.status(401).send('Unauthorized');
        next();
    });
};

//API.runUpdater();
app.use(logger('combined'));
app.use(cookieParser());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
app.use(methodOverride());
app.use(session({
  secret: 'keyboard cat',
  resave: true,
  saveUninitialized: true
}));

app.use(function (req, res, next) {
    res.setHeader('Access-Control-Allow-Origin', '*');
    res.setHeader("Access-Control-Allow-Headers", "Content-Type");
    next();
});

app.get('/api/', function(req, res){
    res.send('Please visit the GitHub page');
});

app.get('/api/scrape/:id', function(req, res){
    var id = req.param('id');//0003193
    var course = API.CourseScraper().init(id);
    course.then(function(data){
        res.json(data);
    }, function(err){

    });
});

// E.g. ?name=Wilson%20M%20Dr&department=Computer Science
app.get('/api/staff', function(req, res){
    API.getStaffByShort(req.query.name, req.query.department, function(data){
        if(!data){
            res.status(404).send({error: 'Not found'});
        }else{
            res.send(data);
        }
    });
});

app.get('/api/room/:room', function(req, res){
    // Return further room info, including staff details, room is the room code
    API.getRoomInfo(req.params.room, function(data){
        if(!data){
            res.status(404).send({error: 'Not found'});
        }else{
            res.send(data);
        }
    });
});

app.get('/api/module/:code', function(req, res){
    API.getModule(req.params.code, function(data){
        if(!data){
            res.status(404).send({error: 'Not found'});
        }else{
            res.send(data);
        }
    });
});

app.get('/api/courses/modules/username/:username', authenticate, function(req, res){
    API.getCourseByUsername(req.params.username, function(data){
        if(!data){
            res.status(404).send({error: 'Not found'});
        }else{
            res.send(data);
        }
    });
});

app.get('/api/courses/modules/((\\d+))', function(req, res){
    API.getCourse(req.params[0], function(data){
        if(!data) {
            return res.status(404).send({error: 'Not found'});
        }
        var courseData = {};
        courseData.name = data.name;
        var course = API.CourseScraper().init(req.params[0]);
        course.then(function(course){
            courseData.department = course.department;
            if(typeof req.query.exclude !== 'undefined'){
                if(req.query.exclude.length > 0){

                    for(var i = 0; i < course.data.length; i++){
                        var newModules = [];
                        var modules = course.data[i].modules;
                        for(var j = 0; j < modules.length; j++){
                            var module = modules[j];
                            if(req.query.exclude.indexOf(module.code) === -1){
                                newModules.push(module);
                            }
                        }
                        course.data[i].modules = newModules;
                    }
                }
            }
            if(req.query.type === 'csv'){
                var startWeek = new Date(2015, 8, 14);
                var csv = 'Subject,Start Date,Start Time,End Date,End Time,Location\n';
                course.data.forEach(function(day, k){
                    day.modules.forEach(function(module){
                        module.weeks.forEach(function(week){
                            var weekDate = new Date(startWeek);
                            weekDate.setDate(weekDate.getDate() + (week * 7) + k);
                            weekDate = weekDate.getDate() + '/' + (weekDate.getMonth()+1) + '/' + weekDate.getFullYear();
                            csv += module.name + ',' + weekDate + ',' + module.time.start + ',' + weekDate + ',' + module.time.end + ',' + module.room + '\n';
                        });
                    });
                });
                res.set({'Content-Disposition':'attachment; filename=\'timetable.csv\''});
                return res.send(csv);
            }
            courseData.days = course.data;
            res.json(courseData);
        }, function(err){
            console.log(err);
        });
    });
});

app.get('/api/courses/((\\w+))', function(req, res){
    API.getCourses(req.params[0], function(data){
        res.send(data);
    });
});

app.get('/api/verify/:username', verifyBruteForce.prevent, function(req, res){
    API.getStudent(req.params.username, function(err, data){
        if(!err && data.results.length === 1){
            verifier.sendVerificationCode(req.params.username, function(err, json){
                if(!err){
                    res.status(200).send({});
                }else{
                    res.status(400).send(err);
                }
            });
        }else{
            res.status(400).send(err);
        }
    });
});

app.post('/api/verify/:username', globalBruteForce.prevent, function(req, res){
    verifier.verifyUsername(req.params.username, req.body.code, req.body.ionicId, function(err){
        if(!err){
            res.status(200).send({});
        }else{
            res.status(400).send(err);
        }
    });
});

app.use(express.static(__dirname + '/dist'));

app.listen(process.env.PORT || 8081);

exports = module.exports = app;


text:
<?php

namespace OpenCloud\Tests\ObjectStore\Resource;

use Guzzle\Http\Message\Response;
use OpenCloud\Common\Constants\Header;
use OpenCloud\ObjectStore\Constants\UrlType;
use OpenCloud\ObjectStore\Exception\ObjectNotEmptyException;
use OpenCloud\Tests\MockSubscriber;
use OpenCloud\Tests\ObjectStore\ObjectStoreTestCase;

class DataObjectTest extends ObjectStoreTestCase
{
    public function test_Pseudo_Dirs()
    {
        $this->addMockSubscriber($this->makeResponse('[{"subdir": "foo"}]'));
        $list = $this->container->objectList();

        foreach ($list as $object) {
            $this->assertTrue($object->isDirectory());
            $this->assertEquals('foo', $object->getName());
            $this->assertEquals($object->getContainer(), $this->container);
            break;
        }
    }

    /**
     * @mockFile Object
     */
    public function test_Contents()
    {
        $object = $this->container->dataObject('foobar');
        $this->assertEquals('text/html', $object->getContentType());
        $this->assertEquals(512000, $object->getContentLength());
        $this->assertNotNull($object->getEtag());

        $this->assertInstanceOf('OpenCloud\ObjectStore\Resource\DataObject', $object->update());
        $this->assertInstanceOf('Guzzle\Http\Message\Response', $object->delete());
    }

    /**
     * @expectedException OpenCloud\Common\Exceptions\NoNameError
     */
    public function test_Url_Fails()
    {
        $object = $this->container->dataObject();
        $object->getUrl();
    }

    public function test_Copy()
    {
        $object = $this->container->dataObject('foobar');
        $this->assertInstanceOf(
            'Guzzle\Http\Message\Response',
            $object->copy('/new_container/new_object')
        );
    }

    /**
     * @expectedException \OpenCloud\Common\Exceptions\NoNameError
     */
    public function test_Copy_Fails()
    {
        $this->container->dataObject()->copy(null);
    }

    /**
     * @expectedException \OpenCloud\Common\Exceptions\InvalidArgumentError
     */
    public function test_Temp_Url_Fails_With_Incorrect_Method()
    {
        $this->container->dataObject('foobar')->getTemporaryUrl(1000, 'DELETE');
    }

    public function test_Temp_Url_Inherits_Url_Type()
    {
        $service = $this->getClient()->objectStoreService(null, 'IAD', 'internalURL');
        $object = $service->getContainer('foo')->dataObject('bar');

        $this->addMockSubscriber(new Response(204, ['X-Account-Meta-Temp-URL-Key' => 'secret']));

        $tempUrl = $object->getTemporaryUrl(60, 'GET');

        // Check that internal URLs are used
        $this->assertContains('snet-storage', $tempUrl);
    }

    public function test_temp_urls_can_be_forced_to_use_public_urls()
    {
        $service = $this->getClient()->objectStoreService(null, 'IAD', 'internalURL');
        $object = $service->getContainer('foo')->dataObject('bar');

        $this->addMockSubscriber(new Response(204, ['X-Account-Meta-Temp-URL-Key' => 'secret']));

        $tempUrl = $object->getTemporaryUrl(60, 'GET', true);

        // Check that internal URLs are NOT used
        $this->assertNotContains('snet-storage', $tempUrl);
    }

    public function test_Purge()
    {
        $object = $this->container->dataObject('foobar');
        $this->setupCdnContainerMockResponse();
        $this->assertInstanceOf(
            'Guzzle\Http\Message\Response',
            $object->purge('test@example.com')
        );
    }

    public function test_Public_Urls()
    {
        $object = $this->container->dataObject('foobar');

        $this->setupCdnContainerMockResponse();
        $this->assertNotNull($object->getPublicUrl());
        $this->assertNotNull($object->getPublicUrl(UrlType::SSL));
        $this->assertNotNull($object->getPublicUrl(UrlType::STREAMING));
        $this->assertNotNull($object->getPublicUrl(UrlType::IOS_STREAMING));
    }

    public function test_Symlink_To()
    {
        $targetName = 'new_container/new_object';
        $this->addMockSubscriber(new Response(200, array(Header::X_OBJECT_MANIFEST => $targetName)));
        $object = $this->container->dataObject('foobar');
        $this->assertInstanceOf('Guzzle\Http\Message\Response', $object->createSymlinkTo($targetName));
        $this->assertEquals($targetName, $object->getManifest());
    }

    /**
     * @expectedException OpenCloud\Common\Exceptions\NoNameError
     */
    public function test_Symlink_To_Fails_With_NoName()
    {
        $object = $this->container->dataObject()->createSymlinkTo(null);
    }

    /**
     * @expectedException OpenCloud\ObjectStore\Exception\ObjectNotEmptyException
     */
    public function test_Symlink_To_Fails_With_NotEmpty()
    {
        $this->addMockSubscriber(new Response(200, array(Header::CONTENT_LENGTH => 100)));
        $object = $this->container->dataObject('foobar')->createSymlinkTo('new_container/new_object');
    }

    public function test_Symlink_From()
    {
        $symlinkName = 'new_container/new_object';

        // We have to fill the mock response queue to properly get the correct X-Object-Manifest header
        // Container\dataObject( )
        //  - Container\refresh( )
        $this->addMockSubscriber(new Response(200));
        // DataObject\createSymlinkFrom( )
        //  - Container\createRefreshRequest( )
        $this->addMockSubscriber(new Response(200));
        //  - CDNContainer\createRefreshRequest( )
        $this->addMockSubscriber(new Response(200));
        //  - Container\objectExists( )
        $this->addMockSubscriber(new Response(200));
        //  - Container\getPartialObject( )
        $this->addMockSubscriber(new Response(200));
        //  - Container\uploadObject( )
        $this->addMockSubscriber(new Response(200, array(Header::X_OBJECT_MANIFEST => $symlinkName)));

        $object = $this->container->dataObject('foobar')->createSymlinkFrom($symlinkName);
        $this->assertInstanceOf('OpenCloud\ObjectStore\Resource\DataObject', $object);
    }

    /**
     * @expectedException OpenCloud\Common\Exceptions\NoNameError
     */
    public function test_Symlink_From_Fails_With_NoName()
    {
        $object = $this->container->dataObject()->createSymlinkFrom(null);
    }

    /**
     * @expectedException OpenCloud\ObjectStore\Exception\ObjectNotEmptyException
     */
    public function test_Symlink_From_Fails_With_NotEmpty()
    {
        // We have to fill the mock response queue to properly get the correct Content-Length header
        // Container\dataObject( )
        //  - Container\refresh( )
        $this->addMockSubscriber(new Response(200));
        // DataObject\createSymlinkFrom( )
        //  - Container\createRefreshRequest( )
        $this->addMockSubscriber(new Response(200));
        //  - Container\objectExists( )
        $this->addMockSubscriber(new Response(200));
        //  - Container\getPartialObject( )
        $this->addMockSubscriber(new Response(200, array(Header::CONTENT_LENGTH => 100)));

        $object = $this->container->dataObject('foobar')->createSymlinkFrom('new_container/new_object');
    }
}


text:
<!DOCTYPE html>
<html>
  <head>
    <title>ReDoc</title>
    <!-- needed for adaptive design -->
    <meta charset="utf-8"/>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <link href="https://fonts.googleapis.com/css?family=Montserrat:300,400,700|Roboto:300,400,700" rel="stylesheet">

    <!--
    ReDoc doesn't change outer page styles
    -->
    <style>
      body {
        margin: 0;
        padding: 0;
      }
    </style>
  </head>
  <body>
    <redoc spec-url='../v3/api-docs'></redoc>
    <script src="https://cdn.jsdelivr.net/npm/redoc@next/bundles/redoc.standalone.js"> </script>
  </body>
</html>


text:
package com.google.security.zynamics.binnavi.Gui.CriteriaDialog.Conditions.NodeColor;

import java.awt.Color;
import java.util.List;

import com.google.security.zynamics.binnavi.Gui.CriteriaDialog.Conditions.ICachedCriterium;
import com.google.security.zynamics.binnavi.Gui.CriteriaDialog.ExpressionModel.CCachedExpressionTreeNode;
import com.google.security.zynamics.binnavi.yfileswrap.zygraph.NaviNode;

/**
 * Class used to replay a color criterium with constant values.
 */
public final class CCachedColorCriterium implements ICachedCriterium {
  /**
   * Color to compare the node colors to.
   */
  private final Color m_color;

  /**
   * Creates a new color criterium.
   *
   * @param color Color to compare the node colors to.
   */
  public CCachedColorCriterium(final Color color) {
    m_color = color;
  }

  @Override
  public String getFormulaString(final List<CCachedExpressionTreeNode> children) {
    return "COLOR" + "=[" + m_color.getRed() + ", " + m_color.getGreen() + ", " + m_color.getBlue()
        + "]";
  }

  @Override
  public boolean matches(final NaviNode node) {
    return CColorEvaluator.evaluate(node, m_color);
  }
}


text:
import { readerLocalActionSearch } from "../actions";
import { ISearchState, searchDefaultState } from "../state/search";

export function searchReducer(
    state: ISearchState = searchDefaultState(),
    action: readerLocalActionSearch.cancel.TAction |
    readerLocalActionSearch.request.TAction |
    readerLocalActionSearch.focus.TAction |
    readerLocalActionSearch.found.TAction |
    readerLocalActionSearch.enable.TAction |
    readerLocalActionSearch.setCache.TAction,
): ISearchState {

    switch (action.type) {
        case readerLocalActionSearch.request.ID:
        case readerLocalActionSearch.cancel.ID:
        case readerLocalActionSearch.enable.ID:
        case readerLocalActionSearch.focus.ID:
        case readerLocalActionSearch.setCache.ID:
        case readerLocalActionSearch.found.ID:

            // let found = state.foundArray;
            // if (action.payload.foundArray) {
            //     found = [
            //         ...state.foundArray,
            //         ...action.payload.foundArray,
            //     ];
            // }

            // let cache = state.cacheArray;
            // if (action.payload.cacheArray) {
            //     cache = [
            //         ...state.cacheArray,
            //         ...action.payload.cacheArray,
            //     ];
            // }

            return {
                ...state,
                ...action.payload,
                // ...{
                //     foundArray: found,
                //     cacheArray: cache,
                // },
            };
        default:
            return state;
    }
}


text:
using ProtobufCompiler.Compiler.Types;
using ProtobufGenerator.Generation;
using System.Collections.Generic;

namespace ProtobufGenerator.Interfaces
{
    public interface IRoslynGenerator
    {
        ICollection<JobResult> Generate(FileDescriptor descriptor);
    }
}