text:
import re, random, numpy as np
from teafacto.query.lbd2tree import LambdaParser

# GENERATOR OF LAMBDA EXPRESSIONS GIVEN A NUMBER OF RELATIONS AND ENTITIES

class LambdaGrammar(object):
    def __init__(self, rules=None, symbols=None):
        self.symbols = symbols if symbols is not None else {}
        self.types = {}

    def __getitem__(self, item):    # item is string, symbol name
        if item not in self.symbols:
            if re.match("[A-Z]+", item):
                ret = NonTerminalSymbol(item)
            else:
                ret = TerminalSymbol(item)
            self.add_symbol(ret)
        return self.symbols[item]

    def add_symbol(self, sym):
        if sym.name not in self.symbols:
            self.symbols[sym.name] = sym
        else:
            raise KeyError("symbol already in grammar")

    def __add__(self, other):
        if isinstance(other, LambdaSymbol):
            self.add_symbol(other)
            return self
        else:
            raise NotImplementedError("not supported: {}".format(type(other)))

    def __repr__(self):
        return "lambda grammar"

    def __str__(self):
        ret = "SYMBOLS================================\n"
        ret += "\n".join([str(sym) for k, sym in self.symbols.items()])
        return ret

    def parse_rules(self, s):
        # rules must be of form HEAD -> BODY | BODY | ... ;
        g = self
        def f(accu):
            m = re.match("^([A-Z]+)\s?->(\s?[^;]+\s?)\s?;$", accu)
            head = m.group(1)
            body = m.group(2)
            headsym = g[head]
            bodysplits = body.split("|")
            for bodysplit in bodysplits:
                bodysplit = bodysplit.lstrip(" ").rstrip(" ")
                seq = bodysplit.split()
                acc = []
                for seqelem in seq:
                    sym = self[seqelem]
                    acc.append(sym)
                self.add_rule(LambdaRule(headsym, acc))
        self._parse_loop(s, f)

    def add_rule(self, rule):
        headsym = self[rule.src.name]
        headsym.add_rules(rule)

    def add_rule_str(self, fro, to):
        r = LambdaRule(self[fro], [self[to]])
        self.add_rule(r)

    def _parse_loop(self, d, f):
        # rules must be of form HEAD -> BODY | BODY | ... ;
        ss = d.split("\n")
        accu = ""
        for split in ss:
            m = re.match("^([^#]*)(#.+)?$", split)
            t = m.group(1)
            t = t.rstrip(" \t").lstrip(" \t")
            accu += t
            if len(t) > 0 and t[-1] == ";":
                f(accu)
                accu = ""

    def parse_info(self, s):
        def f(accu):
            accu = accu.rstrip(" ;\t").lstrip(" \t")
            splits = accu.split("<")
            head = splits[0].rstrip(" \t").lstrip(" \t")
            body = splits[1]
            if body[0] == "=":  # type expression
                body = body[1:]
                headsym = self[head]
                t = self.get_type(body)
                assert(isinstance(headsym, TerminalSymbol))
                headsym.typ = t
            else:               # subtype expression
                t = self.get_type(body)
                h = self.get_type(head)
                h.supertype = t

        self._parse_loop(s, f)

    def get_type(self, s):
        s = s.lstrip(" \t").rstrip(" \t")
        ss = s.split("->")
        if len(ss) == 1:    # unary type
            if s not in self.types:
                self.types[s] = UnaryType(s)
            return self.types[s]
        elif len(ss) == 2:  # binary type
            fro = map(lambda x: self.get_type(x.rstrip(" ").lstrip(" ")), ss[0].lstrip(" ").rstrip(" ").split("|"))
            to = map(lambda x: self.get_type(x.rstrip(" ").lstrip(" ")), ss[1].lstrip(" ").rstrip(" ").split("|"))
            if s not in self.types:
                self.types[s] = BinaryType(fro, to)
            return self.types[s]
        else:
            raise NotImplementedError("higher-order types not supported")

class LambdaSymbol(object):
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return "Lambda Symbol"

    def __str__(self):
        return self.name + "---"

class NonTerminalSymbol(LambdaSymbol):
    def __init__(self, name):
        super(NonTerminalSymbol, self).__init__(name)
        self.rules = set()

    def add_rules(self, *rules):
        for rule in rules:
            if rule not in self.rules:
                self.rules.add(rule)

    def __repr__(self):
        return "nonterm"

    def __str__(self):
        return self.name + ":\n" + "\n".join(["\t{}".format(x) for x in self.rules]) if len(self.rules) > 0 else self.name

class TerminalSymbol(LambdaSymbol):
    def __init__(self, name, typ=None):
        super(TerminalSymbol, self).__init__(name)
        self.typ = typ

    def __repr__(self):
        return self.name

    def __str__(self):
        return "{}[{}]".format(self.name, self.typ) if self.typ is not None else self.name

class VarSymbol(TerminalSymbol):
    def __init__(self, name, typ=None):
        super(VarSymbol, self).__init__(name, typ=typ)

class SymbolType(object):
    pass

class UnaryType(SymbolType):
    def __init__(self, name):
        self.name = name
        self._supertype = None

    @property
    def supertype(self):
        return self._supertype

    @supertype.setter
    def supertype(self, st):
        self._supertype = st

    def __repr__(self):
        return self.name

class BinaryType(SymbolType):
    def __init__(self, fro, to):    # both args are Unary Types or collections
        self.fro = fro
        self.to = to

    def __repr__(self):
        return "|".join(map(str, self.fro)) + " -> " + "|".join(map(str, self.to))

class LambdaRule(object):
    def __init__(self, src, dst, p=1.0, **kw):
        super(LambdaRule, self).__init__(**kw)
        self.src = src
        self.dst = dst
        self.p = p

    def __str__(self):
        return "{} -> {}".format(self.src.name, " ".join(map(lambda x: x.name, self.dst)))

    def __repr__(self):
        return str(self)

class Generator(object):
    def __init__(self, grammar, start="S", limit=12):
        self.start = start
        self.grammar = grammar
        self.limit = limit

    def generate(self):
        cursym = self.grammar[self.start]
        ret = None
        while ret is None or len(ret) < 3:
            ret = self.generate_rec(cursym, set(), depth=self.limit)
        return ret

    def generate_rec(self, cursym, vars, depth=1):
        if depth == 0:
            return None
        if isinstance(cursym, TerminalSymbol):  # terminal, do nothing
            return [cursym]
        elif isinstance(cursym, NonTerminalSymbol):     # expand
            if cursym.name == "CVAR":       # create var
                for i in xrange(0, 99):
                    ret = VarSymbol("${}".format(i))
                    if ret.name not in set(map(lambda x: x.name, list(vars))):
                        break
                return [ret]
            elif cursym.name == "VAR":
                ret = random.choice(list(vars))
                return [ret]
            if cursym.name == "NUM" and False:
                return [TerminalSymbol(str(random.randint(0, 100)) + ":i")]
            rules = cursym.rules
            if len(rules) == 0:
                print "no rules for non terminal: " + str(cursym)
            rules = list(rules)
            probs = np.asarray([rule.p for rule in rules])
            probs = probs / np.sum(probs)
            rule = np.random.choice(list(rules), p=probs)
            dst = rule.dst
            ret = []
            for dste in dst:
                r = self.generate_rec(dste, set().union(vars), depth=depth-1)
                if r is None:
                    return r
                if len(r) == 1 and isinstance(r[0], VarSymbol):
                    vars = {r[0],}.union(vars)
                ret.extend(r)
            return ret
        else:
            raise Exception("???")

def estimate_weights(grammar, parser, path):
    curline = ""
    for line in open(path):
        if len(curline) == 0:
            curline = line[:-1]
        else:
            if line == "\n":
                #print curline
                #d.append(""+curline)
                curline = ""
            elif line[:2] == "//":
                pass
            else:
                line = line[:-1]
                curline = "{}\t{}".format(curline, line)
                lbd = curline.split("\t")[1]
                parsed = parser.parse(lbd)

# deterministic CFG parser for unambiguous lambda language
class DCFGParser(object):
    def __init__(self, grammar):
        self.grammar = grammar

    def parse(self, s):
        stack = []
        for rule in self.grammar.rules:
            pass

class ParseNode(object):
    def __init__(self):
        pass

def get_terminals_from_data(path, parser):
    curline = ""
    ents = set()
    rels = set()
    rellits = set()
    types = set()
    for line in open(path):
        if len(curline) == 0:
            curline = line[:-1]
        else:
            if line == "\n":
                #print curline
                #d.append(""+curline)
                curline = ""
            elif line[:2] == "//":
                pass
            else:
                line = line[:-1]
                curline = "{}\t{}".format(curline, line)
                lbd = curline.split("\t")[1]
                parsed = parser.parse(lbd)
                q = [parsed]
                while len(q) > 0:
                    qt = q.pop()
                    m = re.match("(\w+):(\w+)", qt.name)
                    if m:
                        if len(qt.children) == 1:       # type or relsel
                            done = False
                            if qt.parent is not None and \
                                    qt.parent.name in ("argmax", "argmin", "sum"):
                                pos = 0
                                while pos < 3:
                                    if qt.parent.children[pos] == qt:
                                        break
                                    pos += 1
                                if pos == 2:
                                    rellits.add(qt.name)
                                    done = True
                            if not done and \
                                    re.match("\$\d", qt.children[0].name) and \
                                    (qt.parent is not None
                                     and not re.match("\w+:\w+", qt.parent.name) and not qt.parent.name in ("=", ">", "<")):
                                if qt.name[-1] == "i":
                                    pass
                                types.add(qt.name)
                            elif not done:
                                if m.group(2) == "i":
                                    rellits.add(qt.name)
                                else:
                                    rels.add(qt.name)
                        elif len(qt.children) == 2:     # relation
                            rels.add(qt.name)
                        else:
                            ents.add(qt.name)
                    q.extend(qt.children)
                #print parsed
    return ents, rels, rellits, types

if __name__ == "__main__":
    # non-terminals:  ENT/NUM
    # terminals: specific entities/values, language elements (lambda, ...)
    # types and relation domain/range necessary
    # example generations:
    #   EXPR -> CITY -> (REL[STATE-CITY] STATE) -> (capital:c Ohio:s)
    #   S -> INT -> (REL[STATE-NUM] STATE) -> (population:i Ohio:s)
    #   S -> STATE -> (lambda $0 e EXPR)
    #             -> (lambda $0 e (and EXPR EXPR))
    #             -> (lambda $0 e (and (
    # (lambda $0 e (and (state:t $0) (loc:t dallas_tx:c $0)))
    lbdrules = """
        S -> ENT | SEL ;
        COND -> ( TYPE VAR )
                | ( REL VAR ENT ) | ( REL ENT VAR ) | ( REL VAR VAR )
                | ( and COND COND )
                | ( not COND )
                | ( exists CVAR COND )
                | ( COMP SELLITORNUM SELLITORNUM );
        ENT ->  SENT
                | ( the CVAR COND )
                | ( argmax CVAR COND SELLITVAR )
                | ( argmin CVAR COND SELLITVAR ) ;
        SEL ->  ( REL ENT );
        SELLIT -> SELLITVAR | ( count CVAR COND ) | ( sum CVAR COND SEL ) ;
        SELLITVAR -> ( RELLIT VAR ) ;
        SELLITORNUM -> SELLIT | NUM ;
        VAR -> $ ;
        COMP -> = | > | < ;
        CVAR -> $ ;
        NUM -> 0:i ;
   """
    terminallbdrules = """
        SENT -> ohio:s | washington:s | mississippi_river:r ;
        REL -> capital:c | capital:t | next_to:t | equals:t ;
        RELLIT -> population:i | size:i ;
        TYPE -> state:t | city:t ;
    """
    typeinfo = """
        ohio:s <= state:t ;
        washington:s <= state:t ;
        seattle:c <= city:t ;
        capital:c <= state:t | country:t -> city:t ;
        population:i <= country:t | state:t | city:t -> int ;
        city:t < place:t ;
    """
    abstractent = True
    g = LambdaGrammar()
    g.parse_rules(lbdrules)
    p = "../../data/semparse/geoquery.lbd.dev"
    parser = LambdaParser()
    ents, rels, rellits, types = get_terminals_from_data(p, parser)
    tp = "../../data/semparse/geoquery.lbd.test"
    tents, trels, trellits, ttypes = get_terminals_from_data(tp, parser)
    print map(len, [tents.difference(ents), trels.difference(rels), trellits.difference(rellits), ttypes.difference(types)])
    ents = ents.union(tents)
    rels = rels.union(trels)
    rellits = rellits.union(trellits)
    types = types.union(ttypes)
    for ent in ents:
        g.add_rule_str("SENT", ent)
    for rel in rels:
        g.add_rule_str("REL", rel)
    for rellit in rellits:
        g.add_rule_str("RELLIT", rellit)
    for type in types:
        g.add_rule_str("TYPE", type)
    g.parse_info(typeinfo)
    #print g
    gen = Generator(g)
    #print "\n"
    from teafacto.util import ticktock as TT
    tt = TT()
    tt.tick()
    k = 50000
    outp = "../../data/semparse/geoquery.lbd.abstr.autogen"
    with open(outp, "w") as f:
        parser = LambdaParser()
        for i in range(k):
            x = " ".join(map(lambda x: x.name, gen.generate()))
            y = parser.parse(x)
            if y.name == "the":
                y.name = "lambda"
                y.children = (y.children[0], "e", y.children[1])
            print y.greedy_linearize()
            print y
            f.write("{}\n{}\n\n".format(y.greedy_linearize(), str(y)))
    tt.tock("generated {} samples".format(k))

    #p = "../../data/semparse/geoquery.lbd.dev"
    #parser = LambdaParser()
    #estimate_weights(g, parser, p)


text:
<!DOCTYPE html>

    <meta http-equiv="content-type" content="text/html; charset=utf-8"/>
    <title></title>
    <link rel="Stylesheet" href="../css/analysis.css" />

Izvedi nove lokacije

Ovaj alat izvodi nove geoobjekte u području proučavanja koji zadovoljavaju niz kriterija koji odredite. Ti se kriteriji mogu temeljiti na atributnim upitima (npr. parcele koje su prazne) i prostornim upitima (npr. parcele koje su unutar poplavnih zona).

Ako je označena opcija  Upotrijebi trenutačni obuhvat karte, analizirat će se samo geoobjekti koji su vidljivi unutar trenutačnog obuhvata karte.  Ako opcija nije označena, analizirat će se svi geoobjekti, čak i ako se nalaze izvan trenutačnog obuhvata karte.

Izvedi nove lokacije koje odgovaraju sljedećem(im) izrazu(ima)

Kliknite na gumb  Dodaj izraz za definiranje izraza.  Možete dodati više izraza.  Nakon što unesete izraze i kliknete na gumb Pokreni analizu, geoobjekti u vašem sloju za analizu filtrirat će se kroz sve izraze.  Dijelovi geoobjekata koji zadovoljavaju sve izraze pišu se na sloj rezultata.

Dvije su osnovne vrste izraza koje možete stvoriti; atributi i prostorni.  Izrazi atributa upiti su o vrijednostima polja, primjerice  gdje je naziv „San Francisco”.  Četiri su vrste prostornih izraza:

 unutar udaljenosti od: dijelovi geoobjekta unutar udaljenosti od geoobjekta u drugom sloju.  Na primjer, pronađi dijelove parcela koji su unutar 500 metara od rijeke Los Angeles.

 potpuno sadrži: geoobjekt mora sadržavati čitav geoobjekt u drugom sloju.  Na primjer, pronađi slivove koji potpuno sadrže parcele predviđene za industrijsku uporabu.  Nijedan dio parcele ne smije biti izvan sliva.

 potpuno unutar: geoobjekt mora biti potpuno okružen geoobjektom u drugom sloju.  Na primjer, pronađi školske okruge koji su potpuno unutar granica grada.  Nijedan dio školskog okruga ne smije biti izvan granica grada.

 presjek: geoobjekt ili dijelovi geoobjekta moraju se preklapati ili sjeći s geoobjektom u drugom sloju.  Na primjer, pronađi opožarena područja izgorene vegetacije u nacionalnom parku. Opožarena područja izvan granica nacionalnog parka neće biti uključena.

Svaki prostorni izraz ima odgovarajuću negaciju;  nije unutar udaljenosti od, ne sadrži potpuno, nije potpuno unutar i ne siječe se.

Naziv sloja rezultata

Ovo je naziv sloja koji će se stvoriti u dijelu  Moj sadržaj i dodati karti.  Zadani naziv temelji se na nazivu alata i nazivu sloja unosa. Ako sloj već postoji, od vas će se zatražiti da unesete drugi naziv.

Upotrebom padajućeg izbornika  Spremi rezultat u možete odrediti naziv mape u dijelu Moj sadržaj gdje će se spremiti rezultat.

redpajama_set_name: RedPajamaGithub


text:
package org.deeplearning4j.nn.layers.convolution;

import org.deeplearning4j.datasets.iterator.impl.MnistDataSetIterator;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.exception.DL4JException;
import org.deeplearning4j.nn.api.Layer;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.inputs.InputType;
import org.deeplearning4j.nn.conf.layers.ConvolutionLayer;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.conf.layers.SubsamplingLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.buffer.DataBuffer;
import org.nd4j.linalg.api.buffer.util.DataTypeUtil;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.shape.Shape;
import org.nd4j.linalg.convolution.Convolution;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.INDArrayIndex;
import org.nd4j.linalg.indexing.NDArrayIndex;
import org.nd4j.linalg.lossfunctions.LossFunctions;

import static org.junit.Assert.*;

/**
 * @author Adam Gibson
 */
public class ConvolutionLayerTest {

    @Before
    public void before() {
        DataTypeUtil.setDTypeForContext(DataBuffer.Type.DOUBLE);
        Nd4j.factory().setDType(DataBuffer.Type.DOUBLE);
        Nd4j.EPS_THRESHOLD = 1e-4;
    }

    @Test
    public void testTwdFirstLayer() throws Exception {
        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).iterations(5)
                        .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).l2(2e-4)
                        .regularization(true).momentum(0.9).updater(Updater.NESTEROVS).useDropConnect(true).dropOut(0.5)
                        .list().layer(0,
                                        new ConvolutionLayer.Builder(8, 8) //16 filters kernel size 8 stride 4
                                                        .stride(4, 4).nOut(16).dropOut(0.5)
                                                        .activation(Activation.RELU).weightInit(
                                                                        WeightInit.XAVIER)
                                                        .build())
                        .layer(1, new ConvolutionLayer.Builder(4, 4) //32 filters kernel size 4 stride 2
                                        .stride(2, 2).nOut(32).dropOut(0.5).activation(Activation.RELU)
                                        .weightInit(WeightInit.XAVIER).build())
                        .layer(2, new DenseLayer.Builder() //fully connected with 256 rectified units
                                        .nOut(256).activation(Activation.RELU).weightInit(WeightInit.XAVIER)
                                        .dropOut(0.5).build())
                        .layer(3, new OutputLayer.Builder(LossFunctions.LossFunction.SQUARED_LOSS) //output layer
                                        .nOut(10).weightInit(WeightInit.XAVIER).activation(Activation.SOFTMAX).build())
                        .setInputType(InputType.convolutionalFlat(28, 28, 1)).backprop(true).pretrain(false);

        DataSetIterator iter = new MnistDataSetIterator(10, 10);
        MultiLayerConfiguration conf = builder.build();
        MultiLayerNetwork network = new MultiLayerNetwork(conf);
        network.init();
        network.fit(iter.next());
    }

    @Test
    public void testCNNSubComboWithMixedHW() {
        int imageHeight = 20;
        int imageWidth = 23;
        int nChannels = 1;
        int classes = 2;
        int numSamples = 200;

        int kernelHeight = 3;
        int kernelWidth = 3;

        DataSet trainInput;
        MultiLayerConfiguration.Builder builder =
                        new NeuralNetConfiguration.Builder()
                                        .seed(123).iterations(
                                                        1)
                                        .list()
                                        .layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 1)
                                                        .nOut(2).activation(Activation.RELU)
                                                        .weightInit(WeightInit.XAVIER).build())
                                        .layer(1, new SubsamplingLayer.Builder()
                                                        .poolingType(SubsamplingLayer.PoolingType.MAX)
                                                        .kernelSize(imageHeight - kernelHeight, 1).stride(1, 1).build())
                                        .layer(2, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER)
                                                        .activation(Activation.SOFTMAX).build())
                                        .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, nChannels))
                                        .backprop(true).pretrain(false);

        MultiLayerConfiguration conf = builder.build();
        MultiLayerNetwork model = new MultiLayerNetwork(conf);
        model.init();

        INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels);
        INDArray emptyLables = Nd4j.zeros(numSamples, classes);

        trainInput = new DataSet(emptyFeatures, emptyLables);
        model.fit(trainInput);
    }

    @Test(expected = DL4JException.class)
    public void testCNNTooLargeKernel() {
        int imageHeight = 20;
        int imageWidth = 23;
        int nChannels = 1;
        int classes = 2;
        int numSamples = 200;

        int kernelHeight = imageHeight;
        int kernelWidth = imageWidth + 1;

        DataSet trainInput;
        MultiLayerConfiguration.Builder builder =
                        new NeuralNetConfiguration.Builder()
                                        .seed(123).iterations(
                                                        1)
                                        .list()
                                        .layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth) //(img-kernel+2*padding)/stride + 1: must be >= 1. Therefore: with p=0, kernel <= img size
                                                        .stride(1, 1).nOut(2).activation(Activation.RELU)
                                                        .weightInit(WeightInit.XAVIER).build())
                                        .layer(1, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER)
                                                        .activation(Activation.SOFTMAX).build())
                                        .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, nChannels))
                                        .backprop(true).pretrain(false);

        MultiLayerConfiguration conf = builder.build();
        MultiLayerNetwork model = new MultiLayerNetwork(conf);
        model.init();

        INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels);
        INDArray emptyLables = Nd4j.zeros(numSamples, classes);

        trainInput = new DataSet(emptyFeatures, emptyLables);
        model.fit(trainInput);
    }

    @Test(expected = Exception.class)
    public void testCNNZeroStride() {
        int imageHeight = 20;
        int imageWidth = 23;
        int nChannels = 1;
        int classes = 2;
        int numSamples = 200;

        int kernelHeight = imageHeight;
        int kernelWidth = imageWidth;

        DataSet trainInput;
        MultiLayerConfiguration.Builder builder =
                        new NeuralNetConfiguration.Builder()
                                        .seed(123).iterations(
                                                        1)
                                        .list()
                                        .layer(0, new ConvolutionLayer.Builder(kernelHeight, kernelWidth).stride(1, 0)
                                                        .nOut(2).activation(Activation.RELU)
                                                        .weightInit(WeightInit.XAVIER).build())
                                        .layer(1, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER)
                                                        .activation(Activation.SOFTMAX).build())
                                        .backprop(true).pretrain(false)
                                        .setInputType(InputType.convolutional(imageHeight, imageWidth, nChannels));

        MultiLayerConfiguration conf = builder.build();
        MultiLayerNetwork model = new MultiLayerNetwork(conf);
        model.init();

        INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels);
        INDArray emptyLables = Nd4j.zeros(numSamples, classes);

        trainInput = new DataSet(emptyFeatures, emptyLables);
        model.fit(trainInput);
    }

    @Test(expected = Exception.class)
    @Ignore //Test is based on the assumption that there is no default kernel size. Needs to be revisited.
    public void testCNNKernelNoSize() {
        int imageHeight = 20;
        int imageWidth = 23;
        int nChannels = 1;
        int classes = 2;
        int numSamples = 200;

        int kernelHeight = 3;
        int kernelWidth = 3;

        DataSet trainInput;
        MultiLayerConfiguration.Builder builder = new NeuralNetConfiguration.Builder().seed(123).iterations(1).list()
                        .layer(0, new ConvolutionLayer.Builder().nOut(2).activation(Activation.RELU)
                                        .weightInit(WeightInit.XAVIER).build())
                        .layer(1, new SubsamplingLayer.Builder().poolingType(SubsamplingLayer.PoolingType.MAX).build())
                        .layer(2, new OutputLayer.Builder().nOut(classes).weightInit(WeightInit.XAVIER)
                                        .activation(Activation.SOFTMAX).build())
                        .setInputType(InputType.convolutionalFlat(imageHeight, imageWidth, nChannels)).backprop(true)
                        .pretrain(false);

        MultiLayerConfiguration conf = builder.build();
        MultiLayerNetwork model = new MultiLayerNetwork(conf);
        model.init();

        INDArray emptyFeatures = Nd4j.zeros(numSamples, imageWidth * imageHeight * nChannels);
        INDArray emptyLables = Nd4j.zeros(numSamples, classes);

        trainInput = new DataSet(emptyFeatures, emptyLables);
        model.fit(trainInput);
    }

    @Test
    public void testCNNBiasInit() {
        ConvolutionLayer cnn = new ConvolutionLayer.Builder().nIn(1).nOut(3).biasInit(1).build();

        NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().layer(cnn).build();

        int numParams = conf.getLayer().initializer().numParams(conf);
        INDArray params = Nd4j.create(1, numParams);
        Layer layer = conf.getLayer().instantiate(conf, null, 0, params, true);

        assertEquals(1, layer.getParam("b").size(0));
    }

    @Test
    public void testCNNInputSetupMNIST() throws Exception {
        INDArray input = getMnistData();
        Layer layer = getMNISTConfig();
        layer.activate(input);

        assertEquals(input, layer.input());
        assertArrayEquals(input.shape(), layer.input().shape());
    }

    @Test
    public void testFeatureMapShapeMNIST() throws Exception {
        int inputWidth = 28;
        int[] stride = new int[] {1, 1};
        int[] padding = new int[] {0, 0};
        int[] kernelSize = new int[] {9, 9};
        int nChannelsIn = 1;
        int depth = 20;
        int featureMapWidth = (inputWidth + padding[1] * 2 - kernelSize[1]) / stride[1] + 1;

        INDArray input = getMnistData();

        Layer layer = getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding);
        INDArray convActivations = layer.activate(input);

        assertEquals(featureMapWidth, convActivations.size(2));
        assertEquals(depth, convActivations.size(1));
    }

    @Test
    public void testActivateResultsContained() {
        Layer layer = getContainedConfig();
        INDArray input = getContainedData();
        INDArray expectedOutput = Nd4j.create(new double[] {0.98201379, 0.98201379, 0.98201379, 0.98201379, 0.99966465,
                        0.99966465, 0.99966465, 0.99966465, 0.98201379, 0.98201379, 0.98201379, 0.98201379, 0.99966465,
                        0.99966465, 0.99966465, 0.99966465, 0.98201379, 0.98201379, 0.98201379, 0.98201379, 0.99966465,
                        0.99966465, 0.99966465, 0.99966465, 0.98201379, 0.98201379, 0.98201379, 0.98201379, 0.99966465,
                        0.99966465, 0.99966465, 0.99966465}, new int[] {1, 2, 4, 4});

        INDArray convActivations = layer.activate(input);

        assertArrayEquals(expectedOutput.shape(), convActivations.shape());
        assertEquals(expectedOutput, convActivations);
    }

    //////////////////////////////////////////////////////////////////////////////////

    private static Layer getCNNConfig(int nIn, int nOut, int[] kernelSize, int[] stride, int[] padding) {

        ConvolutionLayer layer = new ConvolutionLayer.Builder(kernelSize, stride, padding).nIn(nIn).nOut(nOut)
                        .activation(Activation.SIGMOID).build();

        NeuralNetConfiguration conf = new NeuralNetConfiguration.Builder().iterations(1).layer(layer).build();

        int numParams = conf.getLayer().initializer().numParams(conf);
        INDArray params = Nd4j.create(1, numParams);
        return conf.getLayer().instantiate(conf, null, 0, params, true);
    }

    public Layer getMNISTConfig() {
        int[] kernelSize = new int[] {9, 9};
        int[] stride = new int[] {1, 1};
        int[] padding = new int[] {1, 1};
        int nChannelsIn = 1;
        int depth = 20;

        return getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding);

    }

    public INDArray getMnistData() throws Exception {
        int inputWidth = 28;
        int inputHeight = 28;
        int nChannelsIn = 1;
        int nExamples = 5;

        DataSetIterator data = new MnistDataSetIterator(nExamples, nExamples);
        DataSet mnist = data.next();
        nExamples = mnist.numExamples();
        return mnist.getFeatureMatrix().reshape(nExamples, nChannelsIn, inputHeight, inputWidth);
    }

    public Layer getContainedConfig() {
        int[] kernelSize = new int[] {2, 2};
        int[] stride = new int[] {2, 2};
        int[] padding = new int[] {0, 0};
        int nChannelsIn = 1;
        int depth = 2;

        INDArray W = Nd4j.create(new double[] {0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5}, new int[] {2, 1, 2, 2});
        INDArray b = Nd4j.create(new double[] {1, 1});
        Layer layer = getCNNConfig(nChannelsIn, depth, kernelSize, stride, padding);
        layer.setParam("W", W);
        layer.setParam("b", b);

        return layer;

    }

    public INDArray getContainedData() {
        INDArray ret = Nd4j.create(new double[] {1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
                        4, 4, 4, 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
                        4, 4, 4, 4, 4, 4, 4, 4}, new int[] {1, 1, 8, 8});
        return ret;
    }

    public INDArray getContainedCol() {
        return Nd4j.create(new double[] {1, 1, 1, 1, 3, 3, 3, 3, 1, 1, 1, 1, 3, 3, 3, 3, 1, 1, 1, 1, 3, 3, 3, 3, 1, 1,
                        1, 1, 3, 3, 3, 3, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2,
                        2, 2, 4, 4, 4, 4}, new int[] {1, 1, 2, 2, 4, 4});
    }

    //////////////////////////////////////////////////////////////////////////////////

    @Test
    public void testCNNMLNPretrain() throws Exception {
        // Note CNN does not do pretrain
        int numSamples = 10;
        int batchSize = 10;
        DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true);

        MultiLayerNetwork model = getCNNMLNConfig(false, true);
        model.fit(mnistIter);

        mnistIter.reset();

        MultiLayerNetwork model2 = getCNNMLNConfig(false, true);
        model2.fit(mnistIter);
        mnistIter.reset();

        DataSet test = mnistIter.next();

        Evaluation eval = new Evaluation();
        INDArray output = model.output(test.getFeatureMatrix());
        eval.eval(test.getLabels(), output);
        double f1Score = eval.f1();

        Evaluation eval2 = new Evaluation();
        INDArray output2 = model2.output(test.getFeatureMatrix());
        eval2.eval(test.getLabels(), output2);
        double f1Score2 = eval2.f1();

        assertEquals(f1Score, f1Score2, 1e-4);

    }

    @Test
    public void testCNNMLNBackprop() throws Exception {
        int numSamples = 10;
        int batchSize = 10;
        DataSetIterator mnistIter = new MnistDataSetIterator(batchSize, numSamples, true);

        MultiLayerNetwork model = getCNNMLNConfig(true, false);
        model.fit(mnistIter);

        MultiLayerNetwork model2 = getCNNMLNConfig(true, false);
        model2.fit(mnistIter);

        DataSet test = mnistIter.next();

        Evaluation eval = new Evaluation();
        INDArray output = model.output(test.getFeatureMatrix());
        eval.eval(test.getLabels(), output);
        double f1Score = eval.f1();

        Evaluation eval2 = new Evaluation();
        INDArray output2 = model2.output(test.getFeatureMatrix());
        eval2.eval(test.getLabels(), output2);
        double f1Score2 = eval2.f1();

        assertEquals(f1Score, f1Score2, 1e-4);

    }

    @Test
    public void testGetSetParams() {

        MultiLayerNetwork net = getCNNMLNConfig(true, false);

        INDArray paramsOrig = net.params().dup();
        net.setParams(paramsOrig);

        INDArray params2 = net.params();

        assertEquals(paramsOrig, params2);
    }

    private static final int kH = 2;
    private static final int kW = 2;
    private static final int[] strides = {1, 1};
    private static final int[] pad = {0, 0};

    private static final int miniBatch = 2;
    private static final int inDepth = 2;
    private static final int height = 3;
    private static final int width = 3;

    private static final int outW = 2;
    private static final int outH = 2;

    private static INDArray getInput() {

        /*
         ----- Input images -----
        example 0:
        depth 0     depth 1
        [ 0  1  2      [ 9 10 11
          3  4  5       12 13 14
          6  7  8]      15 16 17]
        example 1:
        [18 19 20      [27 28 29
         21 22 23       30 31 32
         24 25 26]      33 34 35]
         */

        INDArray input = Nd4j.create(new int[] {miniBatch, inDepth, height, width}, 'c');
        input.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{0, 1, 2}, {3, 4, 5}, {6, 7, 8}}));
        input.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{9, 10, 11}, {12, 13, 14}, {15, 16, 17}}));
        input.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{18, 19, 20}, {21, 22, 23}, {24, 25, 26}}));
        input.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{27, 28, 29}, {30, 31, 32}, {33, 34, 35}}));

        return input;
    }

    @Test
    public void testCnnIm2ColReshaping() {
        //This test: a bit unusual in that it tests the *assumptions* of the CNN implementation rather than the implementation itself
        //Specifically, it tests the row and column orders after reshaping on im2col is reshaped (both forward and backward pass)
        INDArray input = getInput();

        //im2col in the required order: want [outW,outH,miniBatch,depthIn,kH,kW], but need to input [miniBatch,depth,kH,kW,outH,outW]
        // given the current im2col implementation
        //To get this: create an array of the order we want, permute it to the order required by im2col implementation, and then do im2col on that
        //to get old order from required order: permute(2,3,4,5,1,2)
        INDArray col = Nd4j.create(new int[] {miniBatch, outH, outW, inDepth, kH, kW}, 'c');
        INDArray col2 = col.permute(0, 3, 4, 5, 1, 2);
        Convolution.im2col(input, kH, kW, strides[0], strides[1], pad[0], pad[1], false, col2);

        /*
        Expected Output, im2col
        - example 0 -
            depth 0                        depth 1
        h0,w0      h0,w1               h0,w0      h0,w1
        0  1     1  2                 9 10      10 11
        3  4     4  5                12 13      13 14

        h1,w0      h1,w1               h1,w0      h1,w1
        3  4     4  5                12 13      13 14
        6  7     7  8                15 16      16 17

        - example 1 -
            depth 0                        depth 1
        h0,w0      h0,w1               h0,w0      h0,w1
        18 19     19 20               27 28      28 29
        21 22     22 23               30 31      31 32

        h1,w0      h1,w1               h1,w0      h1,w1
        21 22     22 23               30 31      31 32
        24 25     25 26               33 34      34 35
        */

        //Now, after reshaping im2col to 2d, we expect:
        //Rows with order (wOut0,hOut0,mb0), (wOut1,hOut0,mb0), (wOut0,hOut1,mb0), (wOut1,hOut1,mb0), (wOut0,hOut0,mb1), ...
        //Columns with order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), ...

        INDArray reshapedCol = Shape.newShapeNoCopy(col, new int[] {miniBatch * outH * outW, inDepth * kH * kW}, false);

        INDArray exp2d = Nd4j.create(outW * outH * miniBatch, inDepth * kH * kW);
        exp2d.putRow(0, Nd4j.create(new double[] {0, 1, 3, 4, 9, 10, 12, 13})); //wOut0,hOut0,mb0 -> both depths, in order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), (d1,kh0,kw1), (d1,kh1,kw0), (d1,kh1,kw1)
        exp2d.putRow(1, Nd4j.create(new double[] {1, 2, 4, 5, 10, 11, 13, 14})); //wOut1,hOut0,mb0
        exp2d.putRow(2, Nd4j.create(new double[] {3, 4, 6, 7, 12, 13, 15, 16})); //wOut0,hOut1,mb0
        exp2d.putRow(3, Nd4j.create(new double[] {4, 5, 7, 8, 13, 14, 16, 17})); //wOut1,hOut1,mb0
        exp2d.putRow(4, Nd4j.create(new double[] {18, 19, 21, 22, 27, 28, 30, 31})); //wOut0,hOut0,mb1
        exp2d.putRow(5, Nd4j.create(new double[] {19, 20, 22, 23, 28, 29, 31, 32})); //wOut1,hOut0,mb1
        exp2d.putRow(6, Nd4j.create(new double[] {21, 22, 24, 25, 30, 31, 33, 34})); //wOut0,hOut1,mb1
        exp2d.putRow(7, Nd4j.create(new double[] {22, 23, 25, 26, 31, 32, 34, 35})); //wOut1,hOut1,mb1

        assertEquals(exp2d, reshapedCol);

        //Check the same thing for the backprop im2col (different order)
        INDArray colBackprop = Nd4j.create(new int[] {miniBatch, outH, outW, inDepth, kH, kW}, 'c');
        INDArray colBackprop2 = colBackprop.permute(0, 3, 4, 5, 1, 2);

        Convolution.im2col(input, kH, kW, strides[0], strides[1], pad[0], pad[1], false, colBackprop2);

        INDArray reshapedColBackprop = Shape.newShapeNoCopy(colBackprop,
                        new int[] {miniBatch * outH * outW, inDepth * kH * kW}, false);

        //Rows with order (mb0,h0,w0), (mb0,h0,w1), (mb0,h1,w0), (mb0,h1,w1), (mb1,h0,w0), (mb1,h0,w1), (mb1,h1,w0), (mb1,h1,w1)
        //Columns with order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), ...

        INDArray exp2dv2 = Nd4j.create(outW * outH * miniBatch, inDepth * kH * kW);
        exp2dv2.putRow(0, Nd4j.create(new double[] {0, 1, 3, 4, 9, 10, 12, 13})); //wOut0,hOut0,mb0 -> both depths, in order (d0,kh0,kw0), (d0,kh0,kw1), (d0,kh1,kw0), (d0,kh1,kw1), (d1,kh0,kw0), (d1,kh0,kw1), (d1,kh1,kw0), (d1,kh1,kw1)
        exp2dv2.putRow(1, Nd4j.create(new double[] {1, 2, 4, 5, 10, 11, 13, 14})); //wOut1,hOut0,mb0
        exp2dv2.putRow(2, Nd4j.create(new double[] {3, 4, 6, 7, 12, 13, 15, 16})); //wOut0,hOut1,mb0
        exp2dv2.putRow(3, Nd4j.create(new double[] {4, 5, 7, 8, 13, 14, 16, 17})); //wOut1,hOut1,mb0
        exp2dv2.putRow(4, Nd4j.create(new double[] {18, 19, 21, 22, 27, 28, 30, 31})); //wOut0,hOut0,mb1
        exp2dv2.putRow(5, Nd4j.create(new double[] {19, 20, 22, 23, 28, 29, 31, 32})); //wOut1,hOut0,mb1
        exp2dv2.putRow(6, Nd4j.create(new double[] {21, 22, 24, 25, 30, 31, 33, 34})); //wOut0,hOut1,mb1
        exp2dv2.putRow(7, Nd4j.create(new double[] {22, 23, 25, 26, 31, 32, 34, 35})); //wOut1,hOut1,mb1

        assertEquals(exp2dv2, reshapedColBackprop);
    }

    @Test
    public void testDeltaReshaping() {
        //As per above test: testing assumptions of cnn implementation...

        //Delta: initially shape [miniBatch,dOut,outH,outW]
        //permute to [dOut,miniB,outH,outW]
        //then reshape to [dOut,miniB*outH*outW]
        //Expect columns of delta2d to be like: (mb0,h0,w0), (mb0,h0,w1), (mb1,h0,w2), (mb0,h1,w0), ... (mb1,...), ..., (mb2,...)
        int miniBatch = 3;
        int depth = 2;
        int outW = 3;
        int outH = 3;

        /*
         ----- Input delta -----
        example 0:
        depth 0     depth 1
        [ 0  1  2      [ 9 10 11
          3  4  5       12 13 14
          6  7  8]      15 16 17]
        example 1:
        [18 19 20      [27 28 29
         21 22 23       30 31 32
         24 25 26]      33 34 35]
        example 2:
        [36 37 38      [45 46 47
         39 40 41       48 49 50
         42 43 44]      51 52 53]
         */

        INDArray deltaOrig = Nd4j.create(new int[] {miniBatch, depth, outH, outW}, 'c');
        deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{0, 1, 2}, {3, 4, 5}, {6, 7, 8}}));
        deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{9, 10, 11}, {12, 13, 14}, {15, 16, 17}}));
        deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{18, 19, 20}, {21, 22, 23}, {24, 25, 26}}));
        deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{27, 28, 29}, {30, 31, 32}, {33, 34, 35}}));
        deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(2), NDArrayIndex.point(0), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{36, 37, 38}, {39, 40, 41}, {42, 43, 44}}));
        deltaOrig.put(new INDArrayIndex[] {NDArrayIndex.point(2), NDArrayIndex.point(1), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{45, 46, 47}, {48, 49, 50}, {51, 52, 53}}));

        INDArray deltaPermute = deltaOrig.permute(1, 0, 2, 3).dup('c');
        INDArray delta2d = Shape.newShapeNoCopy(deltaPermute, new int[] {depth, miniBatch * outW * outH}, false);

        INDArray exp = Nd4j.create(new double[][] {
                        {0, 1, 2, 3, 4, 5, 6, 7, 8, 18, 19, 20, 21, 22, 23, 24, 25, 26, 36, 37, 38, 39, 40, 41, 42, 43,
                                        44}, //depth0
                        {9, 10, 11, 12, 13, 14, 15, 16, 17, 27, 28, 29, 30, 31, 32, 33, 34, 35, 45, 46, 47, 48, 49, 50,
                                        51, 52, 53} //depth1
        });

        assertEquals(exp, delta2d);
    }

    @Test
    public void testWeightReshaping() {
        //Test assumptions of weight reshaping
        //Weights: originally c order, shape [outDepth, inDepth, kH, kw]
        //permute (3,2,1,0)

        int depthOut = 2;
        int depthIn = 3;
        int kH = 2;
        int kW = 2;

        /*
         ----- Weights -----
         - dOut 0 -
        dIn 0      dIn 1        dIn 2
        [ 0  1      [ 4  5      [ 8  9
          2  3]       6  7]      10 11]
         - dOut 1 -
        [12 13      [16 17      [20 21
         14 15]      18 19]      22 23]
         */

        INDArray weightOrig = Nd4j.create(new int[] {depthOut, depthIn, kH, kW}, 'c');
        weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(0), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{0, 1}, {2, 3}}));
        weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(1), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{4, 5}, {6, 7}}));
        weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(0), NDArrayIndex.point(2), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{8, 9}, {10, 11}}));
        weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(0), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{12, 13}, {14, 15}}));
        weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(1), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{16, 17}, {18, 19}}));
        weightOrig.put(new INDArrayIndex[] {NDArrayIndex.point(1), NDArrayIndex.point(2), NDArrayIndex.all(),
                        NDArrayIndex.all()}, Nd4j.create(new double[][] {{20, 21}, {22, 23}}));

        INDArray weightPermute = weightOrig.permute(3, 2, 1, 0);
        INDArray w2d = Shape.newShapeNoCopy(weightPermute, new int[] {depthIn * kH * kW, depthOut}, true);

        assertNotNull(w2d);

        //Expected order of weight rows, after reshaping: (kw0,kh0,din0), (kw1,kh0,din0), (kw0,kh1,din0), (kw1,kh1,din0), (kw0,kh0,din1), ...
        INDArray wExp = Nd4j.create(new double[][] {{0, 12}, {1, 13}, {2, 14}, {3, 15}, {4, 16}, {5, 17}, {6, 18},
                        {7, 19}, {8, 20}, {9, 21}, {10, 22}, {11, 23}});

        assertEquals(wExp, w2d);
    }

    //////////////////////////////////////////////////////////////////////////////////

    private static MultiLayerNetwork getCNNMLNConfig(boolean backprop, boolean pretrain) {
        Nd4j.ENFORCE_NUMERICAL_STABILITY = true;

        final int numRows = 28;
        final int numColumns = 28;
        int nChannels = 1;
        int outputNum = 10;
        int iterations = 10;
        int seed = 123;

        MultiLayerConfiguration.Builder conf =
                        new NeuralNetConfiguration.Builder().seed(seed).iterations(iterations)
                                        .optimizationAlgo(OptimizationAlgorithm.LINE_GRADIENT_DESCENT).list()
                                        .layer(0, new ConvolutionLayer.Builder(new int[] {10, 10}).nOut(6).build())
                                        .layer(1, new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX,
                                                        new int[] {2, 2}).stride(1, 1).build())
                                        .layer(2, new OutputLayer.Builder(
                                                        LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                                                                        .nOut(outputNum).weightInit(WeightInit.XAVIER)
                                                                        .activation(Activation.SOFTMAX).build())
                                        .setInputType(InputType.convolutionalFlat(28, 28, 1)).backprop(backprop)
                                        .pretrain(pretrain);

        MultiLayerNetwork model = new MultiLayerNetwork(conf.build());
        model.init();

        return model;
    }
}


text:
package com.twitter.finatra

import scala.collection.mutable.Map
import com.twitter.finagle.http.{Request => FinagleRequest, RequestProxy}
import util.Sorting
import com.google.common.base.Splitter
import scala.collection.JavaConversions._

object Request {
  def apply() = new Request(FinagleRequest("/"))
  def apply(path: String) = new Request(FinagleRequest(path))
}

class Request(val request: FinagleRequest) extends RequestProxy {

  var multiParams:  Map[String, MultipartItem]  = Map.empty
  var routeParams:  Map[String, String]         = Map.empty
  var error:        Option[Throwable]           = None

  def accepts: Seq[ContentType] = {
    val accept = this.headers.get("Accept")

    if (accept != null) {
      var acceptParts = Splitter.on(',').split(accept).toArray

      Sorting.quickSort(acceptParts)(AcceptOrdering)

      val seq = acceptParts.map { xs =>
        val part = Splitter.on(";q=").split(xs).toArray.head

        ContentType(part).getOrElse(new ContentType.All)
      }.toSeq

      seq
    } else {
      Seq.empty[ContentType]
    }
  }
}

object AcceptOrdering extends Ordering[String] {

  def getWeight(str: String): Double = {
    val parts = Splitter.on(';').split(str).toArray

    if (parts.length < 2) {
      1.0
    } else {
      try {
        Splitter.on("q=").split(parts(1)).toArray.last.toFloat
      } catch {
        case e:java.lang.NumberFormatException =>
          1.0
      }
    }
  }

  def compare(a: String, b: String): Int = {
    getWeight(b) compare getWeight(a)
  }
}


text:
package org.axonframework.common.property;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;

public abstract class AbstractPropertyAccessStrategyTest<T> {

    @Test
    void getValue() {
        final Property<T> actualProperty = getProperty(regularPropertyName());
        assertNotNull(actualProperty);
        assertNotNull(actualProperty.<String>getValue(propertyHoldingInstance()));
    }

    @Test
    void getValue_BogusProperty() {
        assertNull(getProperty(unknownPropertyName()));
    }

    @Test
    void getValue_ExceptionOnAccess() {
        Property<T> property = getProperty(exceptionPropertyName());

        assertThrows(PropertyAccessException.class, () -> property.getValue(propertyHoldingInstance()));
    }

    @Test
    void voidReturnTypeRejected() {
        Property property = getProperty(voidPropertyName());
        assertNull(property, "void methods should not be accepted as property");
    }

    protected abstract String voidPropertyName();

    protected abstract String exceptionPropertyName();
    protected abstract String regularPropertyName();
    protected abstract String unknownPropertyName();

    protected abstract T propertyHoldingInstance();

    protected abstract Property<T> getProperty(String property);
}


text:
#region Using
    using System.Collections.Generic;
    using PureMVC.CSharp.Demos.Silverlight.EmployeeAdmin.Model.Enum;
using System.ComponentModel;
using System.Collections.ObjectModel;
#endregion

namespace PureMVC.CSharp.Demos.Silverlight.EmployeeAdmin.Model.Vo
{

	public class RoleVo : INotifyPropertyChanged
	{
		private string username = "";
		private ObservableCollection<RoleEnum> roles;

		#region Constructors
        public RoleVo ( string username=null, ObservableCollection<RoleEnum> roles=null )
		{
			if( username != null ) Username = username;
			if( roles != null ) Roles = roles;
		}
		#endregion

        public string Username
        {
            get { return username; }
            set { username = value; OnPropertyChanged("Username"); }
        }

        public ObservableCollection<RoleEnum> Roles
        {
            get { return roles; }
            set { roles = value; OnPropertyChanged("Roles"); }
        }

        #region INotifyPropertyChanged Members

        public event PropertyChangedEventHandler PropertyChanged;
        protected void OnPropertyChanged(string name)
        {
            if (PropertyChanged != null)
            {
                PropertyChanged(this, new PropertyChangedEventArgs(name));
            }
        }

        #endregion
	}
}


text:
#include <stdlib.h>
#include <string.h>

#define CFC_NEED_BASE_STRUCT_DEF
#include "CFCBase.h"
#include "CFCUri.h"
#include "CFCClass.h"
#include "CFCDocument.h"
#include "CFCFunction.h"
#include "CFCMethod.h"
#include "CFCParcel.h"
#include "CFCUtil.h"

struct CFCUri {
    CFCBase      base;
    char        *string;
    CFCClass    *base_class;
    CFCUriType   type;
    CFCClass    *klass;
    CFCDocument *document;
    char        *callable;
    char        *error;
};

static const CFCMeta CFCURI_META = {
    "Clownfish::CFC::Uri",
    sizeof(CFCUri),
    (CFCBase_destroy_t)CFCUri_destroy
};

static void
S_parse(CFCUri *self);

static void
S_resolve(CFCUri *self, const char *prefix, const char *struct_sym,
          const char *callable);

static CFCClass*
S_find_class_full(CFCUri *self, const char *parcel, const char *struct_sym);

static CFCClass*
S_find_class_short(CFCUri *self, const char *struct_sym);

static void
S_set_error(CFCUri *self, const char *error);

static char*
S_next_component(char **iter);

int
CFCUri_is_clownfish_uri(const char *uri) {
    return strncmp(uri, "cfish:", 6) == 0 || !strchr(uri, ':');
}

CFCUri*
CFCUri_new(const char *uri, CFCClass *base_class) {
    CFCUri *self = (CFCUri*)CFCBase_allocate(&CFCURI_META);
    return CFCUri_init(self, uri, base_class);
}

CFCUri*
CFCUri_init(CFCUri *self, const char *uri, CFCClass *base_class) {
    CFCUTIL_NULL_CHECK(uri);

    self->string     = CFCUtil_strdup(uri);
    self->base_class = (CFCClass*)CFCBase_incref((CFCBase*)base_class);

    return self;
}

void
CFCUri_destroy(CFCUri *self) {
    FREEMEM(self->string);
    FREEMEM(self->callable);
    FREEMEM(self->error);
    CFCBase_decref((CFCBase*)self->base_class);
    CFCBase_decref((CFCBase*)self->klass);
    CFCBase_decref((CFCBase*)self->document);
    CFCBase_destroy((CFCBase*)self);
}

static void
S_parse(CFCUri *self) {
    const char *uri_part = self->string;

    if (strncmp(uri_part, "cfish:", 6) == 0) {
        uri_part += 6;
    }

    if (strcmp(uri_part, "@null") == 0) {
        self->type = CFC_URI_NULL;
        return;
    }

    const char *parcel     = NULL;
    const char *struct_sym = NULL;
    const char *callable   = NULL;

    char       *buf  = CFCUtil_strdup(uri_part);
    char       *iter = buf;
    const char *component = S_next_component(&iter);

    if (CFCUtil_islower(component[0])) {
        // Parcel
        parcel = component;
        component = S_next_component(&iter);

        if (!component) {
            S_set_error(self, "Missing component in Clownfish URI");
            goto done;
        }
    }

    // struct_sym == NULL for "cfish:.Method" style URL.
    // parcel implies struct_sym.
    if (parcel || component[0] != '\0') {
        struct_sym = component;
    }

    component = S_next_component(&iter);

    if (component) {
        callable = component;
        component = S_next_component(&iter);
    }

    if (component) {
        S_set_error(self, "Trailing component in Clownfish URI");
        goto done;
    }

    S_resolve(self, parcel, struct_sym, callable);

done:
    FREEMEM(buf);
}

static void
S_resolve(CFCUri *self, const char *parcel, const char *struct_sym,
          const char *callable) {
    // Try to find a CFCClass.
    CFCClass *klass     = NULL;

    if (parcel) {
        klass = S_find_class_full(self, parcel, struct_sym);
    }
    else if (struct_sym) {
        klass = S_find_class_short(self, struct_sym);
    }
    else if (callable) {
        klass = self->base_class;
    }

    if (klass) {
        if (!CFCClass_public(klass)) {
            CFCUtil_warn("Non-public class '%s' in Clownfish URI: %s",
                          CFCClass_get_struct_sym(klass), self->string);
        }

        self->type  = CFC_URI_CLASS;
        self->klass = klass;
        CFCBase_incref((CFCBase*)klass);

        if (callable) {
            if (CFCUtil_islower(callable[0])) {
                CFCFunction *function = CFCClass_function(klass, callable);

                if (!function) {
                    CFCUtil_warn("Unknown function '%s' in Clownfish URI: %s",
                                 callable, self->string);
                }
                else if (!CFCFunction_public(function)) {
                    CFCUtil_warn("Non-public function '%s' in Clownfish URI:"
                                 " %s", callable, self->string);
                }

                self->type     = CFC_URI_FUNCTION;
                self->callable = CFCUtil_strdup(callable);
            }
            else {
                CFCMethod *method = CFCClass_method(klass, callable);

                if (!method) {
                    CFCUtil_warn("Unknown method '%s' in Clownfish URI: %s",
                                 callable, self->string);
                }
                else if (!CFCMethod_public(method)) {
                    CFCUtil_warn("Non-public method '%s' in Clownfish URI:"
                                 " %s", callable, self->string);
                }

                self->type     = CFC_URI_METHOD;
                self->callable = CFCUtil_strdup(callable);
            }
        }

        return;
    }

    // Try to find a CFCDocument.
    if (!parcel && struct_sym && !callable) {
        CFCDocument *doc = CFCDocument_fetch(struct_sym);

        if (doc) {
            self->type     = CFC_URI_DOCUMENT;
            self->document = doc;
            CFCBase_incref((CFCBase*)doc);
            return;
        }
    }

    S_set_error(self, "Couldn't resolve Clownfish URI");
}

static CFCClass*
S_find_class_full(CFCUri *self, const char *parcel, const char *struct_sym) {
    char *full_struct_sym = CFCUtil_sprintf("%s_%s", parcel, struct_sym);
    CFCClass *klass = NULL;

    if (self->base_class) {
        // Only look in parcel of base class.
        CFCParcel *parcel = CFCClass_get_parcel(self->base_class);
        klass = CFCParcel_class_by_full_sym(parcel, full_struct_sym);
    }
    else {
        // Search all source parcels For standalone .md docs.
        CFCParcel **parcels = CFCParcel_all_parcels();
        for (size_t i = 0; parcels[i]; i++) {
            CFCParcel *parcel = parcels[i];
            if (CFCParcel_included(parcel)) { continue; }
            klass = CFCParcel_class_by_full_sym(parcel, full_struct_sym);
            if (klass) { break; }
        }
    }

    FREEMEM(full_struct_sym);
    return klass;
}

static CFCClass*
S_find_class_short(CFCUri *self, const char *struct_sym) {
    CFCClass *klass = NULL;

    if (self->base_class) {
        // Only look in parcel of base class.
        CFCParcel *parcel = CFCClass_get_parcel(self->base_class);
        klass = CFCParcel_class_by_short_sym(parcel, struct_sym);
    }
    else {
        // Search all source parcels For standalone .md docs.
        CFCParcel **parcels = CFCParcel_all_parcels();
        for (size_t i = 0; parcels[i]; i++) {
            CFCParcel *parcel = parcels[i];
            if (CFCParcel_included(parcel)) { continue; }
            CFCClass *candidate
                = CFCParcel_class_by_short_sym(parcel, struct_sym);
            if (candidate && candidate != klass) {
                if (klass) {
                    // Found in multiple parcels.
                    CFCUtil_warn("Ambiguous class name '%s' in URI. Do you"
                                 " mean '%s' or '%s'?", struct_sym,
                                 CFCClass_full_struct_sym(klass),
                                 CFCClass_full_struct_sym(candidate));
                    return NULL;
                }
                klass = candidate;
            }
        }
    }

    return klass;
}

static void
S_set_error(CFCUri *self, const char *error) {
    self->type  = CFC_URI_ERROR;
    self->error = CFCUtil_sprintf("%s: %s", error, self->string);

    CFCUtil_warn(self->error);
}

static char*
S_next_component(char **iter) {
    char *component = *iter;

    if (!component) { return NULL; }

    for (char *ptr = component; *ptr != '\0'; ptr++) {
        if (*ptr == '.') {
            *ptr  = '\0';
            *iter = ptr + 1;
            return component;
        }
    }

    *iter = NULL;
    return component;
}

const char*
CFCUri_get_string(CFCUri *self) {
    return self->string;
}

CFCUriType
CFCUri_get_type(CFCUri *self) {
    if (self->type == 0) { S_parse(self); }
    return self->type;
}

CFCClass*
CFCUri_get_class(CFCUri *self) {
    if (self->type == 0) { S_parse(self); }
    if (self->klass == NULL) {
        CFCUtil_die("Not a class URI");
    }
    return self->klass;
}

CFCDocument*
CFCUri_get_document(CFCUri *self) {
    if (self->type == 0) { S_parse(self); }
    if (self->document == NULL) {
        CFCUtil_die("Not a document URI");
    }
    return self->document;
}

const char*
CFCUri_get_callable_name(CFCUri *self) {
    if (self->type == 0) { S_parse(self); }
    if (self->callable == NULL) {
        CFCUtil_die("Not a callable URI");
    }
    return self->callable;
}

const char*
CFCUri_get_error(CFCUri *self) {
    if (self->type == 0) { S_parse(self); }
    if (self->error == NULL) {
        CFCUtil_die("Not an error URI");
    }
    return self->error;
}


text:
package pilot

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"testing"

	"github.com/google/uuid"
	"github.com/stretchr/testify/assert"

	v2 "istio.io/istio/pilot/pkg/proxy/envoy/v2"
	"istio.io/istio/tests/util"
)

var (
	preDefinedNonce = newNonce()
)

func newNonce() string {
	return uuid.New().String()
}

func TestStatusWriter_PrintAll(t *testing.T) {
	tests := []struct {
		name    string
		input   map[string][]v2.SyncStatus
		want    string
		wantErr bool
	}{
		{
			name: "prints multiple pilot inputs to buffer in alphabetical order by pod name",
			input: map[string][]v2.SyncStatus{
				"pilot1": statusInput1(),
				"pilot2": statusInput2(),
				"pilot3": statusInput3(),
			},
			want: "testdata/multiStatusMultiPilot.txt",
		},
		{
			name: "prints single pilot input to buffer in alphabetical order by pod name",
			input: map[string][]v2.SyncStatus{
				"pilot1": append(statusInput1(), statusInput2()...),
			},
			want: "testdata/multiStatusSinglePilot.txt",
		},
		{
			name: "error if given non-syncstatus info",
			input: map[string][]v2.SyncStatus{
				"pilot1": {},
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := &bytes.Buffer{}
			sw := StatusWriter{Writer: got}
			input := map[string][]byte{}
			for key, ss := range tt.input {
				b, _ := json.Marshal(ss)
				input[key] = b
			}
			if len(tt.input["pilot1"]) == 0 {
				input = map[string][]byte{
					"pilot1": []byte(`gobbledygook`),
				}
			}
			err := sw.PrintAll(input)
			if tt.wantErr {
				assert.Error(t, err)
			} else {
				assert.NoError(t, err)
			}
			want, _ := ioutil.ReadFile(tt.want)
			if err := util.Compare(got.Bytes(), want); err != nil {
				t.Errorf(err.Error())
			}
		})
	}
}

func TestStatusWriter_PrintSingle(t *testing.T) {
	tests := []struct {
		name      string
		input     map[string][]v2.SyncStatus
		filterPod string
		want      string
		wantErr   bool
	}{
		{
			name: "prints multiple pilot inputs to buffer filtering for pod",
			input: map[string][]v2.SyncStatus{
				"pilot1": statusInput1(),
				"pilot2": statusInput2(),
			},
			filterPod: "proxy2",
			want:      "testdata/singleStatus.txt",
		},
		{
			name: "single pilot input to buffer filtering for pod",
			input: map[string][]v2.SyncStatus{
				"pilot2": append(statusInput1(), statusInput2()...),
			},
			filterPod: "proxy2",
			want:      "testdata/singleStatus.txt",
		},
		{
			name: "fallback to proxy version",
			input: map[string][]v2.SyncStatus{
				"pilot2": statusInputProxyVersion(),
			},
			filterPod: "proxy2",
			want:      "testdata/singleStatusFallback.txt",
		},
		{
			name: "error if given non-syncstatus info",
			input: map[string][]v2.SyncStatus{
				"pilot1": {},
			},
			wantErr: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := &bytes.Buffer{}
			sw := StatusWriter{Writer: got}
			input := map[string][]byte{}
			for key, ss := range tt.input {
				b, _ := json.Marshal(ss)
				input[key] = b
			}
			if len(tt.input["pilot1"]) == 0 && len(tt.input["pilot2"]) == 0 {
				input = map[string][]byte{
					"pilot1": []byte(`gobbledygook`),
				}
			}
			err := sw.PrintSingle(input, tt.filterPod)
			if tt.wantErr {
				assert.Error(t, err)
			} else {
				assert.NoError(t, err)
			}
			want, _ := ioutil.ReadFile(tt.want)
			if err := util.Compare(got.Bytes(), want); err != nil {
				t.Errorf(err.Error())
			}
		})
	}
}

func statusInput1() []v2.SyncStatus {
	return []v2.SyncStatus{
		{
			ProxyID:       "proxy1",
			IstioVersion:  "1.1",
			ClusterSent:   preDefinedNonce,
			ClusterAcked:  newNonce(),
			ListenerSent:  preDefinedNonce,
			ListenerAcked: preDefinedNonce,
			EndpointSent:  preDefinedNonce,
			EndpointAcked: preDefinedNonce,
		},
	}
}

func statusInput2() []v2.SyncStatus {
	return []v2.SyncStatus{
		{
			ProxyID:       "proxy2",
			IstioVersion:  "1.1",
			ClusterSent:   preDefinedNonce,
			ClusterAcked:  newNonce(),
			ListenerSent:  preDefinedNonce,
			ListenerAcked: preDefinedNonce,
			EndpointSent:  preDefinedNonce,
			EndpointAcked: newNonce(),
			RouteSent:     preDefinedNonce,
			RouteAcked:    preDefinedNonce,
		},
	}
}

func statusInput3() []v2.SyncStatus {
	return []v2.SyncStatus{
		{
			ProxyID:       "proxy3",
			IstioVersion:  "1.1",
			ClusterSent:   preDefinedNonce,
			ClusterAcked:  "",
			ListenerAcked: preDefinedNonce,
			EndpointSent:  preDefinedNonce,
			EndpointAcked: "",
			RouteSent:     preDefinedNonce,
			RouteAcked:    preDefinedNonce,
		},
	}
}

func statusInputProxyVersion() []v2.SyncStatus {
	return []v2.SyncStatus{
		{
			ProxyID:       "proxy2",
			ProxyVersion:  "1.1",
			ClusterSent:   preDefinedNonce,
			ClusterAcked:  newNonce(),
			ListenerSent:  preDefinedNonce,
			ListenerAcked: preDefinedNonce,
			EndpointSent:  preDefinedNonce,
			EndpointAcked: newNonce(),
			RouteSent:     preDefinedNonce,
			RouteAcked:    preDefinedNonce,
		},
	}
}


text:
<Alloy>
	<Collection src="MyCollection" />
	<Window title="Explore California">
		<TableView id="table" onClick="showContent" dataCollection="MyCollection">
			<TableViewRow nid="{nid}">
				<ImageView id="thumb" image="{thumbnail}"></ImageView>>
				<Label id="title" text="{node_title}"/>
				<Label id="summary" text="{shortDescription}"/>
			</TableViewRow>
		</TableView>
	</Window>
</Alloy>


text:
namespace HiRezApi.Paladins
{
    using Microsoft.Rest;
    using HiRezApi.Common.Models;
    using HiRezApi.Paladins.Models;
    using Newtonsoft.Json;
    using System.Collections;
    using System.Collections.Generic;
    using System.Threading;
    using System.Threading.Tasks;

    /// <summary>
    /// Paladins Api Client
    /// </summary>
    public partial interface IPaladinsApiClient : System.IDisposable
    {
        /// <summary>
        /// The base URI of the service.
        /// </summary>
        System.Uri BaseUri { get; set; }

        /// <summary>
        /// Gets or sets json serialization settings.
        /// </summary>
        JsonSerializerSettings SerializationSettings { get; }

        /// <summary>
        /// Gets or sets json deserialization settings.
        /// </summary>
        JsonSerializerSettings DeserializationSettings { get; }

        /// <summary>
        /// Subscription credentials which uniquely identify client
        /// subscription.
        /// </summary>
        ServiceClientCredentials Credentials { get; }

        /// <summary>
        /// createsession
        /// </summary>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<Session>> CreateSessionWithHttpMessagesAsync(Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getdataused
        /// </summary>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<DataUsed>>> GetDataUsedWithHttpMessagesAsync(Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getdemodetails
        /// </summary>
        /// <param name='matchId'>
        /// The matchId
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<DemoDetails>>> GetDemoDetailsWithHttpMessagesAsync(string matchId, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getfriends
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<Friends>>> GetFriendsWithHttpMessagesAsync(string player, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getchampionranks
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<ChampionRanks>>> GetChampionRanksWithHttpMessagesAsync(string player, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getchampions
        /// </summary>
        /// <param name='languageCode'>
        /// The languageCode
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<Champions>>> GetChampionsWithHttpMessagesAsync(string languageCode, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getchampionskins
        /// </summary>
        /// <param name='championId'>
        /// The championId
        /// </param>
        /// <param name='languageCode'>
        /// The languageCode
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<ChampionSkins>>> GetChampionSkinsWithHttpMessagesAsync(string championId, string languageCode, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getitems
        /// </summary>
        /// <param name='languageCode'>
        /// The languageCode
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<Items>>> GetItemsWithHttpMessagesAsync(string languageCode, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getmatchdetails
        /// </summary>
        /// <param name='matchId'>
        /// The matchId
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<MatchDetails>>> GetMatchDetailsWithHttpMessagesAsync(string matchId, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getmatchdetailsbatch
        /// </summary>
        /// <param name='matchIds'>
        /// The matchIds
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<MatchDetails>>> GetMatchDetailsBatchWithHttpMessagesAsync(string matchIds, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getmatchplayerdetails
        /// </summary>
        /// <param name='matchId'>
        /// The matchId
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<MatchPlayerDetails>>> GetMatchPlayerDetailsWithHttpMessagesAsync(string matchId, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getmatchidsbyqueue
        /// </summary>
        /// <param name='queue'>
        /// The queue
        /// </param>
        /// <param name='date'>
        /// The date
        /// </param>
        /// <param name='hour'>
        /// The hour
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<MatchIdsByQueue>>> GetMatchIdsByQueueWithHttpMessagesAsync(string queue, string date, string hour, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getleagueleaderboard
        /// </summary>
        /// <param name='queue'>
        /// The queue
        /// </param>
        /// <param name='tier'>
        /// The tier
        /// </param>
        /// <param name='season'>
        /// The season
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<LeagueLeaderBoard>>> GetLeagueLeaderBoardWithHttpMessagesAsync(string queue, string tier, string season, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getmatchhistory
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<MatchHistory>>> GetMatchHistoryWithHttpMessagesAsync(string player, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getplayer
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<Player>>> GetPlayerWithHttpMessagesAsync(string player, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getplayerloadouts
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='languageCode'>
        /// The languageCode
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<PlayerLoadouts>>> GetPlayerLoadoutsWithHttpMessagesAsync(string player, string languageCode, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getplayerstatus
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<PlayerStatus>>> GetPlayerStatusWithHttpMessagesAsync(string player, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getqueuestats
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='queue'>
        /// The queue
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<QueueStats>>> GetQueueStatsWithHttpMessagesAsync(string player, string queue, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getplayerachievements
        /// </summary>
        /// <param name='player'>
        /// The player
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<PlayerAchievements>> GetPlayerAchievementsWithHttpMessagesAsync(string player, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// getpatchinfo
        /// </summary>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<PatchInfo>> GetPatchInfoWithHttpMessagesAsync(Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// gethirezserverstatus
        /// </summary>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<IList<HirezServerStatus>>> GetHirezServerStatusWithHttpMessagesAsync(Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// ping
        /// </summary>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<string>> PingWithHttpMessagesAsync(Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

        /// <summary>
        /// TestSession
        /// </summary>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        Task<HttpOperationResponse<string>> TestSessionWithHttpMessagesAsync(Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));

    }
}


text:
let DescriptorView = require('./descriptor');
let AdvancedTable = require('../../main/views/advancedtable');
let DescriptorsColumnsView = require('../../descriptor/mixins/descriptorscolumns');

let View = AdvancedTable.extend({
    className: "object descriptor-group-list advanced-table-container",
    childView: DescriptorView,
    userSettingName: 'descriptor_list_columns',
    userSettingVersion: '1.4',

    defaultColumns: [
        {name: 'lock', width: 'auto', sort_by: null},
        {name: 'code', width: 'auto', sort_by: null},
        {name: 'name', width: 'auto', sort_by: null},
        {name: 'label', width: 'auto', sort_by: null},
        {name: 'group_name', width: 'auto', sort_by: '+0'}
    ],

    columnsOptions: {
        'lock': {
            label: '',
            width: 'auto',
            glyphicon: ['fa-lock', 'fa-lock'],
            fixed: true
        },
        'code': {label: _t('Code'), width: 'auto', minWidth: true, event: 'view-descriptor-details'},
        'name': {label: _t('Name'), width: 'auto', minWidth: true, event: 'view-descriptor-details'},
        'label': {label: _t('Label'), width: 'auto', minWidth: true, event: 'view-descriptor-details'},
        'group_name': {width: 'auto', minWidth: true}
    },

    initialize: function (options) {
        View.__super__.initialize.apply(this, arguments);

        this.filters = this.getOption('filters');
    },

    onRender: function () {
        View.__super__.onRender.apply(this, arguments);
        this.onShowTab();
    },

    onShowTab: function () {
        let view = this;

        let contextLayout = application.getView().getChildView('right');
        if (!contextLayout) {
            let DefaultLayout = require('../../main/views/defaultlayout');
            contextLayout = new DefaultLayout();
            application.getView().showChildView('right', contextLayout);
        }

        let TitleView = require('../../main/views/titleview');
        contextLayout.showChildView('title', new TitleView({
            title: _t("Actions on descriptors"),
            glyphicon: 'fa-wrench'
        }));

        let actions = [
            'create-descriptor'
        ];

        let ListContextView = require('./descriptorlistcontext');
        let contextView = new ListContextView({actions: actions});
        contextLayout.showChildView('content', contextView);

        contextView.on("descriptor:create", function () {
            view.onCreateDescriptor();
        });

        View.__super__.onShowTab.apply(this, arguments);
    },

    onBeforeDetach: function () {
        application.main.defaultRightView();
    },

    onCreateDescriptor: function () {
        window.application.descriptor.controllers.descriptor.create();
    }
});

// support of descriptors columns extension
_.extend(View.prototype, DescriptorsColumnsView);

module.exports = View;


text:
    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1, user-scalable=no">

    <link href="https://fonts.googleapis.com/css?family=Roboto" rel="stylesheet">
    <link rel="stylesheet" type="text/css" href="css/main.css">
    <link rel="stylesheet" type="text/css" href="css/index.css">

New Character

Import Character

      <textarea id="importTextArea" cols="50" rows="20"></textarea>

Import


text:
'use strict'

const BbPromise = require('bluebird'),
	  path = require('path'),
	  FCli = require('./utils/cli'),
	  FError = require('./Error');

const fs = BbPromise.promisifyAll(require("fs"));

// Global Bluebird Config
BbPromise.onPossiblyUnhandledRejection(function (err) {
	process.stderr.write(err.stack);
	process.abort();
});
BbPromise.longStackTraces();

class Funcy {
	constructor() {
		this._version = require('./../package.json').version;
		this._handlers = [];
		this.config = {
			HANDLER_PATH: __dirname + '/handlers'
		}
	}

	init() {
		let _this = this;
		return fs.readdirAsync(_this.config.HANDLER_PATH)
			.then(function (files) {
				files.forEach(function (file, index) {
					let Handler = require(path.join(_this.config.HANDLER_PATH, file));
					let h = new Handler();
					_this._handlers[h.context] = _this._handlers[h.context] || [];
					_this._handlers[h.context][h.action] = h;
				});
			})
			.catch(function (e) {
				throw new FError('Could not initialize handlers: ' + JSON.stringify(e));
			});
	}

	command(argv) {
		let cli = {
			context: null,
			action: null,
			raw: argv
		};

		if (cli.raw._[0] === 'version' || cli.raw._[0] === 'v') {
			console.log(this._version);
			return;
		}

		if (cli.raw._.length === 0) {
			FCli.asciiGreeting();
			FCli.quickHelp(this._handlers);
			return;
		}

		cli.context = cli.raw._[0];
		cli.action = cli.raw._[1];

		if (cli.raw.help || cli.raw.h || cli.raw._[2] === 'help') {
			this._handlers[cli.context][cli.action].help();
			return;
		}

		if (this._handlers[cli.context] === undefined) {
			throw new FError('In the command you just typed context "' + cli.context + '" is not valid. Enter "funcy-azure --help" to see available contexts and actions.');
		}

		if (this._handlers[cli.context][cli.action] === undefined) {
			throw new FError('In the command you just typed action "' + cli.action + '" is not valid. Enter "funcy-azure --help" to see available contexts and actions.');
		}

		this._handlers[cli.context][cli.action].process(cli.raw);
	}
}

module.exports = Funcy;


text:
package cruise.umple.umple;

import org.eclipse.emf.ecore.EObject;

/**
 *
 * A representation of the model object 'Generate '.
 *
 *
 *

 * The following features are supported:
 *

 *
{@link cruise.umple.umple.Generate_#getGenerate_1 Generate 1}

 *

 *

 *
 * @see cruise.umple.umple.UmplePackage#getGenerate_()
 * @model
 * @generated
 */
public interface Generate_ extends EObject
{
  /**
   * Returns the value of the 'Generate 1' attribute.
   *
   *

   * If the meaning of the 'Generate 1' attribute isn't clear,
   * there really should be more of a description here...
   *

   *
   * @return the value of the 'Generate 1' attribute.
   * @see #setGenerate_1(String)
   * @see cruise.umple.umple.UmplePackage#getGenerate__Generate_1()
   * @model
   * @generated
   */
  String getGenerate_1();

  /**
   * Sets the value of the '{@link cruise.umple.umple.Generate_#getGenerate_1 Generate 1}' attribute.
   *
   *
   * @param value the new value of the 'Generate 1' attribute.
   * @see #getGenerate_1()
   * @generated
   */
  void setGenerate_1(String value);

} // Generate_


text:
use nix::fcntl::OFlag;
use std::{
    cell::RefCell,
    os::unix::io::RawFd,
    path::Path,
    rc::Rc,
    sync::{Arc, Mutex},
};

/// General session interface.
///
/// Provides a way to open and close devices and change the active vt.
pub trait Session {
    /// Error type of the implementation
    type Error: AsErrno;

    /// Opens a device at the given `path` with the given flags.
    ///
    /// Returns a raw file descriptor
    fn open(&mut self, path: &Path, flags: OFlag) -> Result<RawFd, Self::Error>;
    /// Close a previously opened file descriptor
    fn close(&mut self, fd: RawFd) -> Result<(), Self::Error>;

    /// Change the currently active virtual terminal
    fn change_vt(&mut self, vt: i32) -> Result<(), Self::Error>;

    /// Check if this session is currently active
    fn is_active(&self) -> bool;
    /// Which seat this session is on
    fn seat(&self) -> String;
}

/// Signals that can be generated by a session
///
/// Objects that need to be notifier about activation and deactivation
/// of devices need to be linked to the `Signaler` of the session providing
/// these signals.
#[derive(Copy, Clone, Debug)]
pub enum Signal {
    /// The whole session has been paused
    ///
    /// All devices should be considered as paused
    PauseSession,
    /// A given device has been paused
    PauseDevice {
        /// Major number identifying the device
        major: u32,
        /// Minor number identifying the device
        minor: u32,
    },
    /// The whole session has been activated
    ActivateSession,
    /// A given device has been activated
    ActivateDevice {
        /// Major number identifying the device
        major: u32,
        /// Minor number identifying the device
        minor: u32,
        /// New file descriptor for accessing the device
        ///
        /// If it is not `None`, you should use this FD in place of the
        /// previous one for accessing this device. In case the old FD
        /// is different from the new one, the old one should be closed.
        new_fd: Option<RawFd>,
    },
}

impl Session for () {
    type Error = ();

    fn open(&mut self, _path: &Path, _flags: OFlag) -> Result<RawFd, Self::Error> {
        Err(())
    }
    fn close(&mut self, _fd: RawFd) -> Result<(), Self::Error> {
        Err(())
    }

    fn change_vt(&mut self, _vt: i32) -> Result<(), Self::Error> {
        Err(())
    }

    fn is_active(&self) -> bool {
        false
    }
    fn seat(&self) -> String {
        String::from("seat0")
    }
}

impl<S: Session> Session for Rc<RefCell<S>> {
    type Error = S::Error;

    fn open(&mut self, path: &Path, flags: OFlag) -> Result<RawFd, Self::Error> {
        self.borrow_mut().open(path, flags)
    }

    fn close(&mut self, fd: RawFd) -> Result<(), Self::Error> {
        self.borrow_mut().close(fd)
    }

    fn change_vt(&mut self, vt: i32) -> Result<(), Self::Error> {
        self.borrow_mut().change_vt(vt)
    }

    fn is_active(&self) -> bool {
        self.borrow().is_active()
    }

    fn seat(&self) -> String {
        self.borrow().seat()
    }
}

impl<S: Session> Session for Arc<Mutex<S>> {
    type Error = S::Error;

    fn open(&mut self, path: &Path, flags: OFlag) -> Result<RawFd, Self::Error> {
        self.lock().unwrap().open(path, flags)
    }

    fn close(&mut self, fd: RawFd) -> Result<(), Self::Error> {
        self.lock().unwrap().close(fd)
    }

    fn change_vt(&mut self, vt: i32) -> Result<(), Self::Error> {
        self.lock().unwrap().change_vt(vt)
    }

    fn is_active(&self) -> bool {
        self.lock().unwrap().is_active()
    }

    fn seat(&self) -> String {
        self.lock().unwrap().seat()
    }
}

/// Allows errors to be described by an error number
pub trait AsErrno: ::std::fmt::Debug {
    /// Returns the error number representing this error if any
    fn as_errno(&self) -> Option<i32>;
}

impl AsErrno for () {
    fn as_errno(&self) -> Option<i32> {
        None
    }
}

pub mod auto;
pub mod direct;
#[cfg(feature = "backend_session_libseat")]
pub mod libseat;

#[cfg(feature = "backend_session_logind")]
mod dbus;
#[cfg(feature = "backend_session_logind")]
pub use self::dbus::*;


text:
import difflib
import re

from ..edit import Edit
from . import strip_determiners, pos_tag
from .tokenize import default_tokenize_func

MULTIPLE_PUNCT_REGEX = re.compile(r'([.!?-]){2,}')
MULTIPLE_SPACE_REGEX = re.compile(r'([ ]){2,}')
GARBAGE_REGEX = re.compile(r'[^\w\s]')

def _get_line_num(edit_file):
    line_n = 0
    with open(edit_file) as input:
        line_n = sum(1 for _ in input)
    return line_n

def _prefilter_line(row):
    edits = []
    for edit in row:
        edit = edit.strip()
        if '\n' in edit:
            return
        # replace special formatting
        edit = edit.replace('*', '')
        edit = MULTIPLE_PUNCT_REGEX.sub('\g<1>', edit)
        edit = MULTIPLE_SPACE_REGEX.sub('\g<1>', edit)
        if not edit or edit == "null":
            edit = None
        edits.append(edit)
    return edits

def _is_garbage(ngram1, ngram2):
    """Filter useless edits"""
    if not ngram1 and not ngram2:
        return True
    elif ngram1.lower() == ngram2.lower():
        return True
    ngram1 = GARBAGE_REGEX.search(ngram1)
    ngram2 = GARBAGE_REGEX.search(ngram2)
    if ngram1 or ngram2:
        return True
    return False

def extract_edits(edit_file, substitutions=None, tokenize_func=default_tokenize_func):
    """
    Extracts contexts for all n-grams that were changed between two text versions.
    Uses most sequence matcher from difflib, which takes most longest subsequence.

    If ``substitutions'' argument is supplied, extract all n-gram matching substitutions,
    even if they were not changed.

    :returns: list of Edit objects
    """
    from pyutils import print_progress
    edit_n = 0
    edits = []
    line_n = _get_line_num(edit_file)

    with open(edit_file, 'r') as input:
        csvreader = csv.reader(input, delimiter='\t', encoding='utf-8')
        csvreader.__class__.__len__ = lambda x: line_n
        for row in print_progress(csvreader):
            row = _prefilter_line(row)
            edit1, edit2 = row
            if edit1 is None or edit2 is None:
                continue
            # tokenize to words, since we want word diff
            edit1 = strip_determiners(' '.join(tokenize_func(edit1))).split()
            edit2 = strip_determiners(' '.join(tokenize_func(edit2))).split()
            # identify pos tags here to not do it multiple times
            pos_tokens = pos_tag(' '.join(edit2))
            for seq in difflib.SequenceMatcher(None, edit1, edit2).get_grouped_opcodes(0):
                for tag, i1, i2, j1, j2 in seq:
                    if tag == 'equal':
                        continue
                    # extract merged edits into unigrams that match substitutions
                    if substitutions:
                        # TODO: works only for unigram substitutions
                        index1 = [(ix, i) for ix, i in enumerate(range(i1, i2)) if edit1[i] in substitutions]
                        index2 = [(ix, i) for ix, i in enumerate(range(j1, j2)) if edit2[i] in substitutions]
                        if len(index1) != 1 or len(index2) != 1 or index1[0][0] != index2[0][0]:
                            continue
                        i1, i2 = index1[0][1], index1[0][1]+1
                        j1, j2 = index2[0][1], index2[0][1]+1
                    edits.append(Edit(edit1, edit2, (i1, i2), (j1, j2), pos_tokens))
                    edit_n += 1

            # Add all other substitutions if supplied
            # TODO: works only for unigrams
            if substitutions:
                for i, unigram in enumerate(edit2):
                    if unigram in substitutions:
                        edits.append(Edit(edit2, edit2, (i, i+1), (i, i+1), pos_tokens))
                        edit_n += 1

        del csvreader.__class__.__len__
    print('Total edits extracted:', edit_n)
    return edits

def extract_filtered(edit_file, filter_func, tokenize_func=default_tokenize_func):
    """
    Extracts contexts for all words from edit_file that satisfy conditions in ``filter_func``.
    Only second text version is used.
    :returns: list of Edit objects
    """
    from pyutils import print_progress
    edit_n = 0
    edits = []
    line_n = _get_line_num(edit_file)
    with open(edit_file, 'r') as input:
        csvreader = csv.reader(input, delimiter='\t', encoding='utf-8')
        csvreader.__class__.__len__ = lambda x: line_n
        for row in print_progress(csvreader):
            row = _prefilter_line(row)
            _, edit2 = row
            if edit2 is None:
                continue
            # tokenize to words, since we want word diff
            edit2 = tokenize_func(edit2)
            for i1, word in enumerate(edit2):
                if filter_func(word):
                    edits.append(Edit(edit2, edit2, (i1, i1+1), (i1, i1+1)))
                    edit_n += 1
        del csvreader.__class__.__len__
    print('Total edits extracted:', edit_n)
    return edits