text:
import {Album} from "./album";
import {Track} from "./track";
export class DownloadItem {
    album: Album;
    track: Track;
}

redpajama_set_name: RedPajamaGithub


text:
Bound = Class.new()

function Bound:init(size, offset, pos)
    self.size = size or Size()
    -- bound offset is in relation to the offset x,y position of an entity for readability (which is why we use subtraction to center the box)
    self.offset = offset or Vector(-self.size.width / 2, -self.size.height / 2)
    self.pos = pos or Vector()
end


text:
package tech.anima.tinytypes.jackson;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.BeanDescription;
import com.fasterxml.jackson.databind.DeserializationConfig;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.deser.Deserializers;
import java.io.IOException;
import tech.anima.tinytypes.meta.BooleanTinyTypes;
import tech.anima.tinytypes.meta.ByteTinyTypes;
import tech.anima.tinytypes.meta.IntTinyTypes;
import tech.anima.tinytypes.meta.LongTinyTypes;
import tech.anima.tinytypes.meta.MetaTinyTypes;
import tech.anima.tinytypes.meta.ShortTinyTypes;
import tech.anima.tinytypes.meta.StringTinyTypes;

public class TinyTypesDeserializers extends Deserializers.Base {

    @Override
    public JsonDeserializer<?> findBeanDeserializer(JavaType type, DeserializationConfig config, BeanDescription beanDesc) throws JsonMappingException {
        Class<?> candidateTT = type.getRawClass();
        if (MetaTinyTypes.isTinyType(candidateTT)) {
            return new TinyTypesDeserializer(candidateTT);
        }
        return null;
    }

    public static class TinyTypesDeserializer<T> extends JsonDeserializer<T> {

        private final Class<T> type;

        public TinyTypesDeserializer(Class<T> type) {
            if (type == null || !MetaTinyTypes.isTinyType(type)) {
                throw new IllegalArgumentException(String.format("not a tinytype: %s", type == null ? "null" : type.getCanonicalName()));
            }
            this.type = type;
        }

        @Override
        public T deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
            return MetaTinyTypes.metaFor(type).newInstance(type, extractValue(p));
        }

        private Object extractValue(JsonParser p) throws IOException {
            if (StringTinyTypes.includes(type)) {
                return p.getText();
            }
            if (BooleanTinyTypes.includes(type)) {
                return p.getBooleanValue();
            }
            if (ByteTinyTypes.includes(type)) {
                return p.getByteValue();
            }
            if (ShortTinyTypes.includes(type)) {
                return p.getShortValue();
            }
            if (IntTinyTypes.includes(type)) {
                return p.getIntValue();
            }
            if (LongTinyTypes.includes(type)) {
                return p.getLongValue();
            }
            throw new IllegalStateException(String.format("not a tinytype: %s, this call should not be possible (typing enforced by ctor)", type.getCanonicalName()));
        }

    }

}


text:
package org.apache.spark.examples.ml;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SQLContext;

// $example on$
import com.google.common.collect.Lists;

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.attribute.Attribute;
import org.apache.spark.ml.attribute.AttributeGroup;
import org.apache.spark.ml.attribute.NumericAttribute;
import org.apache.spark.ml.feature.VectorSlicer;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.types.*;
// $example off$

public class JavaVectorSlicerExample {
  public static void main(String[] args) {
    SparkConf conf = new SparkConf().setAppName("JavaVectorSlicerExample");
    JavaSparkContext jsc = new JavaSparkContext(conf);
    SQLContext jsql = new SQLContext(jsc);

    // $example on$
    Attribute[] attrs = new Attribute[]{
      NumericAttribute.defaultAttr().withName("f1"),
      NumericAttribute.defaultAttr().withName("f2"),
      NumericAttribute.defaultAttr().withName("f3")
    };
    AttributeGroup group = new AttributeGroup("userFeatures", attrs);

    JavaRDD<Row> jrdd = jsc.parallelize(Lists.newArrayList(
      RowFactory.create(Vectors.sparse(3, new int[]{0, 1}, new double[]{-2.0, 2.3})),
      RowFactory.create(Vectors.dense(-2.0, 2.3, 0.0))
    ));

    DataFrame dataset = jsql.createDataFrame(jrdd, (new StructType()).add(group.toStructField()));

    VectorSlicer vectorSlicer = new VectorSlicer()
      .setInputCol("userFeatures").setOutputCol("features");

    vectorSlicer.setIndices(new int[]{1}).setNames(new String[]{"f3"});
    // or slicer.setIndices(new int[]{1, 2}), or slicer.setNames(new String[]{"f2", "f3"})

    DataFrame output = vectorSlicer.transform(dataset);

    System.out.println(output.select("userFeatures", "features").first());
    // $example off$
    jsc.stop();
  }
}


text:
<?php

/**
 * The bcmath extension in PHP does not implement certain operations
 * for elliptic curve encryption
 * This class implements all neccessary static methods
 *
 */
if (!defined('MAX_BASE'))
    define('MAX_BASE', 128);

class bcmath_Utils {

    public static function bcrand($min, $max=false) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            if (!$max) {
                $max = $min;
                $min = 0;
            }

            return bcadd(bcmul(bcdiv(mt_rand(0, mt_getrandmax()), mt_getrandmax(), strlen($max)), bcsub(bcadd($max, 1), $min)), $min);
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

    public static function bchexdec($hex) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            $len = strlen($hex);
            $dec = '';
            for ($i = 1; $i <= $len; $i++)
                $dec = bcadd($dec, bcmul(strval(hexdec($hex[$i - 1])), bcpow('16', strval($len - $i))));

            return $dec;
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

    public static function bcdechex($dec) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            $hex = '';
            $positive = $dec < 0 ? false : true;

            while ($dec) {
                $hex .= dechex(abs(bcmod($dec, '16')));
                $dec = bcdiv($dec, '16', 0);
            }

            if ($positive)
                return strrev($hex);

            for ($i = 0; $isset($hex[$i]); $i++)
                $hex[$i] = dechex(15 - hexdec($hex[$i]));
            for ($i = 0; isset($hex[$i]) && $hex[$i] == 'f'; $i++)
                $hex[$i] = '0';
            if (isset($hex[$i]))
                $hex[$i] = dechex(hexdec($hex[$i]) + 1);
            return strrev($hex);
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

    public static function bcand($x, $y) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            return self::_bcbitwise_internal($x, $y, 'bcmath_Utils::_bcand');
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

// Bitwise OR

    public static function bcor($x, $y) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            return self::_bcbitwise_internal($x, $y, 'self::_bcor');
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

// Bitwise XOR

    public static function bcxor($x, $y) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            return self::_bcbitwise_internal($x, $y, 'self::_bcxor');
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

// Left shift (<<)

    public static function bcleftshift($num, $shift) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            bcscale(0);
            return bcmul($num, bcpow(2, $shift));
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

// Right shift (>>)

    public static function bcrightshift($num, $shift) {
        if (extension_loaded('bcmath') && USE_EXT=='BCMATH') {
            bcscale(0);
            return bcdiv($num, bcpow(2, $shift));
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

//// INTERNAL ROUTINES
// These routines operate on only one byte. They are used to
// implement _bcbitwise_internal.

    public static function _bcand($x, $y) {

        return $x & $y;
    }

    public static function _bcor($x, $y) {

        return $x | $y;
    }

    public static function _bcxor($x, $y) {

        return $x ^ $y;
    }

// _bcbitwise_internal - The majority of the code that implements
//                       the bitwise functions bcand, bcor, and bcxor.
//
// arguments           - $x and $y are the operands (in decimal format),
//                       and $op is the name of one of the three
//                       internal functions, _bcand, _bcor, or _bcxor.
//
//
// see also            - The interfaces to this function: bcand, bcor,
//                       and bcxor

    public static function _bcbitwise_internal($x, $y, $op) {

        $bx = self::bc2bin($x);
        $by = self::bc2bin($y);

        // Pad $bx and $by so that both are the same length.

        self::equalbinpad($bx, $by);

        $ix = 0;
        $ret = '';

        for ($ix = 0; $ix < strlen($bx); $ix++) {
            $xd = substr($bx, $ix, 1);
            $yd = substr($by, $ix, 1);
            $ret .= call_user_func($op, $xd, $yd);
        }

        return self::bin2bc($ret);
    }

    public static function bc2bin($num) {

        return self::dec2base($num, MAX_BASE);
    }

    public static function bin2bc($num) {
        return self::base2dec($num, MAX_BASE);
    }

    public static function dec2base($dec, $base, $digits=FALSE) {
        if (extension_loaded('bcmath')) {
            if ($base < 2 or $base > 256)
                die("Invalid Base: " . $base);
            bcscale(0);
            $value = "";
            if (!$digits)
                $digits = self::digits($base);
            while ($dec > $base - 1) {
                $rest = bcmod($dec, $base);
                $dec = bcdiv($dec, $base);
                $value = $digits[$rest] . $value;
            }
            $value = $digits[intval($dec)] . $value;
            return (string) $value;
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

    public static function base2dec($value, $base, $digits=FALSE) {
        if (extension_loaded('bcmath')) {
            if ($base < 2 or $base > 256)
                die("Invalid Base: " . $base);
            bcscale(0);
            if ($base < 37)
                $value = strtolower($value);
            if (!$digits)
                $digits = self::digits($base);
            $size = strlen($value);
            $dec = "0";
            for ($loop = 0; $loop < $size; $loop++) {
                $element = strpos($digits, $value[$loop]);
                $power = bcpow($base, $size - $loop - 1);
                $dec = bcadd($dec, bcmul($element, $power));
            }
            return (string) $dec;
        } else {
            throw new ErrorException("Please install BCMATH");
        }
    }

    public static function digits($base) {
        if ($base > 64) {
            $digits = "";
            for ($loop = 0; $loop < 256; $loop++) {
                $digits.=chr($loop);
            }
        } else {
            $digits = "0123456789abcdefghijklmnopqrstuvwxyz";
            $digits.="ABCDEFGHIJKLMNOPQRSTUVWXYZ-_";
        }
        $digits = substr($digits, 0, $base);
        return (string) $digits;
    }

    public static function equalbinpad(&$x, &$y) {

        $xlen = strlen($x);
        $ylen = strlen($y);

        $length = max($xlen, $ylen);
        self::fixedbinpad($x, $length);
        self::fixedbinpad($y, $length);
    }

    public static function fixedbinpad(&$num, $length) {

        $pad = '';
        for ($ii = 0; $ii < $length - strlen($num); $ii++) {
            $pad .= self::bc2bin('0');
        }

        $num = $pad . $num;
    }

}
?>


text:
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Linq;

using Android.App;
using Android.Widget;
using Android.OS;
using Android.Content;
using Android.Views;

using zsquared;

namespace a_vitavol
{
    [Activity(Theme = "@android:style/Theme.DeviceDefault.NoActionBar", Label = "A_AdminSites")]
    public class A_AdminSites : Activity
    {
        C_Global Global;
        C_VitaUser LoggedInUser;

        Button B_NewSite;
        Spinner SP_WhichSites;
        ListView LV_Sites;
        //ProgressBar PB_Busy;
        C_BusyBox BusyBox;

        List<E_SiteType> SiteTypes;
        C_ListViewHelper<C_VitaSite> SitesAdapter;
        C_SPinnerHelper<E_SiteType> SitesTypeSpinner;

        protected override void OnCreate(Bundle savedInstanceState)
        {
            base.OnCreate(savedInstanceState);

            MyAppDelegate g = (MyAppDelegate)Application;
            if (g.Global == null)
                g.Global = new C_Global();
            Global = g.Global;
            LoggedInUser = Global.GetUserFromCacheNoFetch(Global.LoggedInUserId);

            SetContentView(Resource.Layout.AdminSites);

            B_NewSite = FindViewById<Button>(Resource.Id.B_NewSite);
            SP_WhichSites = FindViewById<Spinner>(Resource.Id.SP_WhichSite);
            LV_Sites = FindViewById<ListView>(Resource.Id.LV_Sites);
            //PB_Busy = FindViewById<ProgressBar>(Resource.Id.PB_Busy);
            BusyBox = new C_BusyBox(this, "Loading...");

            C_Common.SetViewColors(this, Resource.Id.V_AdminSites);

            B_NewSite.Click += (sender, e) =>
            {
                Global.SelectedSiteName = null;
                Global.SelectedSiteSlug = null;
                Global.SelectedSiteTemp = null;
                StartActivity(new Intent(this, typeof(A_AdminSite)));
            };

            LV_Sites.ItemClick += (object sender, AdapterView.ItemClickEventArgs e) =>
            {
                C_VitaSite site = SitesAdapter.Items[e.Position];
                Global.SelectedSiteName = site.Name;
                Global.SelectedSiteSlug = site.Slug;
                Global.SelectedSiteTemp = null;
                StartActivity(new Intent(this, typeof(A_AdminSite)));
            };

            SiteTypes = Tools.EnumToList<E_SiteType>();
            SitesTypeSpinner = new C_SPinnerHelper<E_SiteType>(this, SP_WhichSites, SiteTypes);
            SitesTypeSpinner.SetValue(E_SiteType.All);
            SitesTypeSpinner.ItemSelected += (object sender, SpinnerEventArgs<E_SiteType> args) =>
            {
                E_SiteType t = args.Item;

                List<C_VitaSite> ourSites = Global.GetAllSitesNoFetch();
                var ou = ourSites.Where(s => (t == E_SiteType.All) || (s.SiteType == t));
                List<C_VitaSite> sitesList = ou.ToList();
                sitesList.Sort(C_VitaSite.CompareSitesByNameAscending);

                StartLV(sitesList);
            };

            //PB_Busy.Visibility = ViewStates.Visible;
            if (!Global.AllSitesFetched)
                BusyBox.Show();
            EnableUI(false);

            Task.Run(async () =>
            {
                List<C_VitaSite> ourSites = await Global.FetchAllSites(LoggedInUser.Token);
                ourSites.Sort(C_VitaSite.CompareSitesByNameAscendingLower);

                void p()
                {
                    //PB_Busy.Visibility = ViewStates.Gone;
                    BusyBox.Hide();
                    EnableUI(true);

                    StartLV(ourSites);
                }
                RunOnUiThread(p);
            });
        }

        private void StartLV(List<C_VitaSite> siteList)
        {
            SitesAdapter = new C_ListViewHelper<C_VitaSite>(this, LV_Sites, siteList);
            SitesAdapter.GetTextLabel += (sender, args) =>
            {
                C_VitaSite site = args.Item;
                return site.Name;
            };
            SitesAdapter.GetDetailTextLabel += (sender, args) =>
            {
                C_VitaSite site = args.Item;
                string ms = site.SiteType == E_SiteType.Mobile ? " [Mobile]" : "";
                return site.Street + ", " + site.City + " " + site.State + ms;
            };
        }

        bool UIIsEnabled;
        private void EnableUI(bool en)
        {
            UIIsEnabled = en;
            B_NewSite.Enabled = en;
            SP_WhichSites.Enabled = en;
            LV_Sites.Enabled = en;
        }

        public override void OnBackPressed()
        {
            if (UIIsEnabled)
                StartActivity(new Intent(this, typeof(A_AdminMenu)));
        }
    }
}


text:
@interface PodsDummy_Pods_testGitHub3_Tests : NSObject
@end
@implementation PodsDummy_Pods_testGitHub3_Tests
@end


text:
#pragma once
#include <aws/iam/IAM_EXPORTS.h>
#include <aws/iam/IAMRequest.h>

namespace Aws
{
namespace IAM
{
namespace Model
{

  /*
  */
  class AWS_IAM_API ListSAMLProvidersRequest : public IAMRequest
  {
  public:
    ListSAMLProvidersRequest();
    Aws::String SerializePayload() const override;

  };

} // namespace Model
} // namespace IAM
} // namespace Aws


text:
package org.apache.jackrabbit.oak.plugins.index.lucene;

import static java.util.Arrays.asList;
import static org.apache.jackrabbit.oak.api.QueryEngine.NO_BINDINGS;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.useV2;
import static org.hamcrest.CoreMatchers.containsString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;

import java.text.ParseException;
import java.util.Collections;
import java.util.List;
import java.util.Set;

import javax.jcr.PropertyType;

import ch.qos.logback.classic.Level;
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.ContentRepository;
import org.apache.jackrabbit.oak.api.Result;
import org.apache.jackrabbit.oak.api.ResultRow;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexDefinitionBuilder;
import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
import org.apache.jackrabbit.oak.InitialContentHelper;
import org.apache.jackrabbit.oak.query.AbstractQueryTest;
import org.apache.jackrabbit.oak.spi.commit.Observer;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.Assert;
import org.junit.Test;

import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;

public class FunctionIndexTest extends AbstractQueryTest {

    private LuceneIndexEditorProvider editorProvider;

    private NodeStore nodeStore;

    @Override
    protected ContentRepository createRepository() {
        editorProvider = new LuceneIndexEditorProvider();
        LuceneIndexProvider provider = new LuceneIndexProvider();
        nodeStore = new MemoryNodeStore(InitialContentHelper.INITIAL_CONTENT);
        return new Oak(nodeStore)
                .with(new OpenSecurityProvider())
                .with((QueryIndexProvider) provider)
                .with((Observer) provider)
                .with(editorProvider)
                .with(new PropertyIndexEditorProvider())
                .with(new NodeTypeIndexProvider())
                .createContentRepository();
    }

    @Test
    public void noIndexTest() throws Exception {
        Tree test = root.getTree("/").addChild("test");
        for (int idx = 0; idx < 3; idx++) {
            Tree low = test.addChild("" + (char) ('a' + idx));
            low.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            Tree up = test.addChild("" + (char) ('A' + idx));
            up.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        }
        root.commit();

        String query = "select [jcr:path] from [nt:base] where lower(localname()) = 'b'";
        assertThat(explain(query), containsString("traverse"));
        assertQuery(query, Lists.newArrayList("/test/b", "/test/B"));

        String queryXPath = "/jcr:root/test//*[fn:lower-case(fn:local-name()) = 'b']";
        assertThat(explainXpath(queryXPath), containsString("traverse"));
        assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/b", "/test/B"));

        queryXPath = "/jcr:root/test//*[fn:lower-case(fn:local-name()) > 'b']";
        assertThat(explainXpath(queryXPath), containsString("traverse"));
        assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/c", "/test/C"));

        query = "select [jcr:path] from [nt:base] where lower(localname()) = 'B'";
        assertThat(explain(query), containsString("traverse"));
        assertQuery(query, Lists.<String>newArrayList());
    }

    @Test
    public void lowerCaseLocalName() throws Exception {
        Tree luceneIndex = createIndex("lowerLocalName", Collections.<String>emptySet());
        luceneIndex.setProperty("excludedPaths",
                Lists.newArrayList("/jcr:system", "/oak:index"), Type.STRINGS);
        Tree func = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("lowerLocalName");
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "lower(localname())");

        Tree test = root.getTree("/").addChild("test");
        for (int idx = 0; idx < 3; idx++) {
            Tree low = test.addChild("" + (char) ('a' + idx));
            low.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            Tree up = test.addChild("" + (char) ('A' + idx));
            up.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        }
        root.commit();

        String query = "select [jcr:path] from [nt:base] where lower(localname()) = 'b'";
        assertThat(explain(query), containsString("lucene:lowerLocalName"));
        assertQuery(query, Lists.newArrayList("/test/b", "/test/B"));

        String queryXPath = "/jcr:root//*[fn:lower-case(fn:local-name()) = 'b']";
        assertThat(explainXpath(queryXPath), containsString("lucene:lowerLocalName"));
        assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/b", "/test/B"));

        queryXPath = "/jcr:root//*[fn:lower-case(fn:local-name()) > 'b']";
        assertThat(explainXpath(queryXPath), containsString("lucene:lowerLocalName"));
        assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/c", "/test/C", "/test"));

        query = "select [jcr:path] from [nt:base] where lower(localname()) = 'B'";
        assertThat(explain(query), containsString("lucene:lowerLocalName"));
        assertQuery(query, Lists.<String>newArrayList());
    }

    @Test
    public void lengthName() throws Exception {
        Tree luceneIndex = createIndex("lengthName", Collections.<String>emptySet());
        luceneIndex.setProperty("excludedPaths",
                Lists.newArrayList("/jcr:system", "/oak:index"), Type.STRINGS);
        Tree func = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("lengthName");
        func.setProperty(FulltextIndexConstants.PROP_ORDERED, true);
        func.setProperty(FulltextIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:string-length(fn:name())");

        Tree test = root.getTree("/").addChild("test");
        for (int idx = 1; idx < 1000; idx *= 10) {
            Tree testNode = test.addChild("test" + idx);
            testNode.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        }
        root.commit();

        String query = "select [jcr:path] from [nt:base] where length(name()) = 6";
        assertThat(explain(query), containsString("lucene:lengthName"));
        assertQuery(query, Lists.newArrayList("/test/test10"));

        String queryXPath = "/jcr:root//*[fn:string-length(fn:name()) = 7]";
        assertThat(explainXpath(queryXPath), containsString("lucene:lengthName"));
        assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/test100"));

        queryXPath = "/jcr:root//* order by fn:string-length(fn:name())";
        assertThat(explainXpath(queryXPath), containsString("lucene:lengthName"));
        assertQuery(queryXPath, "xpath", Lists.newArrayList(
                "/test", "/test/test1", "/test/test10", "/test/test100"));
    }

    @Test
    public void length() throws Exception {
        Tree luceneIndex = createIndex("length", Collections.<String>emptySet());
        luceneIndex.setProperty("excludedPaths",
                Lists.newArrayList("/jcr:system", "/oak:index"), Type.STRINGS);
        Tree func = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("lengthName");
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:string-length(@value)");

        Tree test = root.getTree("/").addChild("test");
        for (int idx = 1; idx <= 1000; idx *= 10) {
            Tree testNode = test.addChild("test" + idx);
            testNode.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            testNode.setProperty("value", new byte[idx]);
        }
        root.commit();

        String query = "select [jcr:path] from [nt:base] where length([value]) = 100";
        assertThat(explain(query), containsString("lucene:length"));
        assertQuery(query, Lists.newArrayList("/test/test100"));

        String queryXPath = "/jcr:root//*[fn:string-length(@value) = 10]";
        assertThat(explainXpath(queryXPath), containsString("lucene:length"));
        assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/test10"));
    }

    @Test
    public void upperCase() throws Exception {
        Tree luceneIndex = createIndex("upper", Collections.<String>emptySet());
        Tree func = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("upperName");
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(@name)");

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        List<String> paths = Lists.newArrayList();
        for (int idx = 0; idx < 15; idx++) {
            Tree a = test.addChild("n"+idx);
            a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            a.setProperty("name", "10% foo");
            paths.add("/test/n" + idx);
        }
        root.commit();

        String query = "select [jcr:path] from [nt:unstructured] where upper([name]) = '10% FOO'";
        assertThat(explain(query), containsString("lucene:upper"));
        assertQuery(query, paths);

        query = "select [jcr:path] from [nt:unstructured] where upper([name]) like '10\\% FOO'";
        assertThat(explain(query), containsString("lucene:upper"));
        assertQuery(query, paths);
    }

    @Test
    public void testOrdering2() throws Exception {
        Tree index = root.getTree("/");
        Tree indexDefn = createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE);
        useV2(indexDefn);
        indexDefn.setProperty(LuceneIndexConstants.TEST_MODE, true);
        indexDefn.setProperty(FulltextIndexConstants.EVALUATE_PATH_RESTRICTION, true);
        Tree props = TestUtil.newRulePropTree(indexDefn, "nt:unstructured");
        props.getParent().setProperty(FulltextIndexConstants.INDEX_NODE_NAME, true);
        TestUtil.enableForFullText(props, FulltextIndexConstants.REGEX_ALL_PROPS, true);
        Tree upper = TestUtil.enableFunctionIndex(props, "upper([foo])");
        upper.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        root.commit();

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree a = test.addChild("n1");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "hello");

        a = test.addChild("n2");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "World!");
        a = test.addChild("n3");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "Hallo");
        a = test.addChild("n4");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "10%");
        a = test.addChild("n5");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "10 percent");

        a = test.addChild("n0");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a = test.addChild("n9");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        String query = "select a.[foo]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo)";

        root.commit();

        assertThat(explain(query), containsString("lucene:test-index(/oak:index/test-index)"));

        List<String> result = executeQuery(query, SQL2);
        assertEquals("Ordering doesn't match", asList("10 percent", "10%", "Hallo", "hello", "World!"), result);

    }

    /*
    Test order by func(a),func(b)
    order by func(b),func(a)
    func(a) DESC,func(b)
    func(a),func(b)DESC
    where both func(a) and func(b) have ordered set = true
    Correct ordering is effectively served by the index
     */
    @Test
    public void testOrdering3() throws Exception {

        Tree index = root.getTree("/");
        Tree indexDefn = createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE);
        useV2(indexDefn);
        indexDefn.setProperty(LuceneIndexConstants.TEST_MODE, true);
        indexDefn.setProperty(FulltextIndexConstants.EVALUATE_PATH_RESTRICTION, true);
        Tree props = TestUtil.newRulePropTree(indexDefn, "nt:unstructured");
        props.getParent().setProperty(FulltextIndexConstants.INDEX_NODE_NAME, true);
        TestUtil.enableForFullText(props, FulltextIndexConstants.REGEX_ALL_PROPS, true);

        Tree upper = TestUtil.enableFunctionIndex(props, "upper([foo])");
        upper.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        Tree upper2 = TestUtil.enableFunctionIndex(props, "upper([foo2])");
        upper2.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        root.commit();

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree a = test.addChild("n1");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b2");

        a = test.addChild("n2");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a2");
        a.setProperty("foo2", "b3");

        a = test.addChild("n3");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a3");
        a.setProperty("foo2", "b1");

        a = test.addChild("n4");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b3");

        a = test.addChild("n5");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b1");

        String query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo),upper(a.foo2)";

        root.commit();

        List<String> result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b1", "a1, b2", "a1, b3", "a2, b3", "a3, b1"), result);

        query = "select a.[foo2],a.[foo]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo2),upper(a.foo)";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("b1, a1", "b1, a3", "b2, a1", "b3, a1", "b3, a2"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo) DESC, upper(a.foo2)";
        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a3, b1", "a2, b3", "a1, b1", "a1, b2", "a1, b3"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo), upper(a.foo2) DESC";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b3", "a1, b2", "a1, b1", "a2, b3", "a3, b1"), result);

    }

    /*
    Test order by func(a),func(b)
    order by func(b),func(a)
    func(a) DESC,func(b)
    func(a),func(b)DESC
    where only func(a) is ordered by index
    The effective ordering in this case will be done by QueryEngine
     */
    @Test
    public void testOrdering4() throws Exception {
        Tree index = root.getTree("/");
        Tree indexDefn = createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE);
        useV2(indexDefn);
        indexDefn.setProperty(LuceneIndexConstants.TEST_MODE, true);
        indexDefn.setProperty(FulltextIndexConstants.EVALUATE_PATH_RESTRICTION, true);
        Tree props = TestUtil.newRulePropTree(indexDefn, "nt:unstructured");
        props.getParent().setProperty(FulltextIndexConstants.INDEX_NODE_NAME, true);
        TestUtil.enableForFullText(props, FulltextIndexConstants.REGEX_ALL_PROPS, true);

        Tree upper = TestUtil.enableFunctionIndex(props, "upper([foo])");
        upper.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        TestUtil.enableFunctionIndex(props, "upper([foo2])");

        root.commit();

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree a = test.addChild("n1");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b2");

        a = test.addChild("n2");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a2");
        a.setProperty("foo2", "b3");

        a = test.addChild("n3");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a3");
        a.setProperty("foo2", "b1");

        a = test.addChild("n4");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b3");

        a = test.addChild("n5");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b1");

        String query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo),upper(a.foo2)";

        root.commit();

        List<String> result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b1", "a1, b2", "a1, b3", "a2, b3", "a3, b1"), result);

        query = "select a.[foo2],a.[foo]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo2),upper(a.foo)";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("b1, a1", "b1, a3", "b2, a1", "b3, a1", "b3, a2"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo) DESC, upper(a.foo2)";
        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a3, b1", "a2, b3", "a1, b1", "a1, b2", "a1, b3"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo), upper(a.foo2) DESC";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b3", "a1, b2", "a1, b1", "a2, b3", "a3, b1"), result);

    }

    /*
    Test order by func(a),b
    order by b,func(a)
    order by func(a) DESC,b
    order by func(a),b DESC
    where both b and func(a) have ordered=true
     */
    @Test
    public void testOrdering5() throws Exception {
        Tree index = root.getTree("/");
        Tree indexDefn = createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE);
        useV2(indexDefn);
        indexDefn.setProperty(LuceneIndexConstants.TEST_MODE, true);
        indexDefn.setProperty(FulltextIndexConstants.EVALUATE_PATH_RESTRICTION, true);
        Tree props = TestUtil.newRulePropTree(indexDefn, "nt:unstructured");
        props.getParent().setProperty(FulltextIndexConstants.INDEX_NODE_NAME, true);
        TestUtil.enableForFullText(props, FulltextIndexConstants.REGEX_ALL_PROPS, true);

        Tree upper = TestUtil.enableFunctionIndex(props, "upper([foo])");
        upper.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        Tree upper2 = TestUtil.enablePropertyIndex(props, "foo2", false);
        upper2.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        root.commit();

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree a = test.addChild("n1");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b2");

        a = test.addChild("n2");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a2");
        a.setProperty("foo2", "b3");

        a = test.addChild("n3");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a3");
        a.setProperty("foo2", "b1");

        a = test.addChild("n4");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b3");

        a = test.addChild("n5");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b1");

        String query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo),a.foo2";

        root.commit();

        List<String> result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b1", "a1, b2", "a1, b3", "a2, b3", "a3, b1"), result);

        query = "select a.[foo2],a.[foo]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by a.foo2,upper(a.foo)";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("b1, a1", "b1, a3", "b2, a1", "b3, a1", "b3, a2"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo) DESC, a.foo2";
        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a3, b1", "a2, b3", "a1, b1", "a1, b2", "a1, b3"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo), a.foo2 DESC";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b3", "a1, b2", "a1, b1", "a2, b3", "a3, b1"), result);

    }

    /*
    Test order by func(a),b
    orrder by b,func(a)
    order by func(a) DESC,b
    order by func(a),b DESC
    where func(a) does not have ordered = true
     */
    @Test
    public void testOrdering6() throws Exception {
        Tree index = root.getTree("/");
        Tree indexDefn = createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE);
        useV2(indexDefn);
        indexDefn.setProperty(LuceneIndexConstants.TEST_MODE, true);
        indexDefn.setProperty(FulltextIndexConstants.EVALUATE_PATH_RESTRICTION, true);
        Tree props = TestUtil.newRulePropTree(indexDefn, "nt:unstructured");
        props.getParent().setProperty(FulltextIndexConstants.INDEX_NODE_NAME, true);
        TestUtil.enableForFullText(props, FulltextIndexConstants.REGEX_ALL_PROPS, true);

        TestUtil.enableFunctionIndex(props, "upper([foo])");

        Tree upper2 = TestUtil.enablePropertyIndex(props, "foo2", false);
        upper2.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        root.commit();

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree a = test.addChild("n1");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b2");

        a = test.addChild("n2");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a2");
        a.setProperty("foo2", "b3");

        a = test.addChild("n3");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a3");
        a.setProperty("foo2", "b1");

        a = test.addChild("n4");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b3");

        a = test.addChild("n5");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "a1");
        a.setProperty("foo2", "b1");

        String query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo),a.foo2";

        root.commit();

        List<String> result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b1", "a1, b2", "a1, b3", "a2, b3", "a3, b1"), result);

        query = "select a.[foo2],a.[foo]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by a.foo2,upper(a.foo)";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("b1, a1", "b1, a3", "b2, a1", "b3, a1", "b3, a2"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo) DESC, a.foo2";
        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a3, b1", "a2, b3", "a1, b1", "a1, b2", "a1, b3"), result);

        query = "select a.[foo],a.[foo2]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where a.foo is not null and isdescendantnode(a , '/test') order by upper(a.foo), a.foo2 DESC";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("a1, b3", "a1, b2", "a1, b1", "a2, b3", "a3, b1"), result);
    }

    /*
    Testing order by for
    different function implementations
     */
    @Test
    public void testOrdering7() throws Exception {
        Tree index = root.getTree("/");
        Tree indexDefn = createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE);
        useV2(indexDefn);
        indexDefn.setProperty(LuceneIndexConstants.TEST_MODE, true);
        indexDefn.setProperty(FulltextIndexConstants.EVALUATE_PATH_RESTRICTION, true);
        Tree props = TestUtil.newRulePropTree(indexDefn, "nt:unstructured");
        props.getParent().setProperty(FulltextIndexConstants.INDEX_NODE_NAME, true);
        TestUtil.enableForFullText(props, FulltextIndexConstants.REGEX_ALL_PROPS, true);

        Tree fn = TestUtil.enableFunctionIndex(props, "upper([foo])");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        fn = TestUtil.enableFunctionIndex(props, "lower([foo])");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        fn = TestUtil.enableFunctionIndex(props, "length([foo])");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);
        // Any function property trying to sory by length needs to explicitly set the type to Long
        fn.setProperty(FulltextIndexConstants.PROP_TYPE, "Long");

        fn = TestUtil.enableFunctionIndex(props, "coalesce([foo2],[foo])");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        fn = TestUtil.enableFunctionIndex(props, "name()");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        fn = TestUtil.enableFunctionIndex(props, "localname()");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        fn = TestUtil.enableFunctionIndex(props, "lower(coalesce([foo2], coalesce([foo], localname())))");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);

        fn = TestUtil.enableFunctionIndex(props, "length(coalesce([foo], coalesce([foo2], localname())))");
        fn.setProperty(FulltextIndexConstants.PROP_ORDERED, true);
        fn.setProperty(FulltextIndexConstants.PROP_TYPE, "Long");

        root.commit();

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree a = test.addChild("d1");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "c");

        a = test.addChild("d2");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "bbbb");
        a.setProperty("foo2", "22");

        a = test.addChild("d3");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "aa");

        a = test.addChild("jcr:content");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        a.setProperty("foo", "test");
        a.setProperty("foo2", "11");

        root.commit();

        String query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  isdescendantnode(a , '/test') order by coalesce([foo2],[foo]) ";

        List<String> result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/jcr:content", "/test/d2", "/test/d3", "/test/d1"), result);

        query = "select a.[foo]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  isdescendantnode(a , '/test') order by lower([a].[foo])";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("aa", "bbbb", "c", "test"), result);

        query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  isdescendantnode(a , '/test') order by localname() ";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/jcr:content", "/test/d1", "/test/d2", "/test/d3"), result);

        query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  isdescendantnode(a , '/test') order by name() ";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/d1", "/test/d2", "/test/d3", "/test/jcr:content"), result);

        query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  isdescendantnode(a , '/test') order by lower(coalesce([a].[foo2], coalesce([a].[foo], localname())))";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/jcr:content", "/test/d2", "/test/d3", "/test/d1"), result);

        query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  isdescendantnode(a , '/test') order by lower(coalesce([a].[foo2], coalesce([a].[foo], localname()))) DESC";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/d1", "/test/d3", "/test/d2", "/test/jcr:content"), result);

        query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  a.[foo] is not null AND isdescendantnode(a , '/test') order by length([a].[foo]) DESC, localname()";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/jcr:content", "/test/d2", "/test/d3", "/test/d1"), result);

        query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  a.[foo] is not null AND isdescendantnode(a , '/test') order by length([a].[foo]), localname()";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/d1", "/test/d3", "/test/jcr:content", "/test/d2"), result);

        query = "select [jcr:path]\n" +
                "\t  from [nt:unstructured] as a\n" +
                "\t  where  a.[foo] is not null AND isdescendantnode(a , '/test') order by length(coalesce([foo], coalesce([foo2], localname()))), localname() DESC";

        result = executeQuery(query, SQL2);

        assertEquals("Ordering doesn't match", asList("/test/d1", "/test/d3", "/test/d2", "/test/jcr:content"), result);

    }

    @Test
    public void testOrdering() throws Exception {
        Tree luceneIndex = createIndex("upper", Collections.<String>emptySet());
        Tree nonFunc = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("foo");
        nonFunc.setProperty(FulltextIndexConstants.PROP_ORDERED, true);
        nonFunc.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX, true);
        nonFunc.setProperty("name", "foo");

        Tree func = luceneIndex.getChild(FulltextIndexConstants.INDEX_RULES)
                .getChild("nt:base")
                .getChild(FulltextIndexConstants.PROP_NODE)
                .addChild("fooUpper");
        func.setProperty(FulltextIndexConstants.PROP_ORDERED, true);
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(@foo)");
        func.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX, true);

        root.commit();

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        List<String> paths = Lists.newArrayList();
        for (int idx = 0; idx < 10; idx++) {
            paths.add("/test/n" + idx);
            if (idx % 2 == 0) continue;
            Tree a = test.addChild("n" + idx);
            a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            a.setProperty("foo", "bar" + idx);

        }
        for (int idx = 0; idx < 10; idx++) {
            if (idx % 2 != 0) continue;
            Tree a = test.addChild("n" + idx);
            a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            a.setProperty("foo", "bar" + idx);
        }
        root.commit();

        String query = "/jcr:root//element(*, nt:unstructured) [jcr:like(fn:upper-case(@foo),'BAR%')] order by foo";
        assertThat(explainXpath(query), containsString("lucene:upper"));
        List<String> result = assertQuery(query, "xpath", paths);
        assertEquals("Ordering doesn't match", paths, result);

        query = "/jcr:root//element(*, nt:unstructured) [jcr:like(fn:upper-case(@foo),'BAR%')] order by fn:upper-case(@foo)";
        assertThat(explainXpath(query), containsString("lucene:upper"));
        List<String> result2 = assertQuery(query, "xpath", paths);
        assertEquals("Ordering doesn't match", paths, result2);
    }

    @Test
    public void upperCaseRelative() throws Exception {
        Tree luceneIndex = createIndex("upper", Collections.<String>emptySet());
        Tree func = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("upperName");
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "upper([data/name])");

        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        List<String> paths = Lists.newArrayList();
        for (int idx = 0; idx < 15; idx++) {
            Tree a = test.addChild("n"+idx);
            a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            Tree b = a.addChild("data");
            b.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            b.setProperty("name", "foo");
            paths.add("/test/n" + idx);
        }
        root.commit();

        String query = "select [jcr:path] from [nt:unstructured] where upper([data/name]) = 'FOO'";
        assertThat(explain(query), containsString("lucene:upper"));
        assertQuery(query, paths);

        String queryXPath = "/jcr:root//element(*, nt:unstructured)[fn:upper-case(data/@name) = 'FOO']";
        assertThat(explainXpath(queryXPath), containsString("lucene:upper"));
        assertQuery(queryXPath, "xpath", paths);

        for (int idx = 0; idx < 15; idx++) {
            Tree a = test.getChild("n"+idx);
            Tree b = a.getChild("data");
            b.setProperty("name", "bar");
        }
        root.commit();

        query = "select [jcr:path] from [nt:unstructured] where upper([data/name]) = 'BAR'";
        assertThat(explain(query), containsString("lucene:upper"));
        assertQuery(query, paths);

        queryXPath = "/jcr:root//element(*, nt:unstructured)[fn:upper-case(data/@name) = 'BAR']";
        assertThat(explainXpath(queryXPath), containsString("lucene:upper"));
        assertQuery(queryXPath, "xpath", paths);
    }

    @Test
    public void coalesce() throws Exception{
        IndexDefinitionBuilder idxb = new LuceneIndexDefinitionBuilder().noAsync();
        idxb.indexRule("nt:base").property("foo", null).function(
                "lower(coalesce([jcr:content/foo2], coalesce([jcr:content/foo], localname())))"
        );

        Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
        idxb.build(idx);
        root.commit();

        Tree rootTree = root.getTree("/");
        rootTree.addChild("a").addChild("jcr:content").setProperty("foo2", "BAR");
        rootTree.addChild("b").addChild("jcr:content").setProperty("foo", "bAr");
        Tree child = rootTree.addChild("c").addChild("jcr:content");
        child.setProperty("foo", "bar");
        child.setProperty("foo2", "bar1");
        rootTree.addChild("bar");

        root.commit();

        assertPlanAndQuery(
                "select * from [nt:base] where lower(coalesce([jcr:content/foo2], coalesce([jcr:content/foo], localname()))) = 'bar'",
                "lucene:test1(/oak:index/test1)", asList("/a", "/b", "/bar"));
    }

    @Test
    public void coalesceOrdering() throws Exception{
        IndexDefinitionBuilder idxb = new LuceneIndexDefinitionBuilder().noAsync();
        idxb.indexRule("nt:base").property("foo", null).function(
                "coalesce([jcr:content/foo2], [jcr:content/foo])"
        ).ordered();

        Tree idx = root.getTree("/").getChild("oak:index").addChild("test1");
        idxb.build(idx);
        root.commit();

        Tree rootTree = root.getTree("/");
        rootTree.addChild("a").addChild("jcr:content").setProperty("foo2", "a");
        rootTree.addChild("b").addChild("jcr:content").setProperty("foo", "b");
        Tree child = rootTree.addChild("c").addChild("jcr:content");
        child.setProperty("foo", "c");
        child.setProperty("foo2", "a1");

        root.commit();

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by coalesce([jcr:content/foo2], [jcr:content/foo])",
                "lucene:test1(/oak:index/test1)", asList("/a", "/c", "/b"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by coalesce([jcr:content/foo2], [jcr:content/foo]) DESC",
                "lucene:test1(/oak:index/test1)", asList("/b", "/c", "/a"));
    }

    /*
    <OAK-8166>
    Given an index def with 2 orderable property definitions(Relative) for same property - one with function and one without
    Indexer should not fail to index the nodes covered by this index
     */
    @Test
    public void sameOrderableRelativePropertyWithAndWithoutFunction() throws Exception {

        LogCustomizer customLogs = LogCustomizer.forLogger(LuceneIndexEditor.class.getName()).enable(Level.WARN).create();
        // Create nodes that will be served by the index definition that follows
        Tree test = root.getTree("/").addChild("test");
        test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree a = test.addChild("jcr:content");
        a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

        Tree b = a.addChild("n");

        b.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
        b.setProperty("foo", "bar");

        root.commit();

        // Index def with same property - ordered - one with function and one without
        Tree luceneIndex = createIndex("upper", Collections.<String>emptySet());
        Tree nonFunc = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:unstructured")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("foo");
        nonFunc.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        nonFunc.setProperty("name", "jcr:content/n/foo");

        Tree func = luceneIndex.getChild(FulltextIndexConstants.INDEX_RULES)
                .getChild("nt:unstructured")
                .getChild(FulltextIndexConstants.PROP_NODE)
                .addChild("testOak");
        func.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(jcr:content/n/@foo)");

        // Now do some change in the node that are covered by above index definition
        try {
            customLogs.starting();
            root.getTree("/").getChild("test").getChild("jcr:content").getChild("n").setProperty("foo","bar2");
            root.commit();
            Assert.assertFalse(customLogs.getLogs().contains("Failed to index the node [/test]"));
            Assert.assertTrue(customLogs.getLogs().size() == 0);
        } finally {
            customLogs.finished();
        }

    }

    /*
    Given an index def with 2 orderable property definitions(Relative) for same property - one with function and one without
    Order by should give correct results
     */
    @Test
    public void sameOrderableRelPropWithAndWithoutFunc_checkOrdering() throws Exception {

        // Index def with same property - ordered - one with function and one without
        Tree luceneIndex = createIndex("upper", Collections.<String>emptySet());
        Tree nonFunc = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("foo");
        nonFunc.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX,true);
        nonFunc.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        nonFunc.setProperty("name", "jcr:content/n/foo");

        Tree func = luceneIndex.getChild(FulltextIndexConstants.INDEX_RULES)
                .getChild("nt:base")
                .getChild(FulltextIndexConstants.PROP_NODE)
                .addChild("testOak");
        func.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(jcr:content/n/@foo)");

        root.commit();

        int i = 1;
        // Create nodes that will be served by the index definition that follows
        for (String node : asList("a", "c", "b","e","d")) {

            Tree test = root.getTree("/").addChild(node);
            test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

            Tree a = test.addChild("jcr:content");
            a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

            Tree b = a.addChild("n");

            b.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            b.setProperty("foo", "bar"+i);
            i++;
        }

        root.commit();

        // Check ordering works for func and non func properties
        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by upper([jcr:content/n/foo])",
                "lucene:upper(/oak:index/upper)", asList("/a","/c","/b","/e","/d"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by [jcr:content/n/foo]",
                "lucene:upper(/oak:index/upper)", asList("/a","/c","/b","/e","/d"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by upper([jcr:content/n/foo]) DESC",
                "lucene:upper(/oak:index/upper)", asList("/d","/e","/b","/c","/a"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by [jcr:content/n/foo] DESC",
                "lucene:upper(/oak:index/upper)", asList("/d","/e","/b","/c","/a"));

        // Now we change the value of foo on already indexed nodes and see if changes get indexed properly.

        i = 5;
        for (String node : asList("a", "c", "b","e","d")) {

            Tree test = root.getTree("/").getChild(node).getChild("jcr:content").getChild("n");

            test.setProperty("foo", "bar" + i);
            i--;
        }
        root.commit();

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by upper([jcr:content/n/foo])",
                "lucene:upper(/oak:index/upper)", asList("/d","/e","/b","/c","/a"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by [jcr:content/n/foo]",
                "lucene:upper(/oak:index/upper)", asList("/d","/e","/b","/c","/a"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by upper([jcr:content/n/foo]) DESC",
                "lucene:upper(/oak:index/upper)", asList("/a","/c","/b","/e","/d"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by [jcr:content/n/foo] DESC",
                "lucene:upper(/oak:index/upper)", asList("/a","/c","/b","/e","/d"));

    }

    @Test
    public void duplicateFunctionInIndex() throws Exception {
        // Index def with same property - ordered - one with function and one without
        Tree luceneIndex = createIndex("upper", Collections.<String>emptySet());
        Tree prop = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE);
        Tree upper1 = prop.addChild("upper1");
        upper1.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        upper1.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(jcr:content/n/@foo)");
        Tree upper2 = prop.addChild("upper2");
        upper2.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        upper2.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(jcr:content/n/@foo)");
        Tree upper3 = prop.addChild("upper3");
        upper3.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(jcr:content/n/@foo)");
        Tree upper4 = prop.addChild("upper4");
        upper4.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(jcr:content/n/@foo)");

        root.commit();

        int i = 1;
        // Create nodes that will be served by the index definition that follows
        for (String node : asList("a", "c", "b", "e", "d")) {

            Tree test = root.getTree("/").addChild(node);
            test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

            Tree a = test.addChild("jcr:content");
            a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

            Tree b = a.addChild("n");

            b.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            b.setProperty("foo", "bar"+i);
            i++;
        }

        root.commit();

        // Check ordering works for func and non func properties
        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by upper([jcr:content/n/foo])",
                "lucene:upper(/oak:index/upper)", asList("/a","/c","/b","/e","/d"));

        assertOrderedPlanAndQuery(
                "select * from [nt:base] order by upper([jcr:content/n/foo]) DESC",
                "lucene:upper(/oak:index/upper)", asList("/d","/e","/b","/c","/a"));

    }

    /*
    Given an index def with 2 orderable property definitions(non-relative) for same property - one with function and one without
    Indexer should index any changes properly and ordering should work as expected.
     */
    @Test
    public void sameOrderablePropertyWithandWithoutFunction() throws Exception {
        LogCustomizer customLogs = LogCustomizer.forLogger(LuceneIndexEditor.class.getName()).enable(Level.WARN).create();
        // Create nodes that will be served by the index definition that follows
        int i = 1;
        // Create nodes that will be served by the index definition that follows
        for (String node : asList("a", "c", "b","e","d")) {

            Tree test = root.getTree("/").addChild(node);
            test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
            test.setProperty("foo", "bar"+i);
            i++;
        }

        root.commit();

        // Index def with same property - ordered - one with function and one without
        Tree luceneIndex = createIndex("upper", Collections.<String>emptySet());
        Tree nonFunc = luceneIndex.addChild(FulltextIndexConstants.INDEX_RULES)
                .addChild("nt:base")
                .addChild(FulltextIndexConstants.PROP_NODE)
                .addChild("foo");
        nonFunc.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        nonFunc.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX,true);
        nonFunc.setProperty("name", "foo");

        Tree func = luceneIndex.getChild(FulltextIndexConstants.INDEX_RULES)
                .getChild("nt:base")
                .getChild(FulltextIndexConstants.PROP_NODE)
                .addChild("testOak");
        func.setProperty(FulltextIndexConstants.PROP_ORDERED,true);
        func.setProperty(FulltextIndexConstants.PROP_FUNCTION, "fn:upper-case(@foo)");

        // Now do some change in the node that are covered by above index definition
        try {
            customLogs.starting();
             i = 5;
            for (String node : asList("a", "c", "b","e","d")) {

                Tree test = root.getTree("/").addChild(node);
                test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);

                test.setProperty("foo", "bar"+i);
                i--;
            }

            root.commit();
            Assert.assertFalse(customLogs.getLogs().contains("Failed to index the node [/test]"));
            Assert.assertTrue(customLogs.getLogs().size() == 0);

            assertOrderedPlanAndQuery(
                    "select * from [nt:base] order by upper([foo])",
                    "lucene:upper(/oak:index/upper)", asList("/d","/e","/b","/c","/a"));

            assertOrderedPlanAndQuery(
                    "select * from [nt:base] order by [foo]",
                    "lucene:upper(/oak:index/upper)", asList("/d","/e","/b","/c","/a"));

            assertOrderedPlanAndQuery(
                    "select * from [nt:base] order by upper([foo]) DESC",
                    "lucene:upper(/oak:index/upper)", asList("/a","/c","/b","/e","/d"));

            assertOrderedPlanAndQuery(
                    "select * from [nt:base] order by [foo] DESC",
                    "lucene:upper(/oak:index/upper)", asList("/a","/c","/b","/e","/d"));

        } finally {
            customLogs.finished();
        }

    }

    protected String explain(String query){
        String explain = "explain " + query;
        return executeQuery(explain, "JCR-SQL2").get(0);
    }

    protected String explainXpath(String query) throws ParseException {
        String explain = "explain " + query;
        Result result = executeQuery(explain, "xpath", NO_BINDINGS);
        ResultRow row = Iterables.getOnlyElement(result.getRows());
        String plan = row.getValue("plan").getValue(Type.STRING);
        return plan;
    }

    private void assertOrderedPlanAndQuery(String query, String planExpectation, List<String> paths) {
        List<String> result = assertPlanAndQuery(query, planExpectation, paths);
        assertEquals("Ordering doesn't match", paths, result);
    }

    private List<String> assertPlanAndQuery(String query, String planExpectation, List<String> paths){
        assertThat(explain(query), containsString(planExpectation));
        return assertQuery(query, paths);
    }

    protected Tree createIndex(String name, Set<String> propNames) {
        Tree index = root.getTree("/");
        return createIndex(index, name, propNames);
    }

    static Tree createIndex(Tree index, String name, Set<String> propNames) {
        Tree def = index.addChild(INDEX_DEFINITIONS_NAME).addChild(name);
        def.setProperty(JcrConstants.JCR_PRIMARYTYPE,
                INDEX_DEFINITIONS_NODE_TYPE, Type.NAME);
        def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE);
        def.setProperty(REINDEX_PROPERTY_NAME, true);
        def.setProperty(FulltextIndexConstants.FULL_TEXT_ENABLED, false);
        def.setProperty(PropertyStates.createProperty(FulltextIndexConstants.INCLUDE_PROPERTY_NAMES, propNames, Type.STRINGS));
        def.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true);
        return index.getChild(INDEX_DEFINITIONS_NAME).getChild(name);
    }

}


text:
package com.meetisan.meetisan.view.tags;

import java.io.File;
import java.util.Map;
import java.util.TreeMap;

import android.app.Activity;
import android.app.AlertDialog;
import android.content.DialogInterface;
import android.content.Intent;
import android.graphics.Bitmap;
import android.net.Uri;
import android.os.Bundle;
import android.os.Environment;
import android.provider.MediaStore;
import android.text.TextUtils;
import android.view.View;
import android.view.View.OnClickListener;
import android.widget.Button;
import android.widget.ImageButton;
import android.widget.ImageView;
import android.widget.RelativeLayout;
import android.widget.RelativeLayout.LayoutParams;
import android.widget.TextView;
import android.widget.Toast;

import com.meetisan.meetisan.R;
import com.meetisan.meetisan.database.UserInfoKeeper;
import com.meetisan.meetisan.utils.DialogUtils;
import com.meetisan.meetisan.utils.DialogUtils.OnDialogClickListener;
import com.meetisan.meetisan.utils.HttpRequest;
import com.meetisan.meetisan.utils.HttpRequest.OnHttpRequestListener;
import com.meetisan.meetisan.utils.ServerKeys;
import com.meetisan.meetisan.utils.ToastHelper;
import com.meetisan.meetisan.utils.Tools;
import com.meetisan.meetisan.utils.Util;
import com.meetisan.meetisan.widget.ClearEditText;
import com.meetisan.meetisan.widget.CustomizedProgressDialog;

public class AddTagMomentsActivity extends Activity implements OnClickListener {

	private long mTagID = -1, mUserID = -1;
	private ClearEditText mTitleTxt;
	private ImageView mMomentView;
	private RelativeLayout mBottomLayout;

	private String[] items = null;
	/* 请求码 */
	private static final int IMAGE_REQUEST_CODE = 0;
	private static final int CAMERA_REQUEST_CODE = 1;
	private static final int RESULT_REQUEST_CODE = 2;
	/* 头像名称 */
	private static final String IMAGE_FILE_NAME = "com.cn.finessBand/faceImage.jpg";
	private String mBase64Str = null;

	public void onCreate(Bundle savedInstanceState) {
		super.onCreate(savedInstanceState);

		setContentView(R.layout.activity_add_tag_moments);

		Bundle bundle = new Bundle();
		bundle = this.getIntent().getExtras();
		if (bundle != null) {
			mTagID = bundle.getLong("TagID");
			mUserID = UserInfoKeeper.readUserInfo(this, UserInfoKeeper.KEY_USER_ID, -1L);
		}

		if (mTagID < 0 || mUserID < 0) {
			ToastHelper.showToast(R.string.app_occurred_exception);
			this.finish();
		}

		initView();
		showPickImageDialog(true);
	}

	private void initView() {
		ImageButton imageButton = (ImageButton) findViewById(R.id.btn_title_left);
		imageButton.setOnClickListener(this);
		imageButton.setVisibility(View.VISIBLE);
		Button mRightBtn = (Button) findViewById(R.id.btn_title_right);
		mRightBtn.setText(R.string.send);
		mRightBtn.setVisibility(View.VISIBLE);
		mRightBtn.setOnClickListener(this);

		((TextView) findViewById(R.id.tv_title_text)).setText(R.string.tag_moments);
		mTitleTxt = (ClearEditText) findViewById(R.id.txt_tag_title);
		mMomentView = (ImageView) findViewById(R.id.iv_moment_add);
		mBottomLayout = (RelativeLayout) findViewById(R.id.layout_bottom);
		mBottomLayout.setOnClickListener(this);

		items = new String[] { getString(R.string.select_picture), getString(R.string.take_picture) };
	}

	private void setMomentPreview(Bitmap photo) {
		int size = (Util.getWindowsSize(this, true) - 40) / 3;
		LayoutParams params = new LayoutParams(size, size);
		mMomentView.setLayoutParams(params);
		mMomentView.setImageBitmap(photo);
	}

	@Override
	public void onClick(View v) {
		switch (v.getId()) {
		case R.id.btn_title_left:
			confirmExitDialog();
			break;
		case R.id.btn_title_right:
			attempUploadPreview();
			break;
		case R.id.layout_bottom:
			showPickImageDialog(false);
			break;
		default:
			break;
		}
	}

	private void attempUploadPreview() {
		if (mBase64Str == null || TextUtils.isEmpty(mBase64Str)) {
			ToastHelper.showToast(R.string.error_empty_moment_pic, Toast.LENGTH_LONG);
			return;
		}

		String title = mTitleTxt.getText().toString();
		if (title == null || TextUtils.isEmpty(title)) {
			ToastHelper.showToast(R.string.error_empty_moment_title, Toast.LENGTH_LONG);
			return;
		}

		uploadTagMomentToServer(title);
	}

	/**
	 * 裁剪图片方法实现
	 *
	 * @param uri
	 */
	public void startPhotoZoom(Uri uri) {
		Intent intent = new Intent("com.android.camera.action.CROP");
		intent.setDataAndType(uri, "image/*");
		// 设置裁剪
		intent.putExtra("crop", "true");
		// aspectX aspectY 是宽高的比例
		intent.putExtra("aspectX", 1);
		intent.putExtra("aspectY", 1);
		// outputX outputY 是裁剪图片宽高
		intent.putExtra("outputX", 128);
		intent.putExtra("outputY", 128);
		intent.putExtra("return-data", true);
		startActivityForResult(intent, RESULT_REQUEST_CODE);
	}

	/**
	 * 保存裁剪之后的图片数据
	 *
	 * @param picdata
	 */
	private void getImageToView(Intent data) {
		Bundle extras = data.getExtras();
		if (extras != null) {
			Bitmap photo = extras.getParcelable("data");
			mBase64Str = Tools.bitmapToBase64(photo);
			setMomentPreview(photo);
		}
	}

	/**
	 * 显示选择对话框
	 */
	private void showPickImageDialog(final boolean isFinish) {
		Intent intentFromGallery = new Intent();
		intentFromGallery.setType("image/*"); // 设置文件类型
		intentFromGallery.setAction(Intent.ACTION_GET_CONTENT);
		startActivityForResult(intentFromGallery, IMAGE_REQUEST_CODE);

		// new AlertDialog.Builder(this).setTitle(R.string.profile_picture)
		// .setItems(items, new DialogInterface.OnClickListener() {
		//
		// @Override
		// public void onClick(DialogInterface dialog, int which) {
		// switch (which) {
		// case 0:
		// Intent intentFromGallery = new Intent();
		// intentFromGallery.setType("image/*"); // 设置文件类型
		// intentFromGallery.setAction(Intent.ACTION_GET_CONTENT);
		// startActivityForResult(intentFromGallery, IMAGE_REQUEST_CODE);
		// break;
		// case 1:
		// Intent intentFromCapture = new
		// Intent(MediaStore.ACTION_IMAGE_CAPTURE);
		// // 判断存储卡是否可以用,可用进行存储
		// if (Tools.hasSdcard()) {
		// intentFromCapture.putExtra(MediaStore.EXTRA_OUTPUT, Uri.fromFile(new
		// File(Environment
		// .getExternalStorageDirectory(), IMAGE_FILE_NAME)));
		// }
		// startActivityForResult(intentFromCapture, CAMERA_REQUEST_CODE);
		// break;
		// }
		// }
		// }).setNegativeButton(R.string.cancel, new
		// DialogInterface.OnClickListener() {
		//
		// @Override
		// public void onClick(DialogInterface dialog, int which) {
		// dialog.dismiss();
		// if (isFinish) {
		// AddTagMomentsActivity.this.finish();
		// }
		// }
		//
		// }).show();

	}

	@Override
	public void onActivityResult(int requestCode, int resultCode, Intent data) {
		if (resultCode != Activity.RESULT_CANCELED) {
			switch (requestCode) {
			case IMAGE_REQUEST_CODE:
				startPhotoZoom(data.getData());
				break;
			case CAMERA_REQUEST_CODE:
				if (Tools.hasSdcard()) {
					File tempFile = new File(Environment.getExternalStorageDirectory() + "/" + IMAGE_FILE_NAME);
					startPhotoZoom(Uri.fromFile(tempFile));
				} else {
					Toast.makeText(this, R.string.can_not_find_sd_card, Toast.LENGTH_LONG).show();
				}
				break;
			case RESULT_REQUEST_CODE:
				if (data != null) {
					getImageToView(data);
				}
				break;
			}
		}
		super.onActivityResult(requestCode, resultCode, data);
	}

	private CustomizedProgressDialog mProgressDialog = null;

	private void uploadTagMomentToServer(String title) {

		if (title == null || mBase64Str == null) {
			return;
		}

		HttpRequest request = new HttpRequest();

		if (mProgressDialog == null) {
			mProgressDialog = new CustomizedProgressDialog(this, R.string.uploading_image);
		} else {
			if (mProgressDialog.isShowing()) {
				mProgressDialog.dismiss();
			}
		}

		request.setOnHttpRequestListener(new OnHttpRequestListener() {

			@Override
			public void onSuccess(String url, String result) {
				mProgressDialog.dismiss();
				ToastHelper.showToast(R.string.upload_success, Toast.LENGTH_LONG);
				AddTagMomentsActivity.this.finish();
			}

			@Override
			public void onFailure(String url, int errorNo, String errorMsg) {
				mProgressDialog.dismiss();
				ToastHelper.showToast(errorMsg, Toast.LENGTH_LONG);
			}
		});

		Map<String, String> data = new TreeMap<String, String>();
		data.put(ServerKeys.KEY_TAG_ID, String.valueOf(mTagID));
		data.put(ServerKeys.KEY_IMAGE, mBase64Str);
		data.put(ServerKeys.KEY_TITLE, title);
		data.put(ServerKeys.KEY_USER_ID, String.valueOf(mUserID));
		request.post(ServerKeys.FULL_URL_ADD_TAG_MOMENT, data);

		mProgressDialog.show();
	}

	private void confirmExitDialog() {
		DialogUtils.showDialog(this, R.string.warning, R.string.give_up_add_tag_tips, R.string.sure, R.string.cancel,
				new OnDialogClickListener() {

					@Override
					public void onClick(boolean isPositiveBtn) {
						if (isPositiveBtn) {
							AddTagMomentsActivity.this.finish();
						}
					}
				});
	}

	@Override
	public void onBackPressed() {
		if (mBase64Str != null) {
			confirmExitDialog();
		} else {
			super.onBackPressed();
		}
	}
}


text:
package notifications

import (
	"bytes"
	"testing"
	"time"

	context "github.com/djbarber/ipfs-hack/Godeps/_workspace/src/golang.org/x/net/context"
	blocks "github.com/djbarber/ipfs-hack/blocks"
	blocksutil "github.com/djbarber/ipfs-hack/blocks/blocksutil"
	key "github.com/djbarber/ipfs-hack/blocks/key"
)

func TestDuplicates(t *testing.T) {
	b1 := blocks.NewBlock([]byte("1"))
	b2 := blocks.NewBlock([]byte("2"))

	n := New()
	defer n.Shutdown()
	ch := n.Subscribe(context.Background(), b1.Key(), b2.Key())

	n.Publish(b1)
	blockRecvd, ok := <-ch
	if !ok {
		t.Fail()
	}
	assertBlocksEqual(t, b1, blockRecvd)

	n.Publish(b1) // ignored duplicate

	n.Publish(b2)
	blockRecvd, ok = <-ch
	if !ok {
		t.Fail()
	}
	assertBlocksEqual(t, b2, blockRecvd)
}

func TestPublishSubscribe(t *testing.T) {
	blockSent := blocks.NewBlock([]byte("Greetings from The Interval"))

	n := New()
	defer n.Shutdown()
	ch := n.Subscribe(context.Background(), blockSent.Key())

	n.Publish(blockSent)
	blockRecvd, ok := <-ch
	if !ok {
		t.Fail()
	}

	assertBlocksEqual(t, blockRecvd, blockSent)

}

func TestSubscribeMany(t *testing.T) {
	e1 := blocks.NewBlock([]byte("1"))
	e2 := blocks.NewBlock([]byte("2"))

	n := New()
	defer n.Shutdown()
	ch := n.Subscribe(context.Background(), e1.Key(), e2.Key())

	n.Publish(e1)
	r1, ok := <-ch
	if !ok {
		t.Fatal("didn't receive first expected block")
	}
	assertBlocksEqual(t, e1, r1)

	n.Publish(e2)
	r2, ok := <-ch
	if !ok {
		t.Fatal("didn't receive second expected block")
	}
	assertBlocksEqual(t, e2, r2)
}

// TestDuplicateSubscribe tests a scenario where a given block
// would be requested twice at the same time.
func TestDuplicateSubscribe(t *testing.T) {
	e1 := blocks.NewBlock([]byte("1"))

	n := New()
	defer n.Shutdown()
	ch1 := n.Subscribe(context.Background(), e1.Key())
	ch2 := n.Subscribe(context.Background(), e1.Key())

	n.Publish(e1)
	r1, ok := <-ch1
	if !ok {
		t.Fatal("didn't receive first expected block")
	}
	assertBlocksEqual(t, e1, r1)

	r2, ok := <-ch2
	if !ok {
		t.Fatal("didn't receive second expected block")
	}
	assertBlocksEqual(t, e1, r2)
}

func TestSubscribeIsANoopWhenCalledWithNoKeys(t *testing.T) {
	n := New()
	defer n.Shutdown()
	ch := n.Subscribe(context.Background()) // no keys provided
	if _, ok := <-ch; ok {
		t.Fatal("should be closed if no keys provided")
	}
}

func TestCarryOnWhenDeadlineExpires(t *testing.T) {

	impossibleDeadline := time.Nanosecond
	fastExpiringCtx, cancel := context.WithTimeout(context.Background(), impossibleDeadline)
	defer cancel()

	n := New()
	defer n.Shutdown()
	block := blocks.NewBlock([]byte("A Missed Connection"))
	blockChannel := n.Subscribe(fastExpiringCtx, block.Key())

	assertBlockChannelNil(t, blockChannel)
}

func TestDoesNotDeadLockIfContextCancelledBeforePublish(t *testing.T) {

	g := blocksutil.NewBlockGenerator()
	ctx, cancel := context.WithCancel(context.Background())
	n := New()
	defer n.Shutdown()

	t.Log("generate a large number of blocks. exceed default buffer")
	bs := g.Blocks(1000)
	ks := func() []key.Key {
		var keys []key.Key
		for _, b := range bs {
			keys = append(keys, b.Key())
		}
		return keys
	}()

	_ = n.Subscribe(ctx, ks...) // ignore received channel

	t.Log("cancel context before any blocks published")
	cancel()
	for _, b := range bs {
		n.Publish(b)
	}

	t.Log("publishing the large number of blocks to the ignored channel must not deadlock")
}

func assertBlockChannelNil(t *testing.T, blockChannel <-chan *blocks.Block) {
	_, ok := <-blockChannel
	if ok {
		t.Fail()
	}
}

func assertBlocksEqual(t *testing.T, a, b *blocks.Block) {
	if !bytes.Equal(a.Data, b.Data) {
		t.Fatal("blocks aren't equal")
	}
	if a.Key() != b.Key() {
		t.Fatal("block keys aren't equal")
	}
}


text:
layout: post
title:  "Code Walkthrough: Tablib, a Python Module for Tabular Datasets"
date:   2018-10-08 21:00:00
comments: true
categories: blog
image: /assets/img/source_code.png
description: Reading Great Code and it's benefits. Code walkthrough of tablib python module by Nipun Sadvilkar
---

Motivation

Oftentimes, I like to dive into open source projects to learn best practices and design patterns programming pundits use to do things correctly and optimally. In addition, [Peter Norvig](https://en.wikipedia.org/wiki/Peter_Norvig) has also said in his famous blog post [Teach Yourself Programming in Ten Years](http://norvig.com/21-days.html)

> *Talk with other programmers; read other programs. This is more important than any book or training course.*

I am big advocate of it. This blog post is to emphasize - how reading open source code helps you identify and understand efficient patterns and coding constructs.

Tablib

I admire [Kenneth Reitz](https://github.com/kennethreitz) very much. Do read and follow his [The Hitchhiker's Guide to Python!](https://docs.python-guide.org) to be a a great Python programmer. Lesson from this book - [Reading Great Code](https://docs.python-guide.org/writing/reading/?highlight=tablib#reading-great-code) - is the main reason why I decided to give a go at reading source code of [Tablib](https://github.com/kennethreitz/tablib). Reading source code is initially daunting because of certain constructs which are obscure or you may not be familiar with it, and which is natural. Despite such hurdles, if you keep con oncentrating you will find lot of "Aha!" moments by identifying useful patterns. Here is my experience, I came across a very simple yet useful code snippet which is very important and widely used task in data cleaning i.e., removing duplicates.

[Source code: tablib removing_duplicates menthod:](http://docs.python-tablib.org/en/master/_modules/tablib/core/#Dataset.remove_duplicates)

```python
def remove_duplicates(self):
    """Removes all duplicate rows from the :class:`Dataset` object
    while maintaining the original order."""
    seen = set()
    self._data[:] = [row for row in self._data if not (tuple(row) in seen or seen.add(tuple(row)))]
```

Check the `if ` statement followed by [_generator expression_](https://dbader.org/blog/python-generator-expressions). If you look closely inside generator expression the technique used to check for duplicate rows is called [_short circuit technique_](https://www.geeksforgeeks.org/short-circuiting-techniques-python/) implemented in python.

[Short circuit explained by official docs](https://docs.python.org/2/library/stdtypes.html#boolean-operations-and-or-not):

|Operation|Result|Notes|
|---|---|---|
|`x or y` |if x is false, then y, else x| Only evaluates the second argument(`y`) if the first one is `False`.|
|`x and y`|if x is false, then x, else y| Only evaluates the second argument(`y`) if the first one(`x`) is `True`.|
|`not x`|if x is false, then True, else False|`not` has a lower priority than non-Boolean operators|

`remove_duplicates` method uses 1st and 3rd Operation from above table.

Key thing to remember is:

**The evaluation of expression takes place from left to right.**

Explained with toy example:
```python
>>> _data = [[1, 2, 3], [4, 5, 6], [1, 2, 3]]
>>> seen = set()
>>> data_deduplicated = [row for row in _data if not (tuple(row) in seen or seen.add(tuple(row)))]

>>> print(data_deduplicated)
# [[1, 2, 3], [4, 5, 6]]
```

To put it into words, within list comprehension - iterate over data row by row and check if given row is present within `seen` _set_. If it's not present, meaning
```python
tuple(row) in seen
```
evaluates to `False` and as per 1st operation from the table, evaluate second argument which is to add given row in `seen` _set_. Furthermore, `if not ()` condition gets satisfied and given row is added to outer list. Subsequently, if the same row occurs then we know it's already in `seen` _set_ and hence that row will not be added to outer list. In overall, resulting into removing of duplicate rows.

If you are more of a visual learning person, following demonstartion using [Python tutor tool](http://pythontutor.com/) built by an outstanding academic and prolific blogger - [Philip Guo](http://pgbovine.net) - would help*:
> *If below IFrame is not visible then please enable **"load unsecure script"** of your browser. Don't worry! it's saying unsecure because of http protocol used by [Python tutor](http://pythontutor.com/) and not **https**.

I hope by now you have understood [_short circuit technique_](https://www.geeksforgeeks.org/short-circuiting-techniques-python/) and importance of reading open source code. So keep exploring and do share your experience with me. Thank you! :)


text:
import Revisions from '../../lib/collections/revisions/collection'
import { htmlToDraft } from '../draftConvert';
import { convertToRaw } from 'draft-js';
import { markdownToHtmlNoLaTeX, dataToMarkdown } from '../editor/make_editable_callbacks'
import { highlightFromHTML, truncate } from '../../lib/editor/ellipsize';
import { augmentFieldsDict } from '../../lib/utils/schemaUtils'
import { JSDOM } from 'jsdom'
import { sanitize, sanitizeAllowedTags } from '../vulcan-lib/utils';
import htmlToText from 'html-to-text'
import sanitizeHtml, {IFrame} from 'sanitize-html';
import * as _ from 'underscore';

const PLAINTEXT_HTML_TRUNCATION_LENGTH = 4000
const PLAINTEXT_DESCRIPTION_LENGTH = 2000

function domBuilder(html: string) {
  const jsdom = new JSDOM(html)
  const document = jsdom.window.document;
  const bodyEl = document.body; // implicitly created
  return bodyEl
}

export function htmlToDraftServer(html: string): Draft.RawDraftContentState {
  // We have to add this type definition to the global object to allow draft-convert to properly work on the server
  const jsdom = new JSDOM();
  const globalHTMLElement = jsdom.window.HTMLElement;
  (global as any).HTMLElement = globalHTMLElement;
  // And alas, it looks like we have to add this global. This seems quite bad, and I am not fully sure what to do about it.
  (global as any).document = jsdom.window.document

  // On the server have to pass in a JS-DOM implementation to make htmlToDraft work
  //
  // The DefinitelyTyped annotation of htmlToDraft, which comes from convertFromHTML
  // in the draft-convert library, is wrong. This actually takes optional second and
  // third arguments, the second being options, and the third being a DOMBuilder
  // (verified by quick source-dive into draft-convert).
  // @ts-ignore
  const result = htmlToDraft(html, {}, domBuilder)

  // We do however at least remove it right afterwards
  delete (global as any).document
  delete (global as any).HTMLElement

  // convertToRaw wants a Draft.ContentState, but htmlToDraft produced a
  // Draft.Model.ImmutableData.ContentState. AFAICT this is the DefinitelyTyped
  // people not being careful with the const plague, not a real issue.
  // @ts-ignore
  return convertToRaw(result)
}

export function dataToDraftJS(data: any, type: string) {
  if (data===undefined || data===null) return null;

  switch (type) {
    case "draftJS": {
      return data
    }
    case "html": {
      return htmlToDraftServer(data)
    }
    case "ckEditorMarkup": {
      // CK Editor markup is just html with extra tags, so we just remove them and then handle it as html
      return htmlToDraftServer(sanitize(data))
    }
    case "markdown": {
      const html = markdownToHtmlNoLaTeX(data)
      return htmlToDraftServer(html)
    }
    default: {
      throw new Error(`Unrecognized type: ${type}`);
    }
  }
}

augmentFieldsDict(Revisions, {
  markdown: {
    type: String,
    resolveAs: {
      type: 'String',
      resolver: ({originalContents: {data, type}}) => dataToMarkdown(data, type)
    }
  },
  draftJS: {
    type: Object,
    resolveAs: {
      type: 'JSON',
      resolver: ({originalContents: {data, type}}) => dataToDraftJS(data, type)
    }
  },
  ckEditorMarkup: {
    type: String,
    resolveAs: {
      type: 'String',
      resolver: ({originalContents: {data, type}, html}) => (type === 'ckEditorMarkup' ? data : html) // For ckEditorMarkup we just fall back to HTML, since it's a superset of html
    }
  },
  htmlHighlight: {
    type: String,
    resolveAs: {
      type: 'String',
      resolver: ({html}) => highlightFromHTML(html)
    }
  },
  plaintextDescription: {
    type: String,
    resolveAs: {
      type: 'String',
      resolver: ({html}) => {
        if (!html) return
        const truncatedHtml = truncate(sanitize(html), PLAINTEXT_HTML_TRUNCATION_LENGTH)
        return htmlToText
          .fromString(truncatedHtml, {ignoreHref: true, ignoreImage: true})
          .substring(0, PLAINTEXT_DESCRIPTION_LENGTH)
      }
    }
  },
  // Plaintext version, except that specially-formatted blocks like blockquotes are filtered out, for use in highly-abridged displays like SingleLineComment.
  plaintextMainText: {
    type: String,
    resolveAs: {
      type: 'String',
      resolver: ({html}) => {
        const mainTextHtml = sanitizeHtml(
          html, {
            allowedTags: _.without(sanitizeAllowedTags, 'blockquote', 'img'),
            nonTextTags: ['blockquote', 'img', 'style'],

            exclusiveFilter: function(element: IFrame) {
              return (element.attribs?.class === 'spoilers');
            }
          }
        )
        const truncatedHtml = truncate(mainTextHtml, PLAINTEXT_HTML_TRUNCATION_LENGTH)
        return htmlToText
          .fromString(truncatedHtml)
          .substring(0, PLAINTEXT_DESCRIPTION_LENGTH)
      }
    }
  },
})


text:
package com.github.nikdon.google.datastore

import org.scalatest.prop.GeneratorDrivenPropertyChecks
import org.scalatest.{FlatSpec, Matchers}
import com.google.cloud.datastore._
import org.scalacheck.Shapeless._

sealed trait Bla                                                             extends Product with Serializable
final case class Foo(bool: Boolean, str: String, long: Long, double: Double) extends Bla
final case class Bar(int: Int, float: Float, byte: Byte)                     extends Bla
final case class Baz(foo: Foo, Bar: Bar)                                     extends Bla

class DerivedDatastoreFormatTest
    extends FlatSpec
    with GeneratorDrivenPropertyChecks
    with Matchers
    with DerivedDatastoreFormat {

  implicit val datastore: Datastore = DatastoreOptions.getDefaultInstance.getService

  val blaDatastoreFormat: DatastoreFormat[Bla] = implicitly[DatastoreFormat[Bla]]

  val fooDatastoreFormat: DatastoreFormat[Foo] = implicitly[DatastoreFormat[Foo]]
  val barDatastoreFormat: DatastoreFormat[Bar] = implicitly[DatastoreFormat[Bar]]
  val bazDatastoreFormat: DatastoreFormat[Baz] = implicitly[DatastoreFormat[Baz]]

  behavior of "DerivedDatastoreFormat"

  it should "derive formats for Foo" in forAll { foo: Foo =>
    fooDatastoreFormat.read(fooDatastoreFormat.write(foo)).foreach(_ shouldBe foo)
  }

  it should "derive formats for Bar" in forAll { bar: Bar =>
    barDatastoreFormat.read(barDatastoreFormat.write(bar)).foreach(_ shouldBe bar)
  }

  it should "derive formats for Baz" in forAll { baz: Baz =>
    bazDatastoreFormat.read(bazDatastoreFormat.write(baz)).foreach(_ shouldBe baz)
  }

  it should "derive formats for coproducts" in forAll { bla: Bla =>
    blaDatastoreFormat.read(blaDatastoreFormat.write(bla)).foreach(_ shouldBe bla)
  }
}


text:
SCRIPT_DIR=$(dirname $(readlink -f $BASH_SOURCE[0]))

set -x
set -e

cd ${SCRIPT_DIR}
host=${1:-pc596}

Checkout project on remote machine
ssh -A -t "$host" << EOF
set -x
mkdir -p ${SCRIPT_DIR}
cd ${SCRIPT_DIR}
git clone git@github.com:statsbiblioteket/digital-pligtaflevering-aviser-tools.git .
EOF

#sshfs so you can edit the project directly
mkdir -p "$HOME/$host"
mount | grep -q " $HOME/$host " || sshfs "$host":. "$HOME/$host"

JAVA7="env JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/"
JAVA8="env JAVA_HOME=/usr/lib/jvm/zulu-8-amd64"

#Start vagrant
ssh -A -t "$host" <<-EOF
	cd ${SCRIPT_DIR};

	vagrant plugin list | grep vagrant-timezone || vagrant plugin install vagrant-timezone
	vagrant plugin list | grep vagrant-scp || vagrant plugin install vagrant-scp
	vagrant up

	vagrant ssh -- <<-EOV
		set -x
		set -e
		${JAVA8} /vagrant/install_bitrepository.sh
		${JAVA8} /vagrant/run-bitrepositorystub-webserver.sh

		${JAVA7} /vagrant/install_doms.sh
		echo export CATALINA_PID=~/7880-doms/catalina.pid >> ~/7880-doms/tomcat/bin/setenv.sh

		${JAVA8} /vagrant/run-verapdf-rest.sh
	EOV

	vagrant ssh  -- <<-EOV
		${JAVA7} ~/7880-doms/tomcat/bin/startup.sh
		${JAVA7} ~/7880-doms/sboi-summarise/bin/start_resident.sh
		${JAVA8} ~/bitrepository-quickstart/quickstart.sh restart
	EOV

EOF

#    vagrant snapshot save up

#Port forwarding
# http://linuxcommand.org/lc3_man_pages/ssh1.html
FEDORA_PORT=7880
SBOI_PORT=58608
BITREPO_PORT=58709
ACTIVEMQ_PORT=61616
VERAPDF_PORT=8090
POSTGRES_PORT=5432
ZOOKEEPER_PORT=2121
ssh -A -L"$FEDORA_PORT:localhost:$FEDORA_PORT" -L"$SBOI_PORT:localhost:$SBOI_PORT" -L"$BITREPO_PORT:localhost:$BITREPO_PORT" -L"$ACTIVEMQ_PORT:localhost:$ACTIVEMQ_PORT" -L"$VERAPDF_PORT:localhost:$VERAPDF_PORT" "$host"  <<-EOF
set -e
set -x
cd $SCRIPT_DIR;
while true; do
	date;
	vagrant ssh -c '${JAVA7} ~/7880-doms/bin/doms.sh update';
	sleep 45;
done
EOF

#export V=$(VBoxManage list runningvms | cut -d\" -f2 | grep digital-pligtaflevering-aviser-tools_default); VBoxManage controlvm $V poweroff ; sleep 4 ; VBoxManage snapshot $V restore "XX"; VBoxManage startvm $V --type headless


text:
package jp.co.yahoo.dataplatform.mds;

import java.io.IOException;

import org.apache.arrow.vector.ValueVector;

import jp.co.yahoo.dataplatform.mds.spread.expression.IExpressionNode;

public interface IArrowLoader{

  void setNode( final IExpressionNode node );

  boolean hasNext() throws IOException;

  ValueVector next() throws IOException;

  void close() throws IOException;

}