text:
package org.elasticsearch.indices.stats;

import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.cache.query.QueryCacheStats;
import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.MergeSchedulerConfig;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.IndicesRequestCache;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;

import java.io.IOException;
import java.util.EnumSet;
import java.util.Random;

import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;

@ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, randomDynamicTemplates = false)
@SuppressCodecs("*") // requires custom completion format
public class IndexStatsIT extends ESIntegTestCase {
    @Override
    protected Settings nodeSettings(int nodeOrdinal) {
        //Filter/Query cache is cleaned periodically, default is 60s, so make sure it runs often. Thread.sleep for 60s is bad
        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
                .put(IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), "1ms")
                .build();
    }

    @Override
    public Settings indexSettings() {
        return Settings.builder().put(super.indexSettings())
            .put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)
            .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), true)
            .build();
    }

    private Settings.Builder settingsBuilder() {
        return Settings.builder().put(indexSettings());
    }

    public void testFieldDataStats() {
        assertAcked(client().admin().indices().prepareCreate("test")
                .setSettings(settingsBuilder().put("index.number_of_shards", 2))
                .addMapping("type", "field", "type=text,fielddata=true",
                        "field2", "type=text,fielddata=true").get());
        ensureGreen();
        client().prepareIndex("test", "type", "1").setSource("field", "value1", "field2", "value1").execute().actionGet();
        client().prepareIndex("test", "type", "2").setSource("field", "value2", "field2", "value2").execute().actionGet();
        client().admin().indices().prepareRefresh().execute().actionGet();

        NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
        IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
        assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));

        // sort to load it to field data...
        client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
        client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();

        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
        indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
        assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));

        // sort to load it to field data...
        client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
        client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();

        // now check the per field stats
        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.FieldData, true).fieldDataFields("*")).execute().actionGet();
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), greaterThan(0L));
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), lessThan(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes()));

        indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).setFieldDataFields("*").execute().actionGet();
        assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
        assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), greaterThan(0L));
        assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes()));

        client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
        indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
        assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));

    }

    public void testClearAllCaches() throws Exception {
        assertAcked(client().admin().indices().prepareCreate("test")
                .setSettings(settingsBuilder().put("index.number_of_replicas", 0).put("index.number_of_shards", 2))
                .addMapping("type", "field", "type=text,fielddata=true").get());
        ensureGreen();
        client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
        client().prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
        client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
        client().admin().indices().prepareRefresh().execute().actionGet();

        NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
                .execute().actionGet();
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
        assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));

        IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test")
                .clear().setFieldData(true).setQueryCache(true)
                .execute().actionGet();
        assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
        assertThat(indicesStats.getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0L));

        // sort to load it to field data and filter to load filter cache
        client().prepareSearch()
                .setPostFilter(QueryBuilders.termQuery("field", "value1"))
                .addSort("field", SortOrder.ASC)
                .execute().actionGet();
        client().prepareSearch()
                .setPostFilter(QueryBuilders.termQuery("field", "value2"))
                .addSort("field", SortOrder.ASC)
                .execute().actionGet();

        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
                .execute().actionGet();
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
        assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), greaterThan(0L));

        indicesStats = client().admin().indices().prepareStats("test")
                .clear().setFieldData(true).setQueryCache(true)
                .execute().actionGet();
        assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
        assertThat(indicesStats.getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0L));

        client().admin().indices().prepareClearCache().execute().actionGet();
        Thread.sleep(100); // Make sure the filter cache entries have been removed...
        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
                .execute().actionGet();
        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
        assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));

        indicesStats = client().admin().indices().prepareStats("test")
                .clear().setFieldData(true).setQueryCache(true)
                .execute().actionGet();
        assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
        assertThat(indicesStats.getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
    }

    public void testQueryCache() throws Exception {
        assertAcked(client().admin().indices().prepareCreate("idx").setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true).get());
        ensureGreen();

        // index docs until we have at least one doc on each shard, otherwise, our tests will not work
        // since refresh will not refresh anything on a shard that has 0 docs and its search response get cached
        int pageDocs = randomIntBetween(2, 100);
        int numDocs = 0;
        int counter = 0;
        while (true) {
            IndexRequestBuilder[] builders = new IndexRequestBuilder[pageDocs];
            for (int i = 0; i < pageDocs; ++i) {
                builders[i] = client().prepareIndex("idx", "type", Integer.toString(counter++)).setSource(jsonBuilder()
                        .startObject()
                        .field("common", "field")
                        .field("str_value", "s" + i)
                        .endObject());
            }
            indexRandom(true, builders);
            numDocs += pageDocs;

            boolean allHaveDocs = true;
            for (ShardStats stats : client().admin().indices().prepareStats("idx").setDocs(true).get().getShards()) {
                if (stats.getStats().getDocs().getCount() == 0) {
                    allHaveDocs = false;
                    break;
                }
            }

            if (allHaveDocs) {
                break;
            }
        }

        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L));
        for (int i = 0; i < 10; i++) {
            assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
            assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));
        }
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), greaterThan(0L));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), greaterThan(0L));

        // index the data again...
        IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
        for (int i = 0; i < numDocs; ++i) {
            builders[i] = client().prepareIndex("idx", "type", Integer.toString(i)).setSource(jsonBuilder()
                    .startObject()
                    .field("common", "field")
                    .field("str_value", "s" + i)
                    .endObject());
        }
        indexRandom(true, builders);
        refresh();
        assertBusy(new Runnable() {
            @Override
            public void run() {
                assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));
            }
        });

        for (int i = 0; i < 10; i++) {
            assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
            assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));
        }

        client().admin().indices().prepareClearCache().setRequestCache(true).get(); // clean the cache
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));

        // test explicit request parameter

        assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(false).get().getHits().getTotalHits(), equalTo((long) numDocs));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));

        assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));

        // set the index level setting to false, and see that the reverse works

        client().admin().indices().prepareClearCache().setRequestCache(true).get(); // clean the cache
        assertAcked(client().admin().indices().prepareUpdateSettings("idx").setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), false)));

        assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get().getHits().getTotalHits(), equalTo((long) numDocs));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), equalTo(0L));

        assertThat(client().prepareSearch("idx").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).setRequestCache(true).get().getHits().getTotalHits(), equalTo((long) numDocs));
        assertThat(client().admin().indices().prepareStats("idx").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(), greaterThan(0L));
    }

    public void testNonThrottleStats() throws Exception {
        assertAcked(prepareCreate("test")
                .setSettings(settingsBuilder()
                                .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
                                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
                                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
                                .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2")
                                .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2")
                                .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
                                .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "10000")
                ));
        ensureGreen();
        long termUpto = 0;
        IndicesStatsResponse stats;
        // Provoke slowish merging by making many unique terms:
        for(int i=0; i<100; i++) {
            StringBuilder sb = new StringBuilder();
            for(int j=0; j<100; j++) {
                sb.append(' ');
                sb.append(termUpto++);
                sb.append(" some random text that keeps repeating over and over again hambone");
            }
            client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get();
        }
        refresh();
        stats = client().admin().indices().prepareStats().execute().actionGet();
        //nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get();

        stats = client().admin().indices().prepareStats().execute().actionGet();
        assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTime().millis(), equalTo(0L));
    }

    public void testThrottleStats() throws Exception {
        assertAcked(prepareCreate("test")
                    .setSettings(settingsBuilder()
                                 .put(IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING.getKey(), "merge")
                                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
                                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
                                 .put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING.getKey(), "2")
                                 .put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), "2")
                                 .put(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING.getKey(), "1")
                                 .put(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING.getKey(), "1")
                                 .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC.name())
                                 ));
        ensureGreen();
        long termUpto = 0;
        IndicesStatsResponse stats;
        // make sure we see throttling kicking in:
        boolean done = false;
        long start = System.currentTimeMillis();
        while (!done) {
            for(int i=0; i<100; i++) {
                // Provoke slowish merging by making many unique terms:
                StringBuilder sb = new StringBuilder();
                for(int j=0; j<100; j++) {
                    sb.append(' ');
                    sb.append(termUpto++);
                }
                client().prepareIndex("test", "type", ""+termUpto).setSource("field" + (i%10), sb.toString()).get();
                if (i % 2 == 0) {
                    refresh();
                }
            }
            refresh();
            stats = client().admin().indices().prepareStats().execute().actionGet();
            //nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get();
            done = stats.getPrimaries().getIndexing().getTotal().getThrottleTime().millis() > 0;
            if (System.currentTimeMillis() - start > 300*1000) { //Wait 5 minutes for throttling to kick in
                fail("index throttling didn't kick in after 5 minutes of intense merging");
            }
        }

        // Optimize & flush and wait; else we sometimes get a "Delete Index failed - not acked"
        // when ESIntegTestCase.after tries to remove indices created by the test:
        logger.info("test: now optimize");
        client().admin().indices().prepareForceMerge("test").get();
        flush();
        logger.info("test: test done");
    }

    public void testSimpleStats() throws Exception {
        createIndex("test1", "test2");
        ensureGreen();

        client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        refresh();

        NumShards test1 = getNumShards("test1");
        long test1ExpectedWrites = 2 * test1.dataCopies;
        NumShards test2 = getNumShards("test2");
        long test2ExpectedWrites = test2.dataCopies;
        long totalExpectedWrites = test1ExpectedWrites + test2ExpectedWrites;

        IndicesStatsResponse stats = client().admin().indices().prepareStats().execute().actionGet();
        assertThat(stats.getPrimaries().getDocs().getCount(), equalTo(3L));
        assertThat(stats.getTotal().getDocs().getCount(), equalTo(totalExpectedWrites));
        assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexCount(), equalTo(3L));
        assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexFailedCount(), equalTo(0L));
        assertThat(stats.getPrimaries().getIndexing().getTotal().isThrottled(), equalTo(false));
        assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTime().millis(), equalTo(0L));
        assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(totalExpectedWrites));
        assertThat(stats.getTotal().getStore(), notNullValue());
        assertThat(stats.getTotal().getMerge(), notNullValue());
        assertThat(stats.getTotal().getFlush(), notNullValue());
        assertThat(stats.getTotal().getRefresh(), notNullValue());

        assertThat(stats.getIndex("test1").getPrimaries().getDocs().getCount(), equalTo(2L));
        assertThat(stats.getIndex("test1").getTotal().getDocs().getCount(), equalTo(test1ExpectedWrites));
        assertThat(stats.getIndex("test1").getPrimaries().getStore(), notNullValue());
        assertThat(stats.getIndex("test1").getPrimaries().getMerge(), notNullValue());
        assertThat(stats.getIndex("test1").getPrimaries().getFlush(), notNullValue());
        assertThat(stats.getIndex("test1").getPrimaries().getRefresh(), notNullValue());

        assertThat(stats.getIndex("test2").getPrimaries().getDocs().getCount(), equalTo(1L));
        assertThat(stats.getIndex("test2").getTotal().getDocs().getCount(), equalTo(test2ExpectedWrites));

        // make sure that number of requests in progress is 0
        assertThat(stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexCurrent(), equalTo(0L));
        assertThat(stats.getIndex("test1").getTotal().getIndexing().getTotal().getDeleteCurrent(), equalTo(0L));
        assertThat(stats.getIndex("test1").getTotal().getSearch().getTotal().getFetchCurrent(), equalTo(0L));
        assertThat(stats.getIndex("test1").getTotal().getSearch().getTotal().getQueryCurrent(), equalTo(0L));

        // check flags
        stats = client().admin().indices().prepareStats().clear()
                .setFlush(true)
                .setRefresh(true)
                .setMerge(true)
                .execute().actionGet();

        assertThat(stats.getTotal().getDocs(), nullValue());
        assertThat(stats.getTotal().getStore(), nullValue());
        assertThat(stats.getTotal().getIndexing(), nullValue());
        assertThat(stats.getTotal().getMerge(), notNullValue());
        assertThat(stats.getTotal().getFlush(), notNullValue());
        assertThat(stats.getTotal().getRefresh(), notNullValue());

        // check types
        stats = client().admin().indices().prepareStats().setTypes("type1", "type").execute().actionGet();
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCount(), equalTo(1L));
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type").getIndexCount(), equalTo(1L));
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexFailedCount(), equalTo(0L));
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2"), nullValue());
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCurrent(), equalTo(0L));
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getDeleteCurrent(), equalTo(0L));

        assertThat(stats.getTotal().getGet().getCount(), equalTo(0L));
        // check get
        GetResponse getResponse = client().prepareGet("test1", "type1", "1").execute().actionGet();
        assertThat(getResponse.isExists(), equalTo(true));

        stats = client().admin().indices().prepareStats().execute().actionGet();
        assertThat(stats.getTotal().getGet().getCount(), equalTo(1L));
        assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1L));
        assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(0L));

        // missing get
        getResponse = client().prepareGet("test1", "type1", "2").execute().actionGet();
        assertThat(getResponse.isExists(), equalTo(false));

        stats = client().admin().indices().prepareStats().execute().actionGet();
        assertThat(stats.getTotal().getGet().getCount(), equalTo(2L));
        assertThat(stats.getTotal().getGet().getExistsCount(), equalTo(1L));
        assertThat(stats.getTotal().getGet().getMissingCount(), equalTo(1L));

        // clear all
        stats = client().admin().indices().prepareStats()
                .setDocs(false)
                .setStore(false)
                .setIndexing(false)
                .setFlush(true)
                .setRefresh(true)
                .setMerge(true)
                .clear() // reset defaults
                .execute().actionGet();

        assertThat(stats.getTotal().getDocs(), nullValue());
        assertThat(stats.getTotal().getStore(), nullValue());
        assertThat(stats.getTotal().getIndexing(), nullValue());
        assertThat(stats.getTotal().getGet(), nullValue());
        assertThat(stats.getTotal().getSearch(), nullValue());

        // index failed
        try {
            client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").setVersion(1)
                    .setVersionType(VersionType.EXTERNAL).execute().actionGet();
            fail("Expected a version conflict");
        } catch (VersionConflictEngineException e) {}
        try {
            client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").setVersion(1)
                    .setVersionType(VersionType.EXTERNAL).execute().actionGet();
            fail("Expected a version conflict");
        } catch (VersionConflictEngineException e) {}
        try {
            client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").setVersion(1)
                    .setVersionType(VersionType.EXTERNAL).execute().actionGet();
            fail("Expected a version conflict");
        } catch (VersionConflictEngineException e) {}

        stats = client().admin().indices().prepareStats().setTypes("type1", "type2").execute().actionGet();
        assertThat(stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(2L));
        assertThat(stats.getIndex("test2").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(1L));
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexFailedCount(), equalTo(1L));
        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2").getIndexFailedCount(), equalTo(1L));
        assertThat(stats.getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(3L));
    }

    public void testMergeStats() {
        createIndex("test1");

        ensureGreen();

        // clear all
        IndicesStatsResponse stats = client().admin().indices().prepareStats()
                .setDocs(false)
                .setStore(false)
                .setIndexing(false)
                .setFlush(true)
                .setRefresh(true)
                .setMerge(true)
                .clear() // reset defaults
                .execute().actionGet();

        assertThat(stats.getTotal().getDocs(), nullValue());
        assertThat(stats.getTotal().getStore(), nullValue());
        assertThat(stats.getTotal().getIndexing(), nullValue());
        assertThat(stats.getTotal().getGet(), nullValue());
        assertThat(stats.getTotal().getSearch(), nullValue());

        for (int i = 0; i < 20; i++) {
            client().prepareIndex("test1", "type1", Integer.toString(i)).setSource("field", "value").execute().actionGet();
            client().prepareIndex("test1", "type2", Integer.toString(i)).setSource("field", "value").execute().actionGet();
            client().admin().indices().prepareFlush().execute().actionGet();
        }
        client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet();
        stats = client().admin().indices().prepareStats()
                .setMerge(true)
                .execute().actionGet();

        assertThat(stats.getTotal().getMerge(), notNullValue());
        assertThat(stats.getTotal().getMerge().getTotal(), greaterThan(0L));
    }

    public void testSegmentsStats() {
        assertAcked(prepareCreate("test1", 2, settingsBuilder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1))));
        ensureGreen();

        NumShards test1 = getNumShards("test1");

        for (int i = 0; i < 100; i++) {
            index("test1", "type1", Integer.toString(i), "field", "value");
            index("test1", "type2", Integer.toString(i), "field", "value");
        }

        IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
        assertThat(stats.getTotal().getSegments().getIndexWriterMemoryInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().getSegments().getIndexWriterMaxMemoryInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().getSegments().getVersionMapMemoryInBytes(), greaterThan(0L));

        client().admin().indices().prepareFlush().get();
        client().admin().indices().prepareForceMerge().setMaxNumSegments(1).execute().actionGet();
        stats = client().admin().indices().prepareStats().setSegments(true).get();

        assertThat(stats.getTotal().getSegments(), notNullValue());
        assertThat(stats.getTotal().getSegments().getCount(), equalTo((long) test1.totalNumShards));
        assertThat(stats.getTotal().getSegments().getMemoryInBytes(), greaterThan(0L));
    }

    public void testAllFlags() throws Exception {
        // rely on 1 replica for this tests
        createIndex("test1");
        createIndex("test2");

        ensureGreen();

        client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").execute().actionGet();

        client().admin().indices().prepareRefresh().execute().actionGet();
        IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
        Flag[] values = CommonStatsFlags.Flag.values();
        for (Flag flag : values) {
            set(flag, builder, false);
        }

        IndicesStatsResponse stats = builder.execute().actionGet();
        for (Flag flag : values) {
            if (flag == Flag.Suggest) {
                // suggest flag is unused
                continue;
            }
            assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
            assertThat(isSet(flag, stats.getTotal()), equalTo(false));
        }

        for (Flag flag : values) {
            set(flag, builder, true);
        }
        stats = builder.execute().actionGet();
        for (Flag flag : values) {
            assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
            assertThat(isSet(flag, stats.getTotal()), equalTo(true));
        }
        Random random = random();
        EnumSet<Flag> flags = EnumSet.noneOf(Flag.class);
        for (Flag flag : values) {
            if (random.nextBoolean()) {
                flags.add(flag);
            }
        }

        for (Flag flag : values) {
            set(flag, builder, false); // clear all
        }

        for (Flag flag : flags) { // set the flags
            set(flag, builder, true);
        }
        stats = builder.execute().actionGet();
        for (Flag flag : flags) { // check the flags
            assertThat(isSet(flag, stats.getPrimaries()), equalTo(true));
            assertThat(isSet(flag, stats.getTotal()), equalTo(true));
        }

        for (Flag flag : EnumSet.complementOf(flags)) { // check the complement
            if (flag == Flag.Suggest) {
                // suggest flag is unused
                continue;
            }
            assertThat(isSet(flag, stats.getPrimaries()), equalTo(false));
            assertThat(isSet(flag, stats.getTotal()), equalTo(false));
        }

    }

    public void testEncodeDecodeCommonStats() throws IOException {
        CommonStatsFlags flags = new CommonStatsFlags();
        Flag[] values = CommonStatsFlags.Flag.values();
        assertThat(flags.anySet(), equalTo(true));

        for (Flag flag : values) {
            flags.set(flag, false);
        }
        assertThat(flags.anySet(), equalTo(false));
        for (Flag flag : values) {
            flags.set(flag, true);
        }
        assertThat(flags.anySet(), equalTo(true));
        Random random = random();
        flags.set(values[random.nextInt(values.length)], false);
        assertThat(flags.anySet(), equalTo(true));

        {
            BytesStreamOutput out = new BytesStreamOutput();
            flags.writeTo(out);
            out.close();
            BytesReference bytes = out.bytes();
            CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes));
            for (Flag flag : values) {
                assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag)));
            }
        }

        {
            for (Flag flag : values) {
                flags.set(flag, random.nextBoolean());
            }
            BytesStreamOutput out = new BytesStreamOutput();
            flags.writeTo(out);
            out.close();
            BytesReference bytes = out.bytes();
            CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes));
            for (Flag flag : values) {
                assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag)));
            }
        }
    }

    public void testFlagOrdinalOrder() {
        Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh,
                Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Completion, Flag.Segments,
                Flag.Translog, Flag.Suggest, Flag.RequestCache, Flag.Recovery};

        assertThat(flags.length, equalTo(Flag.values().length));
        for (int i = 0; i < flags.length; i++) {
            assertThat("ordinal has changed - this breaks the wire protocol. Only append to new values", i, equalTo(flags[i].ordinal()));
        }
    }

    public void testMultiIndex() throws Exception {
        createIndex("test1");
        createIndex("test2");

        ensureGreen();

        client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").execute().actionGet();
        refresh();

        int numShards1 = getNumShards("test1").totalNumShards;
        int numShards2 = getNumShards("test2").totalNumShards;

        IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
        IndicesStatsResponse stats = builder.execute().actionGet();

        assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2));

        stats = builder.setIndices("_all").execute().actionGet();
        assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2));

        stats = builder.setIndices("_all").execute().actionGet();
        assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2));

        stats = builder.setIndices("*").execute().actionGet();
        assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2));

        stats = builder.setIndices("test1").execute().actionGet();
        assertThat(stats.getTotalShards(), equalTo(numShards1));

        stats = builder.setIndices("test1", "test2").execute().actionGet();
        assertThat(stats.getTotalShards(), equalTo(numShards1 + numShards2));

        stats = builder.setIndices("*2").execute().actionGet();
        assertThat(stats.getTotalShards(), equalTo(numShards2));

    }

    public void testFieldDataFieldsParam() throws Exception {
        assertAcked(client().admin().indices().prepareCreate("test1")
                .addMapping("type", "bar", "type=text,fielddata=true",
                        "baz", "type=text,fielddata=true").get());

        ensureGreen();

        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
        refresh();

        client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet();

        IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
        IndicesStatsResponse stats = builder.execute().actionGet();

        assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields(), is(nullValue()));

        stats = builder.setFieldDataFields("bar").execute().actionGet();
        assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
        assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(false));

        stats = builder.setFieldDataFields("bar", "baz").execute().actionGet();
        assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
        assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(true));
        assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L));

        stats = builder.setFieldDataFields("*").execute().actionGet();
        assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
        assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(true));
        assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L));

        stats = builder.setFieldDataFields("*r").execute().actionGet();
        assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("bar"), is(true));
        assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L));
        assertThat(stats.getTotal().fieldData.getFields().containsKey("baz"), is(false));

    }

    public void testCompletionFieldsParam() throws Exception {
        assertAcked(prepareCreate("test1")
                .addMapping(
                        "bar",
                        "{ \"properties\": { \"bar\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}},\"baz\": { \"type\": \"text\", \"fields\": { \"completion\": { \"type\": \"completion\" }}}}}"));
        ensureGreen();

        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
        client().prepareIndex("test1", "baz", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}").execute().actionGet();
        refresh();

        IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
        IndicesStatsResponse stats = builder.execute().actionGet();

        assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields(), is(nullValue()));

        stats = builder.setCompletionFields("bar.completion").execute().actionGet();
        assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
        assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(false));

        stats = builder.setCompletionFields("bar.completion", "baz.completion").execute().actionGet();
        assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
        assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(true));
        assertThat(stats.getTotal().completion.getFields().get("baz.completion"), greaterThan(0L));

        stats = builder.setCompletionFields("*").execute().actionGet();
        assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
        assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(true));
        assertThat(stats.getTotal().completion.getFields().get("baz.completion"), greaterThan(0L));

        stats = builder.setCompletionFields("*r*").execute().actionGet();
        assertThat(stats.getTotal().completion.getSizeInBytes(), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("bar.completion"), is(true));
        assertThat(stats.getTotal().completion.getFields().get("bar.completion"), greaterThan(0L));
        assertThat(stats.getTotal().completion.getFields().containsKey("baz.completion"), is(false));

    }

    public void testGroupsParam() throws Exception {
        createIndex("test1");

        ensureGreen();

        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("foo", "bar").execute().actionGet();
        refresh();

        client().prepareSearch("_all").setStats("bar", "baz").execute().actionGet();

        IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
        IndicesStatsResponse stats = builder.execute().actionGet();

        assertThat(stats.getTotal().search.getTotal().getQueryCount(), greaterThan(0L));
        assertThat(stats.getTotal().search.getGroupStats(), is(nullValue()));

        stats = builder.setGroups("bar").execute().actionGet();
        assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0L));
        assertThat(stats.getTotal().search.getGroupStats().containsKey("baz"), is(false));

        stats = builder.setGroups("bar", "baz").execute().actionGet();
        assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0L));
        assertThat(stats.getTotal().search.getGroupStats().get("baz").getQueryCount(), greaterThan(0L));

        stats = builder.setGroups("*").execute().actionGet();
        assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0L));
        assertThat(stats.getTotal().search.getGroupStats().get("baz").getQueryCount(), greaterThan(0L));

        stats = builder.setGroups("*r").execute().actionGet();
        assertThat(stats.getTotal().search.getGroupStats().get("bar").getQueryCount(), greaterThan(0L));
        assertThat(stats.getTotal().search.getGroupStats().containsKey("baz"), is(false));

    }

    public void testTypesParam() throws Exception {
        createIndex("test1");
        createIndex("test2");

        ensureGreen();

        client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("foo", "bar").execute().actionGet();
        client().prepareIndex("test2", "baz", Integer.toString(1)).setSource("foo", "bar").execute().actionGet();
        refresh();

        IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
        IndicesStatsResponse stats = builder.execute().actionGet();

        assertThat(stats.getTotal().indexing.getTotal().getIndexCount(), greaterThan(0L));
        assertThat(stats.getTotal().indexing.getTypeStats(), is(nullValue()));

        stats = builder.setTypes("bar").execute().actionGet();
        assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
        assertThat(stats.getTotal().indexing.getTypeStats().containsKey("baz"), is(false));

        stats = builder.setTypes("bar", "baz").execute().actionGet();
        assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
        assertThat(stats.getTotal().indexing.getTypeStats().get("baz").getIndexCount(), greaterThan(0L));

        stats = builder.setTypes("*").execute().actionGet();
        assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
        assertThat(stats.getTotal().indexing.getTypeStats().get("baz").getIndexCount(), greaterThan(0L));

        stats = builder.setTypes("*r").execute().actionGet();
        assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
        assertThat(stats.getTotal().indexing.getTypeStats().containsKey("baz"), is(false));

    }

    private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean set) {
        switch (flag) {
            case Docs:
                builder.setDocs(set);
                break;
            case FieldData:
                builder.setFieldData(set);
                break;
            case QueryCache:
                builder.setQueryCache(set);
                break;
            case Flush:
                builder.setFlush(set);
                break;
            case Get:
                builder.setGet(set);
                break;
            case Indexing:
                builder.setIndexing(set);
                break;
            case Merge:
                builder.setMerge(set);
                break;
            case Refresh:
                builder.setRefresh(set);
                break;
            case Search:
                builder.setSearch(set);
                break;
            case Store:
                builder.setStore(set);
                break;
            case Warmer:
                builder.setWarmer(set);
                break;
            case Completion:
                builder.setCompletion(set);
                break;
            case Segments:
                builder.setSegments(set);
                break;
            case Translog:
                builder.setTranslog(set);
                break;
            case Suggest: // unused
                break;
            case RequestCache:
                builder.setRequestCache(set);
                break;
            case Recovery:
                builder.setRecovery(set);
                break;
            default:
                fail("new flag? " + flag);
                break;
        }
    }

    private static boolean isSet(Flag flag, CommonStats response) {
        switch (flag) {
            case Docs:
                return response.getDocs() != null;
            case FieldData:
                return response.getFieldData() != null;
            case QueryCache:
                return response.getQueryCache() != null;
            case Flush:
                return response.getFlush() != null;
            case Get:
                return response.getGet() != null;
            case Indexing:
                return response.getIndexing() != null;
            case Merge:
                return response.getMerge() != null;
            case Refresh:
                return response.getRefresh() != null;
            case Search:
                return response.getSearch() != null;
            case Store:
                return response.getStore() != null;
            case Warmer:
                return response.getWarmer() != null;
            case Completion:
                return response.getCompletion() != null;
            case Segments:
                return response.getSegments() != null;
            case Translog:
                return response.getTranslog() != null;
            case Suggest: // unused
                return true;
            case RequestCache:
                return response.getRequestCache() != null;
            case Recovery:
                return response.getRecoveryStats() != null;
            default:
                fail("new flag? " + flag);
                return false;
        }
    }

    private void assertEquals(QueryCacheStats stats1, QueryCacheStats stats2) {
        assertEquals(stats1.getCacheCount(), stats2.getCacheCount());
        assertEquals(stats1.getCacheSize(), stats2.getCacheSize());
        assertEquals(stats1.getEvictions(), stats2.getEvictions());
        assertEquals(stats1.getHitCount(), stats2.getHitCount());
        assertEquals(stats2.getMemorySizeInBytes(), stats2.getMemorySizeInBytes());
        assertEquals(stats1.getMissCount(), stats2.getMissCount());
        assertEquals(stats1.getTotalCount(), stats2.getTotalCount());
    }

    private void assertCumulativeQueryCacheStats(IndicesStatsResponse response) {
        assertAllSuccessful(response);
        QueryCacheStats total = response.getTotal().queryCache;
        QueryCacheStats indexTotal = new QueryCacheStats();
        QueryCacheStats shardTotal = new QueryCacheStats();
        for (IndexStats indexStats : response.getIndices().values()) {
            indexTotal.add(indexStats.getTotal().queryCache);
            for (ShardStats shardStats : response.getShards()) {
                shardTotal.add(shardStats.getStats().queryCache);
            }
        }
        assertEquals(total, indexTotal);
        assertEquals(total, shardTotal);
    }

    public void testFilterCacheStats() throws Exception {
        assertAcked(prepareCreate("index").setSettings(Settings.builder().put(indexSettings()).put("number_of_replicas", 0).build()).get());
        indexRandom(true,
                client().prepareIndex("index", "type", "1").setSource("foo", "bar"),
                client().prepareIndex("index", "type", "2").setSource("foo", "baz"));
        ensureGreen();

        IndicesStatsResponse response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(response);
        assertEquals(0, response.getTotal().queryCache.getCacheSize());

        SearchResponse r;
        assertSearchResponse(r = client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(response);
        assertThat(response.getTotal().queryCache.getHitCount(), equalTo(0L));
        assertThat(response.getTotal().queryCache.getEvictions(), equalTo(0L));
        assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getCacheSize(), greaterThan(0L));

        assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(response);
        assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getEvictions(), equalTo(0L));
        assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getCacheSize(), greaterThan(0L));

        assertTrue(client().prepareDelete("index", "type", "1").get().isFound());
        assertTrue(client().prepareDelete("index", "type", "2").get().isFound());
        refresh();
        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(response);
        assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getEvictions(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getCacheSize(), equalTo(0L));
        assertThat(response.getTotal().queryCache.getCacheCount(), greaterThan(0L));

        indexRandom(true,
                client().prepareIndex("index", "type", "1").setSource("foo", "bar"),
                client().prepareIndex("index", "type", "2").setSource("foo", "baz"));
        assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());

        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(response);
        assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getEvictions(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getCacheSize(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getMemorySizeInBytes(), greaterThan(0L));

        assertAllSuccessful(client().admin().indices().prepareClearCache("index").setQueryCache(true).get());
        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
        assertCumulativeQueryCacheStats(response);
        assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getEvictions(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
        assertThat(response.getTotal().queryCache.getCacheSize(), equalTo(0L));
        assertThat(response.getTotal().queryCache.getMemorySizeInBytes(), equalTo(0L));
    }

}


text:
<?php

namespace Wasabi\Cms\Model\Entity;

use Cake\ORM\Entity;

/**
 * Class MenuItem
 *
 * @property int id
 * @property int menu_id
 * @property int parent_id
 * @property int lft
 * @property int rght
 * @property string name
 * @property string type
 * @property string target
 * @property string external_link
 * @property string foreign_model
 * @property int foreign_id
 * @property string plugin
 * @property string controller
 * @property string action
 * @property string params
 * @property string query
 * @property \DateTime created
 * @property \DateTime modified
 */
class MenuItem extends Entity
{
    const TYPE_ENTITY = 'entity';
    const TYPE_EXTERNAL = 'external';
    const TYPE_CUSTOM = 'custom';
}

redpajama_set_name: RedPajamaGithub


text:
package org.elasticsearch.cluster.routing;

import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.shard.ShardId;

import java.io.IOException;
import java.io.Serializable;

/**
 * {@link ImmutableShardRouting} immutably encapsulates information about shard
 * routings like id, state, version, etc.
 */
public class ImmutableShardRouting implements Streamable, Serializable, ShardRouting {

    protected String index;

    protected int shardId;

    protected String currentNodeId;

    protected String relocatingNodeId;

    protected boolean primary;

    protected ShardRoutingState state;

    protected long version;

    private transient ShardId shardIdentifier;

    private final transient ImmutableList<ShardRouting> asList;

    ImmutableShardRouting() {
        this.asList = ImmutableList.of((ShardRouting) this);
    }

    public ImmutableShardRouting(ShardRouting copy) {
        this(copy.index(), copy.id(), copy.currentNodeId(), copy.primary(), copy.state(), copy.version());
        this.relocatingNodeId = copy.relocatingNodeId();
    }

    public ImmutableShardRouting(ShardRouting copy, long version) {
        this(copy.index(), copy.id(), copy.currentNodeId(), copy.primary(), copy.state(), copy.version());
        this.relocatingNodeId = copy.relocatingNodeId();
        this.version = version;
    }

    public ImmutableShardRouting(String index, int shardId, String currentNodeId,
                                 String relocatingNodeId, boolean primary, ShardRoutingState state, long version) {
        this(index, shardId, currentNodeId, primary, state, version);
        this.relocatingNodeId = relocatingNodeId;
    }

    public ImmutableShardRouting(String index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state, long version) {
        this.index = index;
        this.shardId = shardId;
        this.currentNodeId = currentNodeId;
        this.primary = primary;
        this.state = state;
        this.asList = ImmutableList.of((ShardRouting) this);
        this.version = version;
    }

    @Override
    public String index() {
        return this.index;
    }

    @Override
    public String getIndex() {
        return index();
    }

    @Override
    public int id() {
        return this.shardId;
    }

    @Override
    public int getId() {
        return id();
    }

    @Override
    public long version() {
        return this.version;
    }

    @Override
    public boolean unassigned() {
        return state == ShardRoutingState.UNASSIGNED;
    }

    @Override
    public boolean initializing() {
        return state == ShardRoutingState.INITIALIZING;
    }

    @Override
    public boolean active() {
        return started() || relocating();
    }

    @Override
    public boolean started() {
        return state == ShardRoutingState.STARTED;
    }

    @Override
    public boolean relocating() {
        return state == ShardRoutingState.RELOCATING;
    }

    @Override
    public boolean assignedToNode() {
        return currentNodeId != null;
    }

    @Override
    public String currentNodeId() {
        return this.currentNodeId;
    }

    @Override
    public String relocatingNodeId() {
        return this.relocatingNodeId;
    }

    @Override
    public boolean primary() {
        return this.primary;
    }

    @Override
    public ShardRoutingState state() {
        return this.state;
    }

    @Override
    public ShardId shardId() {
        if (shardIdentifier != null) {
            return shardIdentifier;
        }
        shardIdentifier = new ShardId(index, shardId);
        return shardIdentifier;
    }

    @Override
    public ShardIterator shardsIt() {
        return new PlainShardIterator(shardId(), asList);
    }

    /**
     * Reads a {@link ImmutableShardRouting} instance of a shard from an {@link InputStream}
     *
     * @param in {@link InputStream} to read the entry from
     * @return {@link ImmutableShardRouting} instances read from the given {@link InputStream}
     *
     * @throws IOException if some exception occurs during the read operations
     */
    public static ImmutableShardRouting readShardRoutingEntry(StreamInput in) throws IOException {
        ImmutableShardRouting entry = new ImmutableShardRouting();
        entry.readFrom(in);
        return entry;
    }

    /**
     * Reads a routingentry from an inputstream with given index and
     * shardId.
     *
     * @param in inputstream to read the entry from
     * @param index shards index
     * @param id id of the shard
     * @return Shard routing entry read
     *
     * @throws IOException if some exception occurs during the read operations
     */
    public static ImmutableShardRouting readShardRoutingEntry(StreamInput in, String index, int shardId) throws IOException {
        ImmutableShardRouting entry = new ImmutableShardRouting();
        entry.readFrom(in, index, shardId);
        return entry;
    }

    /**
     * Read information from an inputstream with given index and
     * shardId.
     *
     * @param in inputstream to read the entry from
     * @param index shards index
     * @param id id of the shard
     *
     * @throws IOException if some exception occurs during the read operations
     */
    public void readFrom(StreamInput in, String index, int shardId) throws IOException {
        this.index = index;
        this.shardId = shardId;
        readFromThin(in);
    }

    @Override
    public void readFromThin(StreamInput in) throws IOException {
        version = in.readLong();
        if (in.readBoolean()) {
            currentNodeId = in.readString();
        }

        if (in.readBoolean()) {
            relocatingNodeId = in.readString();
        }

        primary = in.readBoolean();
        state = ShardRoutingState.fromValue(in.readByte());
    }

    @Override
    public void readFrom(StreamInput in) throws IOException {
        readFrom(in, in.readString(), in.readVInt());
    }

    /**
     * Writes shard information to {@link StreamOutput} without writing index name and shard id
     * @param out {@link StreamOutput} to write shard information to
     * @throws IOException if something happens during write
     */
    public void writeToThin(StreamOutput out) throws IOException {
        out.writeLong(version);
        if (currentNodeId != null) {
            out.writeBoolean(true);
            out.writeString(currentNodeId);
        } else {
            out.writeBoolean(false);
        }

        if (relocatingNodeId != null) {
            out.writeBoolean(true);
            out.writeString(relocatingNodeId);
        } else {
            out.writeBoolean(false);
        }

        out.writeBoolean(primary);
        out.writeByte(state.value());
    }

    @Override
    public void writeTo(StreamOutput out) throws IOException {
        out.writeString(index);
        out.writeVInt(shardId);
        writeToThin(out);
    }

    @Override
    public boolean equals(Object o) {
        if (this == o) return true;
        // we check on instanceof so we also handle the MutableShardRouting case as well
        if (o == null || !(o instanceof ImmutableShardRouting)) return false;

        ImmutableShardRouting that = (ImmutableShardRouting) o;

        if (primary != that.primary) return false;
        if (shardId != that.shardId) return false;
        if (currentNodeId != null ? !currentNodeId.equals(that.currentNodeId) : that.currentNodeId != null)
            return false;
        if (index != null ? !index.equals(that.index) : that.index != null) return false;
        if (relocatingNodeId != null ? !relocatingNodeId.equals(that.relocatingNodeId) : that.relocatingNodeId != null)
            return false;
        if (state != that.state) return false;

        return true;
    }

    @Override
    public int hashCode() {
        int result = index != null ? index.hashCode() : 0;
        result = 31 * result + shardId;
        result = 31 * result + (currentNodeId != null ? currentNodeId.hashCode() : 0);
        result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0);
        result = 31 * result + (primary ? 1 : 0);
        result = 31 * result + (state != null ? state.hashCode() : 0);
        return result;
    }

    @Override
    public String toString() {
        return shortSummary();
    }

    @Override
    public String shortSummary() {
        StringBuilder sb = new StringBuilder();
        sb.append('[').append(index).append(']').append('[').append(shardId).append(']');
        sb.append(", node[").append(currentNodeId).append("], ");
        if (relocatingNodeId != null) {
            sb.append("relocating [").append(relocatingNodeId).append("], ");
        }
        if (primary) {
            sb.append("[P]");
        } else {
            sb.append("[R]");
        }
        sb.append(", s[").append(state).append("]");
        return sb.toString();
    }

    @Override
    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
        return builder.startObject()
                .field("state", state())
                .field("primary", primary())
                .field("node", currentNodeId())
                .field("relocating_node", relocatingNodeId())
                .field("shard", shardId().id())
                .field("index", shardId().index().name())
                .endObject();
    }
}


text:
include_recipe 'basenginx'

include_recipe 'basehttploadbalancer::create_load_balancer_config'
include_recipe 'basehttploadbalancer::delete_default_config'


text:
{{banner.name}}

  {{file.name}}

      {{progress[$index]}}%

<form role="form">

    Client Name
    <input type="name" ng-model="banner.name" class="form-control" placeholder="Enter client name">

      Upload Files

Files

          Title
          Filename

            <input type="text" ng-model="file.title" class="form-control" />

          {{file.filename}}

  <button type="button" class="btn btn-success" ng-click="update(banner)">Update</button>
</form>


text:
Perl string 中可以使用变量,如

    my $var = $object->method($argument);
    print "the output is : $var, please verify";

这个$var 只用了一次,能否省去呢?比如换成如下写法?

    print "the output is : $object->method($argument), please verify";

不行,不过有间接的方法,

    print "the output is : ${\ $object->method($argument) }, please verify";

或者,

    print "the output is : @{[ $object->method($argument) ]}, please verify";

参考

* [Printing function calls in Perl](http://leonerds-code.blogspot.com/2014_11_01_archive.html)

## The non-destructive modifier /r

The non-destructive modifier s///r causes the result of the substitution to be returned instead of modifying $_
 (or whatever variable the substitute was bound to with =~ ):

    $x = "I like dogs.";
    $y = $x =~ s/dogs/cats/r;
    print "$x $y\n"; # prints "I like dogs. I like cats."

这一段直接从perlrequick 中抄过来的。

参考

* [perlrequick](http://perldoc.perl.org/perlrequick.html)

## cron jobs 中使用 perlbrew

perlbrew 很好用,我使用perlbrew 很久以后,才发现可以这样用。

    0 23  * * * /path/to/perlbrew exec --with perl-5.20.0 perl /path/to/app.pl > /dev/null

## 后记

这是我最近才发现的原来可以这样用的小技巧。欢迎分享你的发现,谢谢

## 作者

* [SWUECHO](https://metacpan.org/author/SWUECHO)


text:
namespace vtrc { namespace common {

    namespace basio = VTRC_ASIO;
    namespace bip   = VTRC_ASIO::ip;
    namespace bsys  = VTRC_SYSTEM;

    namespace {
        typedef transport_posixs::socket_type socket_type;
        typedef transport_impl<socket_type, transport_posixs> impl_type;

    }

    struct transport_posixs::impl: public impl_type {

        impl( vtrc::shared_ptr<socket_type> s, const std::string &n )
            :impl_type(s, n)
        { }

        void close_handle( )
        {
            get_socket( ).close( );
        }

    };

    transport_posixs::transport_posixs( vtrc::shared_ptr<socket_type> sock )
        :impl_(new impl(sock, "tcp"))
    {
        impl_->set_parent( this );
    }

    transport_posixs::~transport_posixs(  )
    {
        delete impl_;
    }

    std::string transport_posixs::name( ) const
    {
        return impl_->name( );
    }

    void transport_posixs::close( )
    {
        impl_->close( );
    }

    void transport_posixs::write( const char *data, size_t length )
    {
        return impl_->write( data, length );
    }

    void transport_posixs::write(const char *data, size_t length,
                     const system_closure_type &success, bool on_send_success)
    {
        impl_->write( data, length, success, on_send_success );
    }

    native_handle_type transport_posixs::native_handle( ) const
    {
        native_handle_type nh;
#ifdef _WIN32
        nh.value.win_handle =
            reinterpret_cast<native_handle_type::handle>
                ((SOCKET)impl_->get_socket( ).native_handle( ));
#else
        nh.value.unix_fd = impl_->get_socket( ).native_handle( );
#endif
        return nh;
    }

    socket_type &transport_posixs::get_socket( )
    {
        return impl_->get_socket( );
    }

    const socket_type &transport_posixs::get_socket( ) const
    {
        return impl_->get_socket( );
    }

}}

#endif


text:
'use strict';

var babelify = require('babelify'),
	browserify = require('browserify'),
	buffer = require('vinyl-buffer'),
	concat = require('gulp-concat'),
	config = require('./config'),
	deps = require('./deps'),
	gulp = require('gulp'),
	gulpif = require('gulp-if'),
	gutil = require('gulp-util'),
	less = require('gulp-less'),
	minifyCSS = require('gulp-minify-css'),
	rename = require('gulp-rename'),
	rev = require('gulp-rev'),
	source = require('vinyl-source-stream'),
	sourcemaps = require('gulp-sourcemaps'),
	uglify = require('gulp-uglify');

const debug = config.DEBUG;

function gulper(name, files, dest) {
	gulp.task(name, function() {
		return gulp.src(files)
			.pipe(concat(name))
			.pipe(gulpif(!debug, uglify()))
			.pipe(rev())
			.pipe(rename({suffix: '.' + (debug ? 'debug' : 'min') + '.js'}))
			.pipe(gulp.dest(dest))
			.pipe(rev.manifest(name + '.json'))
			.pipe(gulp.dest('./state'));
	});
}
gulper('mod', deps.mod, './state');

gulp.task('css', function() {
	return gulp.src('./less/*.less')
		.pipe(sourcemaps.init())
		.pipe(less())
		.pipe(minifyCSS({rebase: false}))
		.pipe(sourcemaps.write('./maps/'))
		.pipe(gulp.dest('./www/css'))
		.pipe(gulp.dest('./state'));
});

function build(name, b, dest) {
	gulp.task(name, function() {
		return bundler(name, b, dest);
	});
}

function bundler(name, b, dest) {
	return b.bundle()
		// Transform into vinyl stream
		.pipe(source(name + '.js'))
		.pipe(buffer())
		.pipe(sourcemaps.init({loadMaps: true}))
		// TEMP: Don't minify the client, until we get minification
		// support for ES6
		.pipe(gulpif(!debug && name === 'vendor', uglify()))
		.on('error', gutil.log)
		.pipe(sourcemaps.write('./'))
		.pipe(gulp.dest(dest));
}

{
	let b = browserify({
		entries: './client/main',
		// Needed for sourcemaps
		debug: true,
		bundleExternal: false,
		external: [
			'jquery',
			'jquery.cookie',
			'underscore',
			'backbone',
			'backbone.radio',
			'stack-blur',
			'lang'
		]
	})
		// Make available outside the bundle with require() under a
		// shorthand name
		.require('./client/main', {expose: 'main'})
		// Transpile ES6 functionality that is not yet supported by the latest
		// stable Chrome and FF to ES5. Ancient and hipster browsers can
		// suck my dick.
		.transform(babelify.configure({
			// MUH PERFORMINCE
			blacklist: [
				'es3.memberExpressionLiterals',
				'es3.propertyLiterals',
				'es5.properties.mutators',
				'es6.constants',
				'es6.forOf',
				'es6.spec.templateLiterals',
				'es6.templateLiterals',
				'flow',
				'react',
				'jscript',
				'react',
				'reactCompat',
				'regenerator',
				'runtime'
			]
		}))
		// Exclude these requires on the client
		.exclude('../config')
		.exclude('../lang/')
		.exclude('../server/state');

	build('client', b, './www/js');
}

{
	let b = browserify({
		require: [
			'jquery',
			'jquery.cookie',
			'underscore',
			'backbone',
			'backbone.radio'
		],
		debug: true
	})
		.require('./lib/stack-blur', {expose: 'stack-blur'});

	build('vendor', b, './www/js');
}

gulp.task('lang', function() {
	// Language bundles
	const langs = config.LANGS;
	for (let i = 0, l = langs.length; i < l; i++) {
		const lang = langs[i];
		let b = browserify({debug: true})
			.require(`./lang/${lang}/common`, {expose: 'lang'});
		bundler(lang, b, './www/js/lang');
	}
});


text:
import { IncomingMessage } from "http";
import { PlayerInfo } from "../../../shared/player_info";
import {
  COOKIE_AUTHENTICATION_SUBPROTOCOL,
  CREDENTIALS_AUTHENTICATION_SUBPROTOCOL,
  AuthenticationMessage,
  SocketMessage,
  isAuthenticationMessage,
  newPlayerInfoMessage,
} from "../../../shared/socket_protocol";
import { PlayerSocket } from "./player_socket";

export type PlayerInfoCallback = (error: Error, playerInfo: PlayerInfo) => void;
export type RequestAuthenticator = (request: IncomingMessage, callback: PlayerInfoCallback) => void;
export type CredentialAuthenticator =
    (credentials: AuthenticationMessage, callback: PlayerInfoCallback) => void;

// https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent#Status_codes
// The endpoint is terminating the connection due to a protocol error.
const PROTOCOL_ERROR:number = 1002;
// The server is terminating the connection because it encountered an unexpected condition that
// prevented it from fulfilling the request.
const INTERNAL_ERROR:number = 1011;

/**
 * Class that combines the different authentication strategies for WebSockets.
 * Given a WebSocket to authenticate via the "authenticate" method, it will either return a
 * PlayerSocket with the appropriate PlayerInfo (via the "onAuthentication" method) or close the the
 * connection with the socket if it fails to authenticate.
 */
export class SocketAuthenticator {
  /**
   * Called whenever a socket successfully authenticates.
   * Should be set by the owner of this object to receive authenticated users.
   */
  public onAuthentication: (playerSocket: PlayerSocket) => void;

  constructor(
      private requestAuthentication: RequestAuthenticator,
      private credentialAuthentication: CredentialAuthenticator,
      private authenticationTimeout: number) {}

  /**
   * This asynchronous method either authenticates the socket and calls "onAuthentication" with the
   * appropriate PlayerSocket or closes the connection with "ws".
   */
  public authenticate(ws: WebSocket, request: IncomingMessage) {
    if (ws.protocol == CREDENTIALS_AUTHENTICATION_SUBPROTOCOL) {
      this.authenticateWithCredentials(ws);
    } else if (ws.protocol == COOKIE_AUTHENTICATION_SUBPROTOCOL) {
      this.authenticateWithCookie(ws, request);
    } else {
      ws.close(PROTOCOL_ERROR, "Unknown subprotocol: " + ws.protocol);
    }
  }

  private authenticateWithCookie(ws: WebSocket, request: IncomingMessage) {
    this.requestAuthentication(request, (error: Error, playerInfo: PlayerInfo) => {
      if (error) {
        console.log(error);
        ws.close(INTERNAL_ERROR, error.message);
      } else if (playerInfo) {
        this.onPlayerInfoFound(ws, playerInfo);
      } else {
        ws.close(INTERNAL_ERROR, "Player info not found. Invalid or expired cookie.");
      }
    });
  }

  private authenticateWithCredentials(ws: WebSocket) {
    let seconds = this.authenticationTimeout / 1000;
    let timer: NodeJS.Timer = setTimeout(() => {
      ws.close(PROTOCOL_ERROR, "Expected authentication info within " + seconds + " seconds of connection.");
    }, this.authenticationTimeout);
    ws.onmessage = (ev: MessageEvent) => {
      clearTimeout(timer);
      // Only listen to one message.
      ws.onmessage = null;
      let message: SocketMessage;
      try {
        message = JSON.parse(ev.data);
      } catch(e) {
        console.log("Error parsing authentication message as JSON:");
        console.log(ev.data);
        ws.close(PROTOCOL_ERROR, "Expected first message to be authentication info (as a JSON string).")
        return;
      }
      if (isAuthenticationMessage(message)) {
        this.validateAuthenticationCredentials(ws, message);
      } else {
        ws.close(PROTOCOL_ERROR, "Expected first message to be authentication info.");
      }
    };
  }

  private validateAuthenticationCredentials(ws: WebSocket, authenticationCredentials: AuthenticationMessage) {
    this.credentialAuthentication(authenticationCredentials, (error: Error, playerInfo: PlayerInfo) => {
      if (error) {
        console.log(error);
        ws.close(INTERNAL_ERROR, error.message);
      } else if (playerInfo) {
        this.onPlayerInfoFound(ws, playerInfo);
      } else {
        ws.close(INTERNAL_ERROR, "Invalid authentication info.");
      }
    })
  }

  private onPlayerInfoFound(ws: WebSocket, info: PlayerInfo) {
    ws.send(JSON.stringify(newPlayerInfoMessage(info)));
    this.onAuthentication(new PlayerSocket(info, ws));
  }
}


text:
package org.elasticsearch.index.reindex;

import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.tasks.LoggingTaskListener;
import org.elasticsearch.tasks.Task;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

public abstract class AbstractBaseReindexRestHandler<
                Request extends AbstractBulkByScrollRequest<Request>,
                A extends ActionType<BulkByScrollResponse>
            > extends BaseRestHandler {

    private final A action;

    protected AbstractBaseReindexRestHandler(A action) {
        this.action = action;
    }

    protected RestChannelConsumer doPrepareRequest(RestRequest request, NodeClient client,
                                                   boolean includeCreated, boolean includeUpdated) throws IOException {
        // Build the internal request
        Request internal = setCommonOptions(request, buildRequest(request, client.getNamedWriteableRegistry()));

        // Executes the request and waits for completion
        if (request.paramAsBoolean("wait_for_completion", true)) {
            Map<String, String> params = new HashMap<>();
            params.put(BulkByScrollTask.Status.INCLUDE_CREATED, Boolean.toString(includeCreated));
            params.put(BulkByScrollTask.Status.INCLUDE_UPDATED, Boolean.toString(includeUpdated));

            return channel -> client.executeLocally(action, internal, new BulkIndexByScrollResponseContentListener(channel, params));
        } else {
            internal.setShouldStoreResult(true);
        }

        /*
         * Let's try and validate before forking so the user gets some error. The
         * task can't totally validate until it starts but this is better than
         * nothing.
         */
        ActionRequestValidationException validationException = internal.validate();
        if (validationException != null) {
            throw validationException;
        }
        return sendTask(client.getLocalNodeId(), client.executeLocally(action, internal, LoggingTaskListener.instance()));
    }

    /**
     * Build the Request based on the RestRequest.
     */
    protected abstract Request buildRequest(RestRequest request, NamedWriteableRegistry namedWriteableRegistry) throws IOException;

    /**
     * Sets common options of {@link AbstractBulkByScrollRequest} requests.
     */
    protected Request setCommonOptions(RestRequest restRequest, Request request) {
        assert restRequest != null : "RestRequest should not be null";
        assert request != null : "Request should not be null";

        request.setRefresh(restRequest.paramAsBoolean("refresh", request.isRefresh()));
        request.setTimeout(restRequest.paramAsTime("timeout", request.getTimeout()));

        Integer slices = parseSlices(restRequest);
        if (slices != null) {
            request.setSlices(slices);
        }

        String waitForActiveShards = restRequest.param("wait_for_active_shards");
        if (waitForActiveShards != null) {
            request.setWaitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
        }

        Float requestsPerSecond = parseRequestsPerSecond(restRequest);
        if (requestsPerSecond != null) {
            request.setRequestsPerSecond(requestsPerSecond);
        }

        if (restRequest.hasParam("max_docs")) {
            setMaxDocsValidateIdentical(request, restRequest.paramAsInt("max_docs", -1));
        }

        return request;
    }

    private RestChannelConsumer sendTask(String localNodeId, Task task) {
        return channel -> {
            try (XContentBuilder builder = channel.newBuilder()) {
                builder.startObject();
                builder.field("task", localNodeId + ":" + task.getId());
                builder.endObject();
                channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
            }
        };
    }

    private static Integer parseSlices(RestRequest request) {
        String slicesString = request.param("slices");
        if (slicesString == null) {
            return null;
        }

        if (slicesString.equals(AbstractBulkByScrollRequest.AUTO_SLICES_VALUE)) {
            return AbstractBulkByScrollRequest.AUTO_SLICES;
        }

        int slices;
        try {
            slices = Integer.parseInt(slicesString);
        } catch (NumberFormatException e) {
            throw new IllegalArgumentException(
                "[slices] must be a positive integer or the string \"auto\", but was [" + slicesString + "]", e);
        }

        if (slices < 1) {
            throw new IllegalArgumentException(
                "[slices] must be a positive integer or the string \"auto\", but was [" + slicesString + "]");
        }

        return slices;
    }

    /**
     * @return requests_per_second from the request as a float if it was on the request, null otherwise
     */
    public static Float parseRequestsPerSecond(RestRequest request) {
        String requestsPerSecondString = request.param("requests_per_second");
        if (requestsPerSecondString == null) {
            return null;
        }
        float requestsPerSecond;
        try {
            requestsPerSecond = Float.parseFloat(requestsPerSecondString);
        } catch (NumberFormatException e) {
            throw new IllegalArgumentException(
                    "[requests_per_second] must be a float greater than 0. Use -1 to disable throttling.", e);
        }
        if (requestsPerSecond == -1) {
            return Float.POSITIVE_INFINITY;
        }
        if (requestsPerSecond <= 0) {
            // We validate here and in the setters because the setters use "Float.POSITIVE_INFINITY" instead of -1
            throw new IllegalArgumentException(
                    "[requests_per_second] must be a float greater than 0. Use -1 to disable throttling.");
        }
        return requestsPerSecond;
    }

    static void setMaxDocsValidateIdentical(AbstractBulkByScrollRequest<?> request, int maxDocs) {
        if (request.getMaxDocs() != AbstractBulkByScrollRequest.MAX_DOCS_ALL_MATCHES && request.getMaxDocs() != maxDocs) {
            throw new IllegalArgumentException("[max_docs] set to two different values [" + request.getMaxDocs() + "]" +
                " and [" + maxDocs + "]");
        } else {
            request.setMaxDocs(maxDocs);
        }
    }
}


text:
namespace VRE.Vridge.API.Client
{
    /// <summary>
    /// Endpoint names for API services.
    /// </summary>
    public static class EndpointNames
    {
        public static string HeadTracking = "HeadTracking";
        public static string Controller   = "Controller";
        public static string Broadcast    = "Broadcast";
    }
}


text:
package com.liangdekai.util;

import android.content.Context;

import com.google.gson.Gson;
import com.liangdekai.bean.CityInfo;
import com.liangdekai.bean.FutureWeatherInfo;
import com.liangdekai.bean.JsonCity;
import com.liangdekai.bean.JsonWeather;
import com.liangdekai.bean.TodayInfo;
import com.liangdekai.bean.WeatherId;
import com.liangdekai.bean.WeatherInfo;
import com.liangdekai.db.WeatherDB;

import java.util.ArrayList;
import java.util.List;

/**
 * 对JSON格式数据进行解析,并将结果进行保存
 */
public class HandleResponse {
    /**
     * 对城市的JSON数据进行解析保存
     * @param weatherDB
     * @param result
     * @return boolean
     */
    public static boolean praseCityResponse(WeatherDB weatherDB, String result){
        Gson gson = new Gson();
        String provinceName = "";
        JsonCity jsonCity = gson.fromJson(result , JsonCity.class);
        List<CityInfo> cityInfoList = jsonCity.getCityInfoList();
        for (int i = 0 ; i < cityInfoList.size() ; i++){
            CityInfo cityInfo = cityInfoList.get(i);
            if (!provinceName.equals(cityInfo.getProvince())){
                weatherDB.saveProvince(cityInfo.getProvince());
                provinceName = cityInfo.getProvince();
            }
            weatherDB.saveCity(cityInfo);
        }
        return true;
    }

    /**
     * 对天气的信息进行解析保存
     * @param context
     * @param weatherDB
     * @param result
     * @return boolean
     */
    public static boolean praseWeatherResponse(Context context , WeatherDB weatherDB , String result , String city){
        Gson gson = new Gson();
        JsonWeather jsonWeather = gson.fromJson(result , JsonWeather.class);
        WeatherInfo weatherInfo = jsonWeather.getWeatherInfo();
        TodayInfo todayInfo = weatherInfo.getTodayInfo();
        weatherDB.saveWeather(context , todayInfo);

        List<FutureWeatherInfo> futureWeatherInfoList = weatherInfo.getFutureWeatherInfoList();
        List<FutureWeatherInfo> futureWeatherList = new ArrayList<FutureWeatherInfo>();
        for (int i = 0 ; i < futureWeatherInfoList.size() ; i++){
            FutureWeatherInfo futureWeatherInfo = futureWeatherInfoList.get(i);
            WeatherId weather = futureWeatherInfoList.get(i).getWeatherId();
            futureWeatherInfo.setId(weather.getWeatherId());
            futureWeatherInfo.setCity(city);
            futureWeatherList.add(futureWeatherInfo);
        }
        weatherDB.saveFutureWeather(futureWeatherList);
        return true;
    }

}


text:
"""Multi-platform directory watcher."""

import os

from treadmill.dirwatch.dirwatch_base import DirWatcherEvent

if os.name == 'nt':
    from .windows_dirwatch import WindowsDirWatcher as DirWatcher
else:
    from .linux_dirwatch import LinuxDirWatcher as DirWatcher

__all__ = ['DirWatcherEvent', 'DirWatcher']


text:
 Utah Jazz
Wins: {{utahData.wins}} Losses: {{utahData.losses}}

      Date: {{game.createdAt | date}}

      Utah Jazz - {{game.homeTeamScore}}

      {{game.opponent}} - {{game.opponentScore}}

 Utah Jazz
Wins: {{losangelesData.wins}} Losses: {{losangelesData.losses}}

      Date: {{game.createdAt | date}}

      Utah Jazz - {{game.homeTeamScore}}

      {{game.opponent}} - {{game.opponentScore}}

 Utah Jazz
Wins: {{miamiData.wins}} Losses: {{miamiData.losses}}

      Date: {{game.createdAt | date}}

      Utah Jazz - {{game.homeTeamScore}}

      {{game.opponent}} - {{game.opponentScore}}


text:
package com.android.internal.telephony.cat;

import android.os.Parcel;
import android.os.Parcelable;

/**
 * Class for representing "Duration" object for CAT.
 *
 * {@hide}
 */
public class Duration implements Parcelable {
    public int timeInterval;
    public TimeUnit timeUnit;

    public enum TimeUnit {
        MINUTE(0x00),
        SECOND(0x01),
        TENTH_SECOND(0x02);

        private int mValue;

        TimeUnit(int value) {
            mValue = value;
        }

        public int value() {
            return mValue;
        }
    }

    /**
     * @param timeInterval Between 1 and 255 inclusive.
     */
    public Duration(int timeInterval, TimeUnit timeUnit) {
        this.timeInterval = timeInterval;
        this.timeUnit = timeUnit;
    }

    private Duration(Parcel in) {
        timeInterval = in.readInt();
        timeUnit = TimeUnit.values()[in.readInt()];
    }

    @Override
    public void writeToParcel(Parcel dest, int flags) {
        dest.writeInt(timeInterval);
        dest.writeInt(timeUnit.ordinal());
    }

    @Override
    public int describeContents() {
        return 0;
    }

    public static final Parcelable.Creator<Duration> CREATOR = new Parcelable.Creator<Duration>() {
        @Override
        public Duration createFromParcel(Parcel in) {
            return new Duration(in);
        }

        @Override
        public Duration[] newArray(int size) {
            return new Duration[size];
        }
    };
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using Xamarin.Forms;

namespace SwitchText
{
    public partial class SwitchTextPage : ContentPage
    {
        public SwitchTextPage()
        {
            // Ensure link to Toolkit library.
            new Xamarin.FormsBook.Toolkit.BoolToStringConverter();

            InitializeComponent();
        }
    }
}