text:
package org.elasticsearch.xpack.core.ml.datafeed;

import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.Version;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders;
import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.composite.DateHistogramValuesSourceBuilder;
import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder;
import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
import org.elasticsearch.test.AbstractSerializingTestCase;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xcontent.DeprecationHandler;
import org.elasticsearch.xcontent.NamedXContentRegistry;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentFactory;
import org.elasticsearch.xcontent.XContentParseException;
import org.elasticsearch.xcontent.XContentParser;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig.Mode;
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;

import java.io.IOException;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import static org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigBuilderTests.createRandomizedDatafeedConfigBuilder;
import static org.elasticsearch.xpack.core.ml.job.messages.Messages.DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO;
import static org.elasticsearch.xpack.core.ml.utils.QueryProviderTests.createRandomValidQueryProvider;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;

public class DatafeedConfigTests extends AbstractSerializingTestCase<DatafeedConfig> {

    @Override
    protected DatafeedConfig createTestInstance() {
        return createRandomizedDatafeedConfig(randomAlphaOfLength(10));
    }

    public static DatafeedConfig createRandomizedDatafeedConfig(String jobId) {
        return createRandomizedDatafeedConfig(jobId, 3600000);
    }

    public static DatafeedConfig createRandomizedDatafeedConfig(String jobId, long bucketSpanMillis) {
        return createRandomizedDatafeedConfig(jobId, randomValidDatafeedId(), bucketSpanMillis);
    }

    public static DatafeedConfig createRandomizedDatafeedConfig(String jobId, String datafeedId, long bucketSpanMillis) {
        return createRandomizedDatafeedConfigBuilder(jobId, datafeedId, bucketSpanMillis).build();
    }

    @Override
    protected NamedWriteableRegistry getNamedWriteableRegistry() {
        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
        return new NamedWriteableRegistry(searchModule.getNamedWriteables());
    }

    @Override
    protected NamedXContentRegistry xContentRegistry() {
        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
        return new NamedXContentRegistry(searchModule.getNamedXContents());
    }

    public static List<String> randomStringList(int min, int max) {
        int size = scaledRandomIntBetween(min, max);
        List<String> list = new ArrayList<>();
        for (int i = 0; i < size; i++) {
            list.add(randomAlphaOfLength(10));
        }
        return list;
    }

    @Override
    protected Writeable.Reader<DatafeedConfig> instanceReader() {
        return DatafeedConfig::new;
    }

    @Override
    protected DatafeedConfig doParseInstance(XContentParser parser) {
        return DatafeedConfig.STRICT_PARSER.apply(parser, null).build();
    }

    private static final String FUTURE_DATAFEED = """
        {
            "datafeed_id": "farequote-datafeed",
            "job_id": "farequote",
            "frequency": "1h",
            "indices": ["farequote1", "farequote2"],
            "tomorrows_technology_today": "amazing",
            "scroll_size": 1234
        }""";

    // query:match:type stopped being supported in 6.x
    private static final String ANACHRONISTIC_QUERY_DATAFEED = """
        {
            "datafeed_id": "farequote-datafeed",
            "job_id": "farequote",
            "frequency": "1h",
            "indices": ["farequote1", "farequote2"],
            "query": {"match" : {"query":"fieldName", "type": "phrase"}},
            "scroll_size": 1234
        }""";

    // size: 0 stopped being supported in 6.x
    private static final String ANACHRONISTIC_AGG_DATAFEED = """
        {
            "datafeed_id": "farequote-datafeed",
            "job_id": "farequote",
            "frequency": "1h",
            "indices": ["farequote1", "farequote2"],
            "aggregations": {
            "buckets": {
              "date_histogram": {
                "field": "time",
                "fixed_interval": "360s",
                "time_zone": "UTC"
              },
              "aggregations": {
                "time": {
                  "max": {"field": "time"}
                },
                "airline": {
                  "terms": {
                    "field": "airline",
                    "size": 0
                  }
                }
              }
            }
          }
        }""";

    private static final String AGG_WITH_OLD_DATE_HISTOGRAM_INTERVAL = """
        {
            "datafeed_id": "farequote-datafeed",
            "job_id": "farequote",
            "frequency": "1h",
            "indices": ["farequote1", "farequote2"],
            "aggregations": {
            "buckets": {
              "date_histogram": {
                "field": "time",
                "interval": "360s",
                "time_zone": "UTC"
              },
              "aggregations": {
                "time": {
                  "max": {"field": "time"}
                }
              }
            }
          }
        }""";

    private static final String MULTIPLE_AGG_DEF_DATAFEED = """
        {
            "datafeed_id": "farequote-datafeed",
            "job_id": "farequote",
            "frequency": "1h",
            "indices": ["farequote1", "farequote2"],
            "aggregations": {
            "buckets": {
              "date_histogram": {
                "field": "time",
                "fixed_interval": "360s",
                "time_zone": "UTC"
              },
              "aggregations": {
                "time": {
                  "max": {"field": "time"}
                }
              }
            }
          },    "aggs": {
            "buckets2": {
              "date_histogram": {
                "field": "time",
                "fixed_interval": "360s",
                "time_zone": "UTC"
              },
              "aggregations": {
                "time": {
                  "max": {"field": "time"}
                }
              }
            }
          }
        }""";

    public void testFutureConfigParse() throws IOException {
        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
            .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_DATAFEED);
        XContentParseException e = expectThrows(
            XContentParseException.class,
            () -> DatafeedConfig.STRICT_PARSER.apply(parser, null).build()
        );
        assertEquals("[6:5] [datafeed_config] unknown field [tomorrows_technology_today]", e.getMessage());
    }

    public void testPastQueryConfigParse() throws IOException {
        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, ANACHRONISTIC_QUERY_DATAFEED)
        ) {

            DatafeedConfig config = DatafeedConfig.LENIENT_PARSER.apply(parser, null).build();
            assertThat(
                config.getQueryParsingException().getMessage(),
                equalTo("[match] query doesn't support multiple fields, found [query] and [type]")
            );
        }

        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, ANACHRONISTIC_QUERY_DATAFEED)
        ) {

            XContentParseException e = expectThrows(
                XContentParseException.class,
                () -> DatafeedConfig.STRICT_PARSER.apply(parser, null).build()
            );
            assertEquals("[6:64] [datafeed_config] failed to parse field [query]", e.getMessage());
        }
    }

    public void testPastAggConfigParse() throws IOException {
        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, ANACHRONISTIC_AGG_DATAFEED)
        ) {

            DatafeedConfig datafeedConfig = DatafeedConfig.LENIENT_PARSER.apply(parser, null).build();
            assertThat(
                datafeedConfig.getAggParsingException().getMessage(),
                equalTo("[size] must be greater than 0. Found [0] in [airline]")
            );
        }

        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, ANACHRONISTIC_AGG_DATAFEED)
        ) {

            XContentParseException e = expectThrows(
                XContentParseException.class,
                () -> DatafeedConfig.STRICT_PARSER.apply(parser, null).build()
            );
            assertEquals("[25:3] [datafeed_config] failed to parse field [aggregations]", e.getMessage());
        }
    }

    public void testPastAggConfigOldDateHistogramParse() throws IOException {
        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, AGG_WITH_OLD_DATE_HISTOGRAM_INTERVAL)
        ) {

            DatafeedConfig datafeedConfig = DatafeedConfig.LENIENT_PARSER.apply(parser, null).build();
            assertThat(datafeedConfig.getParsedAggregations(xContentRegistry()), is(not(nullValue())));
        }

        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, ANACHRONISTIC_AGG_DATAFEED)
        ) {

            XContentParseException e = expectThrows(
                XContentParseException.class,
                () -> DatafeedConfig.STRICT_PARSER.apply(parser, null).build()
            );
            assertEquals("[25:3] [datafeed_config] failed to parse field [aggregations]", e.getMessage());
        }
    }

    public void testFutureMetadataParse() throws IOException {
        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
            .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, FUTURE_DATAFEED);
        // Unlike the config version of this test, the metadata parser should tolerate the unknown future field
        assertNotNull(DatafeedConfig.LENIENT_PARSER.apply(parser, null).build());
    }

    public void testMultipleDefinedAggParse() throws IOException {
        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, MULTIPLE_AGG_DEF_DATAFEED)
        ) {
            XContentParseException ex = expectThrows(XContentParseException.class, () -> DatafeedConfig.LENIENT_PARSER.apply(parser, null));
            assertThat(ex.getMessage(), equalTo("[32:3] [datafeed_config] failed to parse field [aggs]"));
            assertNotNull(ex.getCause());
            assertThat(ex.getCause().getMessage(), equalTo("Found two aggregation definitions: [aggs] and [aggregations]"));
        }
        try (
            XContentParser parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, MULTIPLE_AGG_DEF_DATAFEED)
        ) {
            XContentParseException ex = expectThrows(XContentParseException.class, () -> DatafeedConfig.STRICT_PARSER.apply(parser, null));
            assertThat(ex.getMessage(), equalTo("[32:3] [datafeed_config] failed to parse field [aggs]"));
            assertNotNull(ex.getCause());
            assertThat(ex.getCause().getMessage(), equalTo("Found two aggregation definitions: [aggs] and [aggregations]"));
        }
    }

    public void testToXContentForInternalStorage() throws IOException {
        DatafeedConfig.Builder builder = createRandomizedDatafeedConfigBuilder("foo", randomValidDatafeedId(), 300);

        // headers are only persisted to cluster state
        Map<String, String> headers = new HashMap<>();
        headers.put("header-name", "header-value");
        builder.setHeaders(headers);
        DatafeedConfig config = builder.build();

        ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(ToXContentParams.FOR_INTERNAL_STORAGE, "true"));

        BytesReference forClusterstateXContent = XContentHelper.toXContent(config, XContentType.JSON, params, false);
        XContentParser parser = XContentFactory.xContent(XContentType.JSON)
            .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, forClusterstateXContent.streamInput());

        DatafeedConfig parsedConfig = DatafeedConfig.LENIENT_PARSER.apply(parser, null).build();
        assertThat(parsedConfig.getHeaders(), hasEntry("header-name", "header-value"));

        // headers are not written without the FOR_INTERNAL_STORAGE param
        BytesReference nonClusterstateXContent = XContentHelper.toXContent(config, XContentType.JSON, ToXContent.EMPTY_PARAMS, false);
        parser = XContentFactory.xContent(XContentType.JSON)
            .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, nonClusterstateXContent.streamInput());

        parsedConfig = DatafeedConfig.LENIENT_PARSER.apply(parser, null).build();
        assertThat(parsedConfig.getHeaders().entrySet(), hasSize(0));
    }

    public void testCopyConstructor() {
        for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) {
            DatafeedConfig datafeedConfig = createTestInstance();
            DatafeedConfig copy = new DatafeedConfig.Builder(datafeedConfig).build();
            assertEquals(datafeedConfig, copy);
        }
    }

    public void testDefaults() {
        DatafeedConfig.Builder expectedDatafeedConfig = new DatafeedConfig.Builder("datafeed1", "job1");
        expectedDatafeedConfig.setIndices(Collections.singletonList("index"));
        expectedDatafeedConfig.setQueryDelay(TimeValue.timeValueMinutes(1));
        expectedDatafeedConfig.setScrollSize(1000);
        DatafeedConfig.Builder defaultFeedBuilder = new DatafeedConfig.Builder("datafeed1", "job1");
        defaultFeedBuilder.setIndices(Collections.singletonList("index"));
        DatafeedConfig defaultFeed = defaultFeedBuilder.build();

        assertThat(defaultFeed.getScrollSize(), equalTo(1000));
        assertThat(defaultFeed.getQueryDelay().seconds(), greaterThanOrEqualTo(60L));
        assertThat(defaultFeed.getQueryDelay().seconds(), lessThan(120L));
        assertThat(defaultFeed.getMaxEmptySearches(), is(nullValue()));
    }

    public void testDefaultQueryDelay() {
        DatafeedConfig.Builder feedBuilder1 = new DatafeedConfig.Builder("datafeed1", "job1");
        feedBuilder1.setIndices(Collections.singletonList("foo"));
        DatafeedConfig.Builder feedBuilder2 = new DatafeedConfig.Builder("datafeed2", "job1");
        feedBuilder2.setIndices(Collections.singletonList("foo"));
        DatafeedConfig.Builder feedBuilder3 = new DatafeedConfig.Builder("datafeed3", "job2");
        feedBuilder3.setIndices(Collections.singletonList("foo"));
        DatafeedConfig feed1 = feedBuilder1.build();
        DatafeedConfig feed2 = feedBuilder2.build();
        DatafeedConfig feed3 = feedBuilder3.build();

        // Two datafeeds with the same job id should have the same random query delay
        assertThat(feed1.getQueryDelay(), equalTo(feed2.getQueryDelay()));
        // But the query delay of a datafeed with a different job id should differ too
        assertThat(feed1.getQueryDelay(), not(equalTo(feed3.getQueryDelay())));
    }

    public void testCheckValid_GivenNullIndices() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        expectThrows(IllegalArgumentException.class, () -> conf.setIndices(null));
    }

    public void testCheckValid_GivenInvalidMaxEmptySearches() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        ElasticsearchStatusException e = expectThrows(
            ElasticsearchStatusException.class,
            () -> conf.setMaxEmptySearches(randomFrom(-2, 0))
        );
        assertThat(e.getMessage(), containsString("Invalid max_empty_searches value"));
    }

    public void testCheckValid_GivenMaxEmptySearchesMinusOne() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        conf.setIndices(Collections.singletonList("whatever"));
        conf.setMaxEmptySearches(-1);
        assertThat(conf.build().getMaxEmptySearches(), is(nullValue()));
    }

    public void testCheckValid_GivenEmptyIndices() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        conf.setIndices(Collections.emptyList());
        ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, conf::build);
        assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indices", "[]"), e.getMessage());
    }

    public void testCheckValid_GivenIndicesContainsOnlyNulls() {
        List<String> indices = new ArrayList<>();
        indices.add(null);
        indices.add(null);
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        conf.setIndices(indices);
        ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, conf::build);
        assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indices", "[null, null]"), e.getMessage());
    }

    public void testCheckValid_GivenIndicesContainsOnlyEmptyStrings() {
        List<String> indices = new ArrayList<>();
        indices.add("");
        indices.add("");
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        conf.setIndices(indices);
        ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, conf::build);
        assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indices", "[, ]"), e.getMessage());
    }

    public void testCheckValid_GivenNegativeQueryDelay() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        IllegalArgumentException e = ESTestCase.expectThrows(
            IllegalArgumentException.class,
            () -> conf.setQueryDelay(TimeValue.timeValueMillis(-1))
        );
        assertEquals("query_delay cannot be less than 0. Value = -1", e.getMessage());
    }

    public void testCheckValid_GivenZeroFrequency() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setFrequency(TimeValue.ZERO));
        assertEquals("frequency cannot be less or equal than 0. Value = 0s", e.getMessage());
    }

    public void testCheckValid_GivenNegativeFrequency() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        IllegalArgumentException e = ESTestCase.expectThrows(
            IllegalArgumentException.class,
            () -> conf.setFrequency(TimeValue.timeValueMinutes(-1))
        );
        assertEquals("frequency cannot be less or equal than 0. Value = -1", e.getMessage());
    }

    public void testCheckValid_GivenNegativeScrollSize() {
        DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1");
        ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, () -> conf.setScrollSize(-1000));
        assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "scroll_size", -1000L), e.getMessage());
    }

    public void testBuild_GivenScriptFieldsAndAggregations() {
        DatafeedConfig.Builder datafeed = new DatafeedConfig.Builder("datafeed1", "job1");
        datafeed.setIndices(Collections.singletonList("my_index"));
        datafeed.setScriptFields(
            Collections.singletonList(
                new SearchSourceBuilder.ScriptField(randomAlphaOfLength(10), mockScript(randomAlphaOfLength(10)), randomBoolean())
            )
        );
        datafeed.setParsedAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("foo")));

        ElasticsearchException e = expectThrows(ElasticsearchException.class, datafeed::build);

        assertThat(e.getMessage(), equalTo("script_fields cannot be used in combination with aggregations"));
    }

    public void testBuild_GivenRuntimeMappingMissingType() {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("my_index"));
        Map<String, Object> properties = new HashMap<>();
        properties.put("type_field_is_missing", "");
        Map<String, Object> fields = new HashMap<>();
        fields.put("runtime_field_foo", properties);
        builder.setRuntimeMappings(fields);

        ElasticsearchException e = expectThrows(ElasticsearchException.class, builder::build);
        assertThat(e.getMessage(), equalTo("No type specified for runtime field [runtime_field_foo]"));
    }

    public void testBuild_GivenInvalidRuntimeMapping() {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("my_index"));
        Map<String, Object> fields = new HashMap<>();
        fields.put("field_is_not_an_object", "");
        builder.setRuntimeMappings(fields);
        ElasticsearchException e = expectThrows(ElasticsearchException.class, builder::build);
        assertThat(e.getMessage(), equalTo("Expected map for runtime field [field_is_not_an_object] definition but got a String"));
    }

    public void testHasAggregations_GivenNull() {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("myIndex"));
        DatafeedConfig datafeedConfig = builder.build();

        assertThat(datafeedConfig.hasAggregations(), is(false));
    }

    public void testHasAggregations_NonEmpty() {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("myIndex"));
        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
        builder.setParsedAggregations(
            new AggregatorFactories.Builder().addAggregator(
                AggregationBuilders.dateHistogram("time")
                    .fixedInterval(new DateHistogramInterval(300000 + "ms"))
                    .subAggregation(maxTime)
                    .field("time")
            )
        );
        DatafeedConfig datafeedConfig = builder.build();

        assertThat(datafeedConfig.hasAggregations(), is(true));
    }

    public void testBuild_GivenEmptyAggregations() {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("myIndex"));
        builder.setParsedAggregations(new AggregatorFactories.Builder());

        ElasticsearchException e = expectThrows(ElasticsearchException.class, builder::build);

        assertThat(e.getMessage(), equalTo("A date_histogram (or histogram) aggregation is required"));
    }

    public void testBuild_GivenHistogramWithDefaultInterval() {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("myIndex"));
        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
        builder.setParsedAggregations(
            new AggregatorFactories.Builder().addAggregator(AggregationBuilders.histogram("time").subAggregation(maxTime).field("time"))
        );

        ElasticsearchException e = expectThrows(ElasticsearchException.class, builder::build);

        assertThat(e.getMessage(), containsString(DATAFEED_AGGREGATIONS_INTERVAL_MUST_BE_GREATER_THAN_ZERO));
    }

    public void testBuild_GivenDateHistogramWithInvalidTimeZone() {
        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
        DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("bucket")
            .field("time")
            .fixedInterval(new DateHistogramInterval("30000ms"))
            .timeZone(ZoneId.of("CET"))
            .subAggregation(maxTime);
        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> createDatafeedWithDateHistogram(dateHistogram));

        assertThat(e.getMessage(), equalTo("ML requires date_histogram.time_zone to be UTC"));
    }

    public void testBuild_GivenValidDateHistogram() {
        long millisInDay = 24 * 3600000L;

        assertThat(createDatafeedWithDateHistogram("1s").getHistogramIntervalMillis(xContentRegistry()), equalTo(1000L));
        assertThat(createDatafeedWithDateHistogram("2s").getHistogramIntervalMillis(xContentRegistry()), equalTo(2000L));
        assertThat(createDatafeedWithDateHistogram("1m").getHistogramIntervalMillis(xContentRegistry()), equalTo(60000L));
        assertThat(createDatafeedWithDateHistogram("2m").getHistogramIntervalMillis(xContentRegistry()), equalTo(120000L));
        assertThat(createDatafeedWithDateHistogram("1h").getHistogramIntervalMillis(xContentRegistry()), equalTo(3600000L));
        assertThat(createDatafeedWithDateHistogram("2h").getHistogramIntervalMillis(xContentRegistry()), equalTo(7200000L));
        assertThat(createDatafeedWithDateHistogram("1d").getHistogramIntervalMillis(xContentRegistry()), equalTo(millisInDay));
        assertThat(createDatafeedWithDateHistogram("7d").getHistogramIntervalMillis(xContentRegistry()), equalTo(7 * millisInDay));

        assertThat(
            createDatafeedWithDateHistogram(7 * millisInDay + 1).getHistogramIntervalMillis(xContentRegistry()),
            equalTo(7 * millisInDay + 1)
        );
    }

    public void testBuild_GivenDateHistogramWithMoreThanCalendarWeek() {
        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> createDatafeedWithDateHistogram("month"));

        assertThat(e.getMessage(), containsString("When specifying a date_histogram calendar interval [month]"));
    }

    public void testDefaultChunkingConfig_GivenAggregations() {
        assertThat(
            createDatafeedWithDateHistogram("1s").getChunkingConfig(),
            equalTo(ChunkingConfig.newManual(TimeValue.timeValueSeconds(1000)))
        );
        assertThat(
            createDatafeedWithDateHistogram("2h").getChunkingConfig(),
            equalTo(ChunkingConfig.newManual(TimeValue.timeValueHours(2000)))
        );
    }

    public void testChunkingConfig_GivenExplicitSetting() {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder(createDatafeedWithDateHistogram("30s"));
        builder.setChunkingConfig(ChunkingConfig.newAuto());
        assertThat(builder.build().getChunkingConfig(), equalTo(ChunkingConfig.newAuto()));
    }

    public void testCheckHistogramAggregationHasChildMaxTimeAgg() {
        DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("time_agg").field("max_time");

        ElasticsearchException e = expectThrows(
            ElasticsearchException.class,
            () -> DatafeedConfig.Builder.checkHistogramAggregationHasChildMaxTimeAgg(dateHistogram)
        );

        assertThat(e.getMessage(), containsString("Date histogram must have nested max aggregation for time_field [max_time]"));
    }

    public void testValidateCompositeAggValueSources_MustHaveExactlyOneDateValue() {
        CompositeAggregationBuilder aggregationBuilder = AggregationBuilders.composite(
            "buckets",
            Arrays.asList(new TermsValuesSourceBuilder("foo").field("bar"))
        );
        ElasticsearchStatusException ex = expectThrows(
            ElasticsearchStatusException.class,
            () -> DatafeedConfig.Builder.validateCompositeAggregationSources(aggregationBuilder)
        );
        assertThat(ex.getMessage(), containsString("must have exactly one date_histogram source"));

        CompositeAggregationBuilder aggregationBuilderWithMoreDateHisto = AggregationBuilders.composite(
            "buckets",
            Arrays.asList(
                new TermsValuesSourceBuilder("foo").field("bar"),
                new DateHistogramValuesSourceBuilder("date1").field("time").fixedInterval(DateHistogramInterval.days(1)),
                new DateHistogramValuesSourceBuilder("date2").field("time").fixedInterval(DateHistogramInterval.days(1))
            )
        );
        ex = expectThrows(
            ElasticsearchStatusException.class,
            () -> DatafeedConfig.Builder.validateCompositeAggregationSources(aggregationBuilderWithMoreDateHisto)
        );
        assertThat(ex.getMessage(), containsString("must have exactly one date_histogram source"));
    }

    public void testValidateCompositeAggValueSources_DateHistoWithMissingBucket() {
        CompositeAggregationBuilder aggregationBuilder = AggregationBuilders.composite(
            "buckets",
            Arrays.asList(
                new TermsValuesSourceBuilder("foo").field("bar"),
                new DateHistogramValuesSourceBuilder("date1").field("time").fixedInterval(DateHistogramInterval.days(1)).missingBucket(true)
            )
        );
        ElasticsearchStatusException ex = expectThrows(
            ElasticsearchStatusException.class,
            () -> DatafeedConfig.Builder.validateCompositeAggregationSources(aggregationBuilder)
        );
        assertThat(ex.getMessage(), containsString("does not support missing_buckets"));
    }

    public void testValidateCompositeAggValueSources_DateHistoBadOrder() {
        CompositeAggregationBuilder aggregationBuilder = AggregationBuilders.composite(
            "buckets",
            Arrays.asList(
                new TermsValuesSourceBuilder("foo").field("bar"),
                new DateHistogramValuesSourceBuilder("date1").field("time").fixedInterval(DateHistogramInterval.days(1)).order("desc")
            )
        );
        ElasticsearchStatusException ex = expectThrows(
            ElasticsearchStatusException.class,
            () -> DatafeedConfig.Builder.validateCompositeAggregationSources(aggregationBuilder)
        );
        assertThat(ex.getMessage(), containsString("must be sorted in ascending order"));
    }

    public void testValidateAggregations_GivenMulitpleHistogramAggs() {
        DateHistogramAggregationBuilder nestedDateHistogram = AggregationBuilders.dateHistogram("nested_time");
        AvgAggregationBuilder avg = AggregationBuilders.avg("avg").subAggregation(nestedDateHistogram);
        TermsAggregationBuilder nestedTerms = AggregationBuilders.terms("nested_terms");

        DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("time");

        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
        dateHistogram.subAggregation(avg).subAggregation(nestedTerms).subAggregation(maxTime).field("time");

        TermsAggregationBuilder toplevelTerms = AggregationBuilders.terms("top_level");
        toplevelTerms.subAggregation(dateHistogram);

        ElasticsearchException e = expectThrows(
            ElasticsearchException.class,
            () -> DatafeedConfig.validateAggregations(new AggregatorFactories.Builder().addAggregator(toplevelTerms))
        );

        assertEquals("Aggregations can only have 1 date_histogram or histogram aggregation", e.getMessage());
    }

    public void testDefaultFrequency_GivenNegative() {
        DatafeedConfig datafeed = createTestInstance();
        ESTestCase.expectThrows(
            IllegalArgumentException.class,
            () -> datafeed.defaultFrequency(TimeValue.timeValueSeconds(-1), xContentRegistry())
        );
    }

    public void testDefaultFrequency_GivenNoAggregations() {
        DatafeedConfig.Builder datafeedBuilder = new DatafeedConfig.Builder("feed", "job");
        datafeedBuilder.setIndices(Collections.singletonList("my_index"));
        DatafeedConfig datafeed = datafeedBuilder.build();

        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(30), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(60), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(90), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(120), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(121), xContentRegistry()));

        assertEquals(TimeValue.timeValueSeconds(61), datafeed.defaultFrequency(TimeValue.timeValueSeconds(122), xContentRegistry()));
        assertEquals(TimeValue.timeValueSeconds(75), datafeed.defaultFrequency(TimeValue.timeValueSeconds(150), xContentRegistry()));
        assertEquals(TimeValue.timeValueSeconds(150), datafeed.defaultFrequency(TimeValue.timeValueSeconds(300), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1200), xContentRegistry()));

        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1201), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1800), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(1), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(2), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(12), xContentRegistry()));

        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(12 * 3600 + 1), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(13), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(24), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(48), xContentRegistry()));
    }

    public void testDefaultFrequency_GivenAggregationsWithHistogramOrCompositeInterval_1_Second() {
        DatafeedConfig datafeed = randomBoolean() ? createDatafeedWithDateHistogram("1s") : createDatafeedWithCompositeAgg("1s");

        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(60), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(90), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(120), xContentRegistry()));
        assertEquals(TimeValue.timeValueSeconds(125), datafeed.defaultFrequency(TimeValue.timeValueSeconds(250), xContentRegistry()));
        assertEquals(TimeValue.timeValueSeconds(250), datafeed.defaultFrequency(TimeValue.timeValueSeconds(500), xContentRegistry()));

        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(1), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(13), xContentRegistry()));
    }

    public void testDefaultFrequency_GivenAggregationsWithHistogramOrCompositeInterval_1_Minute() {
        DatafeedConfig datafeed = randomBoolean() ? createDatafeedWithDateHistogram("1m") : createDatafeedWithCompositeAgg("1m");

        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(60), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(90), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(120), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(180), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(2), datafeed.defaultFrequency(TimeValue.timeValueSeconds(240), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(20), xContentRegistry()));

        assertEquals(
            TimeValue.timeValueMinutes(10),
            datafeed.defaultFrequency(TimeValue.timeValueSeconds(20 * 60 + 1), xContentRegistry())
        );
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(6), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueHours(12), xContentRegistry()));

        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(13), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(72), xContentRegistry()));
    }

    public void testDefaultFrequency_GivenAggregationsWithHistogramOrCompositeInterval_10_Minutes() {
        DatafeedConfig datafeed = randomBoolean() ? createDatafeedWithDateHistogram("10m") : createDatafeedWithCompositeAgg("10m");
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(10), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(20), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(30), xContentRegistry()));
        assertEquals(TimeValue.timeValueMinutes(10), datafeed.defaultFrequency(TimeValue.timeValueMinutes(12 * 60), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueMinutes(13 * 60), xContentRegistry()));
    }

    public void testDefaultFrequency_GivenAggregationsWithHistogramOrCompositeInterval_1_Hour() {
        DatafeedConfig datafeed = randomBoolean() ? createDatafeedWithDateHistogram("1h") : createDatafeedWithCompositeAgg("1h");
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(1), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(3601), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(2), xContentRegistry()));
        assertEquals(TimeValue.timeValueHours(1), datafeed.defaultFrequency(TimeValue.timeValueHours(12), xContentRegistry()));
    }

    public void testSerializationOfComplexAggs() throws IOException {
        MaxAggregationBuilder maxTime = AggregationBuilders.max("timestamp").field("timestamp");
        AvgAggregationBuilder avgAggregationBuilder = AggregationBuilders.avg("bytes_in_avg").field("system.network.in.bytes");
        DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = PipelineAggregatorBuilders.derivative(
            "bytes_in_derivative",
            "bytes_in_avg"
        );
        BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = PipelineAggregatorBuilders.bucketScript(
            "non_negative_bytes",
            Collections.singletonMap("bytes", "bytes_in_derivative"),
            new Script("params.bytes > 0 ? params.bytes : null")
        );
        DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("histogram_buckets")
            .field("timestamp")
            .fixedInterval(new DateHistogramInterval("300000ms"))
            .timeZone(ZoneOffset.UTC)
            .subAggregation(maxTime)
            .subAggregation(avgAggregationBuilder)
            .subAggregation(derivativePipelineAggregationBuilder)
            .subAggregation(bucketScriptPipelineAggregationBuilder);
        DatafeedConfig.Builder datafeedConfigBuilder = createDatafeedBuilderWithDateHistogram(dateHistogram);
        datafeedConfigBuilder.setQueryProvider(
            createRandomValidQueryProvider(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))
        );
        DatafeedConfig datafeedConfig = datafeedConfigBuilder.build();
        AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder().addAggregator(dateHistogram);

        XContentType xContentType = XContentType.JSON;
        BytesReference bytes = XContentHelper.toXContent(datafeedConfig, xContentType, false);
        XContentParser parser = XContentHelper.createParser(
            xContentRegistry(),
            DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
            bytes,
            xContentType
        );

        DatafeedConfig parsedDatafeedConfig = doParseInstance(parser);
        assertEquals(datafeedConfig, parsedDatafeedConfig);

        // Assert that the parsed versions of our aggs and queries work as well
        assertEquals(aggBuilder, parsedDatafeedConfig.getParsedAggregations(xContentRegistry()));
        assertEquals(datafeedConfig.getQuery(), parsedDatafeedConfig.getQuery());

        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());
        try (BytesStreamOutput output = new BytesStreamOutput()) {
            datafeedConfig.writeTo(output);
            try (StreamInput streamInput = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                DatafeedConfig streamedDatafeedConfig = new DatafeedConfig(streamInput);
                assertEquals(datafeedConfig, streamedDatafeedConfig);

                // Assert that the parsed versions of our aggs and queries work as well
                assertEquals(aggBuilder, streamedDatafeedConfig.getParsedAggregations(xContentRegistry()));
                assertEquals(datafeedConfig.getQuery(), streamedDatafeedConfig.getQuery());
            }
        }
    }

    public void testSerializationOfComplexAggsBetweenVersions() throws IOException {
        MaxAggregationBuilder maxTime = AggregationBuilders.max("timestamp").field("timestamp");
        AvgAggregationBuilder avgAggregationBuilder = AggregationBuilders.avg("bytes_in_avg").field("system.network.in.bytes");
        DerivativePipelineAggregationBuilder derivativePipelineAggregationBuilder = PipelineAggregatorBuilders.derivative(
            "bytes_in_derivative",
            "bytes_in_avg"
        );
        BucketScriptPipelineAggregationBuilder bucketScriptPipelineAggregationBuilder = PipelineAggregatorBuilders.bucketScript(
            "non_negative_bytes",
            Collections.singletonMap("bytes", "bytes_in_derivative"),
            new Script("params.bytes > 0 ? params.bytes : null")
        );
        DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("histogram_buckets")
            .field("timestamp")
            .fixedInterval(new DateHistogramInterval("30000ms"))
            .timeZone(ZoneOffset.UTC)
            .subAggregation(maxTime)
            .subAggregation(avgAggregationBuilder)
            .subAggregation(derivativePipelineAggregationBuilder)
            .subAggregation(bucketScriptPipelineAggregationBuilder);
        DatafeedConfig.Builder datafeedConfigBuilder = createDatafeedBuilderWithDateHistogram(dateHistogram);
        // So equality check between the streamed and current passes
        // Streamed DatafeedConfigs when they are before 6.6.0 require a parsed object for aggs and queries, consequently all the default
        // values are added between them
        datafeedConfigBuilder.setQueryProvider(
            QueryProvider.fromParsedQuery(
                QueryBuilders.boolQuery()
                    .filter(QueryBuilders.termQuery(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)))
            )
        );
        DatafeedConfig datafeedConfig = datafeedConfigBuilder.build();

        SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(searchModule.getNamedWriteables());

        try (BytesStreamOutput output = new BytesStreamOutput()) {
            output.setVersion(Version.CURRENT);
            datafeedConfig.writeTo(output);
            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                in.setVersion(Version.CURRENT);
                DatafeedConfig streamedDatafeedConfig = new DatafeedConfig(in);
                assertEquals(datafeedConfig, streamedDatafeedConfig);

                // Assert that the parsed versions of our aggs and queries work as well
                assertEquals(
                    new AggregatorFactories.Builder().addAggregator(dateHistogram),
                    streamedDatafeedConfig.getParsedAggregations(xContentRegistry())
                );
                assertEquals(datafeedConfig.getParsedQuery(xContentRegistry()), streamedDatafeedConfig.getParsedQuery(xContentRegistry()));
            }
        }
    }

    public void testCopyingDatafeedDoesNotCauseStackOverflow() {
        DatafeedConfig datafeed = createTestInstance();
        for (int i = 0; i < 100000; i++) {
            datafeed = new DatafeedConfig.Builder(datafeed).build();
        }
    }

    public static String randomValidDatafeedId() {
        CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray());
        return generator.ofCodePointsLength(random(), 10, 10);
    }

    private static DatafeedConfig createDatafeedWithCompositeAgg(String interval) {
        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
        DateHistogramValuesSourceBuilder sourceBuilder = new DateHistogramValuesSourceBuilder("time");
        sourceBuilder.field("time");
        if (interval != null) {
            if (DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(interval) != null) {
                sourceBuilder.calendarInterval(new DateHistogramInterval(interval));
            } else {
                sourceBuilder.fixedInterval(new DateHistogramInterval(interval));
            }
        }
        CompositeAggregationBuilder composite = AggregationBuilders.composite("buckets", Arrays.asList(sourceBuilder))
            .subAggregation(maxTime);
        return createDatafeedWithComposite(composite);
    }

    private static DatafeedConfig createDatafeedWithDateHistogram(String interval) {
        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
        DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("buckets").subAggregation(maxTime).field("time");
        if (interval != null) {
            if (DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(interval) != null) {
                dateHistogram.calendarInterval(new DateHistogramInterval(interval));
            } else {
                dateHistogram.fixedInterval(new DateHistogramInterval(interval));
            }
        }
        return createDatafeedWithDateHistogram(dateHistogram);
    }

    private static DatafeedConfig createDatafeedWithDateHistogram(Long interval) {
        MaxAggregationBuilder maxTime = AggregationBuilders.max("time").field("time");
        DateHistogramAggregationBuilder dateHistogram = AggregationBuilders.dateHistogram("buckets").subAggregation(maxTime).field("time");
        if (interval != null) {
            dateHistogram.fixedInterval(new DateHistogramInterval(interval + "ms"));
        }
        return createDatafeedWithDateHistogram(dateHistogram);
    }

    private static DatafeedConfig.Builder createDatafeedBuilderWithDateHistogram(DateHistogramAggregationBuilder dateHistogram) {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("myIndex"));
        AggregatorFactories.Builder aggs = new AggregatorFactories.Builder().addAggregator(dateHistogram);
        DatafeedConfig.validateAggregations(aggs);
        builder.setParsedAggregations(aggs);
        return builder;
    }

    private static DatafeedConfig createDatafeedWithDateHistogram(DateHistogramAggregationBuilder dateHistogram) {
        return createDatafeedBuilderWithDateHistogram(dateHistogram).build();
    }

    private static DatafeedConfig.Builder createDatafeedBuilderWithComposite(CompositeAggregationBuilder compositeAggregationBuilder) {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder("datafeed1", "job1");
        builder.setIndices(Collections.singletonList("myIndex"));
        AggregatorFactories.Builder aggs = new AggregatorFactories.Builder().addAggregator(compositeAggregationBuilder);
        DatafeedConfig.validateAggregations(aggs);
        builder.setParsedAggregations(aggs);
        return builder;
    }

    private static DatafeedConfig createDatafeedWithComposite(CompositeAggregationBuilder dateHistogram) {
        return createDatafeedBuilderWithComposite(dateHistogram).build();
    }

    @Override
    protected DatafeedConfig mutateInstance(DatafeedConfig instance) throws IOException {
        DatafeedConfig.Builder builder = new DatafeedConfig.Builder(instance);
        switch (between(0, 12)) {
            case 0:
                builder.setId(instance.getId() + randomValidDatafeedId());
                break;
            case 1:
                builder.setJobId(instance.getJobId() + randomAlphaOfLength(5));
                break;
            case 2:
                builder.setQueryDelay(new TimeValue(instance.getQueryDelay().millis() + between(100, 100000)));
                break;
            case 3:
                if (instance.getFrequency() == null) {
                    builder.setFrequency(new TimeValue(between(1, 10) * 1000));
                } else {
                    builder.setFrequency(new TimeValue(instance.getFrequency().millis() + between(1, 10) * 1000));
                }
                break;
            case 4:
                List<String> indices = new ArrayList<>(instance.getIndices());
                indices.add(randomAlphaOfLengthBetween(1, 20));
                builder.setIndices(indices);
                break;
            case 5:
                BoolQueryBuilder query = new BoolQueryBuilder();
                if (instance.getParsedQuery(xContentRegistry()) != null) {
                    query.must(instance.getParsedQuery(xContentRegistry()));
                }
                query.filter(new TermQueryBuilder(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)));
                builder.setParsedQuery(query);
                break;
            case 6:
                if (instance.hasAggregations()) {
                    builder.setAggProvider(null);
                } else {
                    AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder();
                    String timeField = randomAlphaOfLength(10);
                    long fixedInterval = between(10000, 3600000);

                    aggBuilder.addAggregator(
                        new DateHistogramAggregationBuilder(timeField).field(timeField)
                            .fixedInterval(new DateHistogramInterval(fixedInterval + "ms"))
                            .subAggregation(new MaxAggregationBuilder(timeField).field(timeField))
                    );
                    builder.setParsedAggregations(aggBuilder);
                    if (instance.getScriptFields().isEmpty() == false) {
                        builder.setScriptFields(Collections.emptyList());
                    }
                }
                break;
            case 7:
                builder.setScriptFields(
                    CollectionUtils.appendToCopy(
                        instance.getScriptFields(),
                        new ScriptField(randomAlphaOfLengthBetween(1, 10), new Script("foo"), true)
                    )
                );
                builder.setAggProvider(null);
                break;
            case 8:
                builder.setScrollSize(instance.getScrollSize() + between(1, 100));
                break;
            case 9:
                if (instance.getChunkingConfig() == null || instance.getChunkingConfig().getMode() == Mode.AUTO) {
                    ChunkingConfig newChunkingConfig = ChunkingConfig.newManual(new TimeValue(randomNonNegativeLong()));
                    builder.setChunkingConfig(newChunkingConfig);
                } else {
                    builder.setChunkingConfig(ChunkingConfig.newAuto());
                }
                break;
            case 10:
                if (instance.getMaxEmptySearches() == null) {
                    builder.setMaxEmptySearches(randomIntBetween(10, 100));
                } else {
                    builder.setMaxEmptySearches(instance.getMaxEmptySearches() + 1);
                }
                break;
            case 11:
                builder.setIndicesOptions(
                    IndicesOptions.fromParameters(
                        randomFrom(IndicesOptions.WildcardStates.values()).name().toLowerCase(Locale.ROOT),
                        Boolean.toString(instance.getIndicesOptions().ignoreUnavailable() == false),
                        Boolean.toString(instance.getIndicesOptions().allowNoIndices() == false),
                        Boolean.toString(instance.getIndicesOptions().ignoreThrottled() == false),
                        SearchRequest.DEFAULT_INDICES_OPTIONS
                    )
                );
                break;
            case 12:
                if (instance.getRuntimeMappings() != null && instance.getRuntimeMappings().isEmpty() == false) {
                    builder.setRuntimeMappings(Collections.emptyMap());
                } else {
                    Map<String, Object> settings = new HashMap<>();
                    settings.put("type", "keyword");
                    settings.put("script", "");
                    Map<String, Object> field = new HashMap<>();
                    field.put("runtime_field_foo", settings);
                    builder.setRuntimeMappings(field);
                }
                break;
            default:
                throw new AssertionError("Illegal randomisation branch");
        }
        return builder.build();
    }
}


text:
package shuaicj.hello.sshj;

import net.schmizz.sshj.SSHClient;
import net.schmizz.sshj.common.IOUtils;
import net.schmizz.sshj.connection.channel.direct.Session;

import java.io.IOException;
import java.util.concurrent.TimeUnit;

/**
 * Test the library: SSHJ.
 *
 * @author shuaicj 2016/12/28
 */
public class Application {

    private SSHClient ssh = new SSHClient();

    public Application() throws IOException {
        ssh.loadKnownHosts();
        ssh.connect("127.0.0.1", 2222);
        ssh.authPassword("shuai", "shuai");
    }

    public void exec() throws IOException {
//        try (final Session session = ssh.startSession()) {
//            final Session.Command cmd = session.exec("/vagrant/xx aa");
//            final Session.Command cmd = session.exec("java -jar /vagrant/java/xx.jar aa");
//            cmd.join(5, TimeUnit.SECONDS);
//            System.out.println("exit code: " + cmd.getExitStatus());
//            System.out.println("exit msg: " + cmd.getExitErrorMessage());
//            String stdout = IOUtils.readFully(cmd.getInputStream()).toString();
//            String stderr = IOUtils.readFully(cmd.getErrorStream()).toString();
//            System.out.println("stdout: " + stdout);
//            System.out.println("stderr: " + stderr);
//        }
        try (final Session session = ssh.startSession()) {
//            final Session.Command cmd = session.exec("/vagrant/xx");
//            final Session.Command cmd = session.exec("java -jar /vagrant/java/xx.jar");
//            final Session.Command cmd = session.exec("cat /proc/cpuinfo | grep \"processor\" | wc -l");
//            final Session.Command cmd = session.exec("free | grep Mem | awk '{print $4}'");
            final Session.Command cmd = session.exec("/usr/bin/time --quiet --format '%P,%K,%r,%s,%e,%x' /vagrant/xx");
            cmd.join(5, TimeUnit.SECONDS);
            System.out.println("exit code: " + cmd.getExitStatus());
            System.out.println("exit msg: " + cmd.getExitErrorMessage());
            String stdout = IOUtils.readFully(cmd.getInputStream()).toString().trim();
            String stderr = IOUtils.readFully(cmd.getErrorStream()).toString().trim();
            System.out.println("stdout: " + stdout);
            System.out.println("stderr: " + stderr);
        }
    }

    public void close() throws IOException {
        ssh.close();
    }
}

redpajama_set_name: RedPajamaGithub


text:
package com.jbburns.simplecurrencywidget;

import java.util.Date;

/**
 * Created by jbburns on 3/18/2016.
 */
public class SMBCTBRate {
    private Date asOfTime;
    private String counterCurrency;
    private Double sellRate;
    private Double midRate;
    private Double buyRate;
    private Double demandRate;
    private Double spotRateMin;

    public SMBCTBRate(Date asOfTime, String counterCurrency, Double sellRate, Double midRate, Double buyRate, Double demandRate, Double spotRateMin) {
        this.asOfTime = asOfTime;
        this.counterCurrency = counterCurrency;
        this.sellRate = sellRate;
        this.midRate = midRate;
        this.buyRate = buyRate;
        this.demandRate = demandRate;
        this.spotRateMin = spotRateMin;
    }

    public Date getAsOfTime() {
        return asOfTime;
    }

    public String getCounterCurrency() {
        return counterCurrency;
    }

    public Double getSellRate() {
        return sellRate;
    }

    public Double getMidRate() {
        return midRate;
    }

    public Double getBuyRate() {
        return buyRate;
    }

    public Double getDemandRate() {
        return demandRate;
    }

    public Double getSpotRateMin() {
        return spotRateMin;
    }

    @Override
    public String toString() {
        return "As of: " + asOfTime.toString()
                + "\nCounter Currency: " + this.counterCurrency
                + "\n Sell Rate: " + sellRate
                + "\n Mid Rate: " + midRate
                + "\n Buy Rate: " + buyRate
                + "\n Demand Rate: " + demandRate
                + "\n Spot Rate Minimum: " + spotRateMin;
    }
}


text:
function model = dnetReconstruct(mapping, dnetInfo, y)

% DNETRECONSTRUCT Reconstruct an DNET form component parts.
% FORMAT
% DESC takes component parts of an DNET model and reconstructs the
% DNET model. The component parts are normally retrieved from a
% saved file.
% ARG kern : a kernel structure for the DNET.
% ARG noise : a noise structure for the DNET (currently ignored).
% ARG dnetInfo : the active set and other information stored in a structure.
% ARG X : the input training data for the DNET.
% ARG y : the output target training data for the DNET.
% RETURN model : an DNET model structure that combines the component
% parts.
%
% SEEALSO : dnetDeconstruct, dnetCreate
%
% COPYRIGHT : Neil D. Lawrence, 2009

% MLTOOLS

  model = dnetInfo;
  model.mapping = mapping;
  model.y = y;

  params = dnetExtractParam(model);
  model = dnetExpandParam(model, params);
  model = dnetEstep(model);

end


text:
/*----------------------------------------------------------------------------*/
/* Copyright (c) 2008-2017 FIRST. All Rights Reserved.                        */
/* Open Source Software - may be modified and shared by FRC teams. The code   */
/* must be accompanied by the FIRST BSD license file in the root directory of */
/* the project.                                                               */
/*----------------------------------------------------------------------------*/

#include "InterruptableSensorBase.h"

#include <HAL/HAL.h>

#include "Utility.h"
#include "WPIErrors.h"

using namespace frc;

/**
 * Request one of the 8 interrupts asynchronously on this digital input.
 *
 * Request interrupts in asynchronous mode where the user's interrupt handler
 * will be called when the interrupt fires. Users that want control over the
 * thread priority should use the synchronous method with their own spawned
 * thread. The default is interrupt on rising edges only.
 */
void InterruptableSensorBase::RequestInterrupts(
    HAL_InterruptHandlerFunction handler, void* param) {
  if (StatusIsFatal()) return;

  wpi_assert(m_interrupt == HAL_kInvalidHandle);
  AllocateInterrupts(false);
  if (StatusIsFatal()) return;  // if allocate failed, out of interrupts

  int32_t status = 0;
  HAL_RequestInterrupts(
      m_interrupt, GetPortHandleForRouting(),
      static_cast<HAL_AnalogTriggerType>(GetAnalogTriggerTypeForRouting()),
      &status);
  SetUpSourceEdge(true, false);
  HAL_AttachInterruptHandler(m_interrupt, handler, param, &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
}

/**
 * Request one of the 8 interrupts synchronously on this digital input.
 *
 * Request interrupts in synchronous mode where the user program will have to
 * explicitly wait for the interrupt to occur using WaitForInterrupt.
 * The default is interrupt on rising edges only.
 */
void InterruptableSensorBase::RequestInterrupts() {
  if (StatusIsFatal()) return;

  wpi_assert(m_interrupt == HAL_kInvalidHandle);
  AllocateInterrupts(true);
  if (StatusIsFatal()) return;  // if allocate failed, out of interrupts

  int32_t status = 0;
  HAL_RequestInterrupts(
      m_interrupt, GetPortHandleForRouting(),
      static_cast<HAL_AnalogTriggerType>(GetAnalogTriggerTypeForRouting()),
      &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
  SetUpSourceEdge(true, false);
}

void InterruptableSensorBase::AllocateInterrupts(bool watcher) {
  wpi_assert(m_interrupt == HAL_kInvalidHandle);
  // Expects the calling leaf class to allocate an interrupt index.
  int32_t status = 0;
  m_interrupt = HAL_InitializeInterrupts(watcher, &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
}

/**
 * Cancel interrupts on this device.
 *
 * This deallocates all the chipobject structures and disables any interrupts.
 */
void InterruptableSensorBase::CancelInterrupts() {
  if (StatusIsFatal()) return;
  wpi_assert(m_interrupt != HAL_kInvalidHandle);
  int32_t status = 0;
  HAL_CleanInterrupts(m_interrupt, &status);
  // ignore status, as an invalid handle just needs to be ignored.
  m_interrupt = HAL_kInvalidHandle;
}

/**
 * In synchronous mode, wait for the defined interrupt to occur.
 *
 * You should <b>NOT</b> attempt to read the sensor from another thread while
 * waiting for an interrupt. This is not threadsafe, and can cause memory
 * corruption
 *
 * @param timeout        Timeout in seconds
 * @param ignorePrevious If true, ignore interrupts that happened before
 *                       WaitForInterrupt was called.
 * @return What interrupts fired
 */
InterruptableSensorBase::WaitResult InterruptableSensorBase::WaitForInterrupt(
    double timeout, bool ignorePrevious) {
  if (StatusIsFatal()) return InterruptableSensorBase::kTimeout;
  wpi_assert(m_interrupt != HAL_kInvalidHandle);
  int32_t status = 0;
  int result;

  result = HAL_WaitForInterrupt(m_interrupt, timeout, ignorePrevious, &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));

  // Rising edge result is the interrupt bit set in the byte 0xFF
  // Falling edge result is the interrupt bit set in the byte 0xFF00
  // Set any bit set to be true for that edge, and AND the 2 results
  // together to match the existing enum for all interrupts
  int32_t rising = (result & 0xFF) ? 0x1 : 0x0;
  int32_t falling = ((result & 0xFF00) ? 0x0100 : 0x0);
  return static_cast<WaitResult>(falling | rising);
}

/**
 * Enable interrupts to occur on this input.
 *
 * Interrupts are disabled when the RequestInterrupt call is made. This gives
 * time to do the setup of the other options before starting to field
 * interrupts.
 */
void InterruptableSensorBase::EnableInterrupts() {
  if (StatusIsFatal()) return;
  wpi_assert(m_interrupt != HAL_kInvalidHandle);
  int32_t status = 0;
  HAL_EnableInterrupts(m_interrupt, &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
}

/**
 * Disable Interrupts without without deallocating structures.
 */
void InterruptableSensorBase::DisableInterrupts() {
  if (StatusIsFatal()) return;
  wpi_assert(m_interrupt != HAL_kInvalidHandle);
  int32_t status = 0;
  HAL_DisableInterrupts(m_interrupt, &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
}

/**
 * Return the timestamp for the rising interrupt that occurred most recently.
 *
 * This is in the same time domain as GetClock().
 * The rising-edge interrupt should be enabled with
 * {@link #DigitalInput.SetUpSourceEdge}
 *
 * @return Timestamp in seconds since boot.
 */
double InterruptableSensorBase::ReadRisingTimestamp() {
  if (StatusIsFatal()) return 0.0;
  wpi_assert(m_interrupt != HAL_kInvalidHandle);
  int32_t status = 0;
  double timestamp = HAL_ReadInterruptRisingTimestamp(m_interrupt, &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
  return timestamp;
}

/**
 * Return the timestamp for the falling interrupt that occurred most recently.
 *
 * This is in the same time domain as GetClock().
 * The falling-edge interrupt should be enabled with
 * {@link #DigitalInput.SetUpSourceEdge}
 *
 * @return Timestamp in seconds since boot.
*/
double InterruptableSensorBase::ReadFallingTimestamp() {
  if (StatusIsFatal()) return 0.0;
  wpi_assert(m_interrupt != HAL_kInvalidHandle);
  int32_t status = 0;
  double timestamp = HAL_ReadInterruptFallingTimestamp(m_interrupt, &status);
  wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
  return timestamp;
}

/**
 * Set which edge to trigger interrupts on
 *
 * @param risingEdge  true to interrupt on rising edge
 * @param fallingEdge true to interrupt on falling edge
 */
void InterruptableSensorBase::SetUpSourceEdge(bool risingEdge,
                                              bool fallingEdge) {
  if (StatusIsFatal()) return;
  if (m_interrupt == HAL_kInvalidHandle) {
    wpi_setWPIErrorWithContext(
        NullParameter,
        "You must call RequestInterrupts before SetUpSourceEdge");
    return;
  }
  if (m_interrupt != HAL_kInvalidHandle) {
    int32_t status = 0;
    HAL_SetInterruptUpSourceEdge(m_interrupt, risingEdge, fallingEdge, &status);
    wpi_setErrorWithContext(status, HAL_GetErrorMessage(status));
  }
}


text:
package org.svnadmin.service;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import javax.annotation.Resource;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Service;
import org.svnadmin.constant.Constants;
import org.svnadmin.dao.PjAuthDao;
import org.svnadmin.dao.PjDao;
import org.svnadmin.dao.PjGrDao;
import org.svnadmin.dao.PjGrUsrDao;
import org.svnadmin.dao.UsrDao;
import org.svnadmin.entity.Pj;
import org.svnadmin.entity.PjAuth;
import org.svnadmin.entity.PjGrUsr;
import org.svnadmin.entity.Usr;
import org.svnadmin.util.EncryptUtil;
import org.svnadmin.util.I18N;

/**
 * 导出svn配置信息服务层
 *
 * @author Huiwu Yuan
 * @since 1.0
 *
 */
@Service(SvnService.BEAN_NAME)
public class SvnService {
	/**
	 * Bean名称
	 */
	public static final String BEAN_NAME = "svnService";

	/**
	 * 分隔符
	 */
	private static final String SEP = System.getProperty("line.separator");
	/**
	 * 日志
	 */
	private final Logger LOG = Logger.getLogger(this.getClass());

	/**
	 * 项目DAO
	 */
	@Resource(name = PjDao.BEAN_NAME)
	protected PjDao pjDao;

	/**
	 * 用户DAO
	 */
	@Resource(name = UsrDao.BEAN_NAME)
	protected UsrDao usrDao;

	/**
	 * 项目组DAO
	 */
	@Resource(name = PjGrDao.BEAN_NAME)
	protected PjGrDao pjGrDao;

	/**
	 * 项目组用户DAO
	 */
	@Resource(name = PjGrUsrDao.BEAN_NAME)
	protected PjGrUsrDao pjGrUsrDao;

	/**
	 * 项目权限DAO
	 */
	@Resource(name = PjAuthDao.BEAN_NAME)
	protected PjAuthDao pjAuthDao;

	/**
	 * 导出到配置文件
	 *
	 * @param pj
	 *            项目id
	 */
	public synchronized void exportConfig(int pjId) {
		this.exportConfig(this.pjDao.getById(pjId));
	}

	/**
	 * 导出到配置文件
	 *
	 * @param pj
	 *            项目
	 */
	public synchronized void exportConfig(Pj pj) {
		if (pj == null) {
			return;
		}
		File parent = new File(pj.getPath());
		if (!parent.exists() || !parent.isDirectory()) {
			throw new RuntimeException(I18N.getLbl("svn.notFoundResp", "找不到仓库 路径{0}",new Object[]{pj.getPath()}));
		}

		if (Constants.HTTP.equalsIgnoreCase(pj.getType())) {// HTTP(单库) SVNPath
			this.exportHTTP(pj);
		} else if (Constants.HTTP_MUTIL.equalsIgnoreCase(pj.getType())) {// HTTP(多库)
																			// SVNParentPath
			File root = new File(pj.getPath()).getParentFile();
			this.exportHTTPMutil(root);
		} else if (Constants.SVN.equalsIgnoreCase(pj.getType())) {// SVN
			this.exportSVN(pj);
		}

	}

	/**
	 * 导出svn协议的配置信息
	 *
	 * @param pj
	 *            项目
	 */
	private void exportSVN(Pj pj) {
		// 项目的用户
		List<Usr> usrList = this.usrDao.getList(pj.getId());
		// 项目的用户组
		Map<String, List<PjGrUsr>> pjGrUsrMap = this.getPjGrUsrs(pj.getId());
		// 项目的权限
		Map<String, List<PjAuth>> pjAuthMap = this.getPjAuths(pj.getId());

		this.exportSvnConf(pj);
		this.exportPasswdSVN(pj, usrList);
		this.exportAuthz(pj, pjGrUsrMap, pjAuthMap);
	}

	/**
	 * 导出http(单库)的配置信息
	 *
	 * @param pj
	 *            项目
	 */
	private void exportHTTP(Pj pj) {
		// 项目的用户
		List<Usr> usrList = this.usrDao.getList(pj.getId());
		// 项目的用户组
		Map<String, List<PjGrUsr>> pjGrUsrMap = this.getPjGrUsrs(pj.getId());
		// 项目的权限
		Map<String, List<PjAuth>> pjAuthMap = this.getPjAuths(pj.getId());

		this.exportSVNPathConf(pj);
		this.exportPasswdHTTP(pj, usrList);
		this.exportAuthz(pj, pjGrUsrMap, pjAuthMap);
	}

	/**
	 * 导出http(多库)的配置信息
	 *
	 * @param root
	 *            svn root
	 */
	private void exportHTTPMutil(File root) {
		String svnRoot = StringUtils.replace(root.getAbsolutePath(), "\\", "/");
		if (!svnRoot.endsWith("/")) {
			svnRoot += "/";
		}
		// 和这个项目在同一个父目录的所有项目的用户
		List<Usr> usrList = this.usrDao.getListByRootPath(svnRoot);
		// 和这个项目在同一个父目录的所有项目的用户组
		Map<String, List<PjGrUsr>> pjGrUsrMap = this
				.getPjGrUsrsByRootPath(svnRoot);
		// 和这个项目在同一个父目录的所有项目的权限
		Map<String, List<PjAuth>> pjAuthMap = this
				.getPjAuthsByRootPath(svnRoot);

		this.exportSVNParentPathConf(root);

		this.exportPasswdHTTPMutil(root, usrList);

		this.exportAuthzHTTPMutil(root, pjGrUsrMap, pjAuthMap);
	}

	/**
	 * 获取有相同svn root的项目的权限列表
	 *
	 * @param rootPath
	 *            svn root
	 * @return 有相同svn root的项目的权限列表
	 */
	private Map<String, List<PjAuth>> getPjAuthsByRootPath(String rootPath) {
		Map<String, List<PjAuth>> results = new LinkedHashMap<String, List<PjAuth>>();// <res,List<PjAuth>>
		List<PjAuth> pjAuthList = this.pjAuthDao.getListByRootPath(rootPath);
		// 格式化返回数据
		for (PjAuth pjAuth : pjAuthList) {
			List<PjAuth> authList = results.get(pjAuth.getRes());
			if (authList == null) {
				authList = new ArrayList<PjAuth>();
				results.put(pjAuth.getRes(), authList);
			}
			authList.add(pjAuth);

		}
		return results;
	}

	/**
	 * 获取项目的权限列表
	 *
	 * @param pj
	 *            项目
	 * @return 项目的权限列表
	 */
	private Map<String, List<PjAuth>> getPjAuths(int pjId) {
		Map<String, List<PjAuth>> results = new LinkedHashMap<String, List<PjAuth>>();// <res,List<PjAuth>>
		List<PjAuth> pjAuthList = this.pjAuthDao.getList(pjId);
		// 格式化返回数据
		for (PjAuth pjAuth : pjAuthList) {
			List<PjAuth> authList = results.get(pjAuth.getRes());
			if (authList == null) {
				authList = new ArrayList<PjAuth>();
				results.put(pjAuth.getRes(), authList);
			}
			authList.add(pjAuth);

		}
		return results;
	}

	/**
	 * 获取项目的组列表
	 *
	 * @param pj
	 *            项目
	 * @return 项目的组列表
	 */
	private Map<String, List<PjGrUsr>> getPjGrUsrs(int pjId) {
		Map<String, List<PjGrUsr>> results = new LinkedHashMap<String, List<PjGrUsr>>();// <gr,List<PjGrUsr>>

		List<PjGrUsr> pjGrUsrs = this.pjGrUsrDao.getList(pjId);

		// 格式化返回数据
		for (PjGrUsr pjGrUsr : pjGrUsrs) {
			List<PjGrUsr> grUsrList = results.get(pjGrUsr.getGr());
			if (grUsrList == null) {
				grUsrList = new ArrayList<PjGrUsr>();
				results.put(pjGrUsr.getGr(), grUsrList);
			}
			grUsrList.add(pjGrUsr);
		}

		return results;
	}

	/**
	 * 获取有相同svn root的项目的权限列表
	 *
	 * @param rootPath
	 *            svn root
	 * @return 有相同svn root的项目的权限列表
	 */
	private Map<String, List<PjGrUsr>> getPjGrUsrsByRootPath(String rootPath) {

		Map<String, List<PjGrUsr>> results = new LinkedHashMap<String, List<PjGrUsr>>();// <pj_gr,List<PjGrUsr>>

		List<PjGrUsr> pjGrUsrs = this.pjGrUsrDao.getListByRootPath(rootPath);

		// 格式化返回数据
		for (PjGrUsr pjGrUsr : pjGrUsrs) {
			String key = pjGrUsr.getPjId() + "_" + pjGrUsr.getGr();
			List<PjGrUsr> grUsrList = results.get(key);// 项目ID_组ID see: Issue 4
			if (grUsrList == null) {
				grUsrList = new ArrayList<PjGrUsr>();
				results.put(key, grUsrList);
			}
			grUsrList.add(pjGrUsr);
		}

		return results;

	}

	/**
	 * 输出http多库方式的密码文件
	 *
	 * @param root
	 *            svn root
	 * @param usrList
	 *            所有用户列表
	 */
	private void exportPasswdHTTPMutil(File root, List<Usr> usrList) {
		File outFile = new File(root, "passwd.http");
		StringBuffer contents = new StringBuffer();

		for (Usr usr : usrList) {
			// 采用SHA加密
			// http://httpd.apache.org/docs/2.2/misc/password_encryptions.html
			String shaPsw = "{SHA}"
					+ EncryptUtil
							.encriptSHA1(EncryptUtil.decrypt(usr.getPsw()));
			contents.append(usr.getUsr()).append(":").append(shaPsw)
					.append(SEP);
		}
		this.write(outFile, contents.toString());
	}

	/**
	 * 输出http单库方式的密码文件
	 *
	 * @param pj
	 *            项目
	 * @param usrList
	 *            项目用户列表
	 */
	private void exportPasswdHTTP(Pj pj, List<Usr> usrList) {
		File outFile = new File(pj.getPath(), "/conf/passwd.http");
		StringBuffer contents = new StringBuffer();

		for (Usr usr : usrList) {
			// 采用SHA加密
			// http://httpd.apache.org/docs/2.2/misc/password_encryptions.html
			String shaPsw = "{SHA}"
					+ EncryptUtil
							.encriptSHA1(EncryptUtil.decrypt(usr.getPsw()));
			contents.append(usr.getUsr()).append(":").append(shaPsw)
					.append(SEP);
		}
		this.write(outFile, contents.toString());
	}

	/**
	 * 输出svn方式的密码文件
	 *
	 * @param pj
	 *            项目
	 * @param usrList
	 *            项目用户列表
	 */
	private void exportPasswdSVN(Pj pj, List<Usr> usrList) {
		File outFile = new File(pj.getPath(), "/conf/passwd");
		StringBuffer contents = new StringBuffer();
		contents.append("[users]").append(SEP);

		for (Usr usr : usrList) {
			contents.append(usr.getUsr()).append("=")
					.append(EncryptUtil.decrypt(usr.getPsw())).append(SEP);// 解密
		}
		this.write(outFile, contents.toString());
	}

	/**
	 * 输出http多库方式的权限文件
	 *
	 * @param root
	 *            svn root
	 * @param pjGrUsrMap
	 *            所有的项目组用户列表
	 * @param resMap
	 *            所有的权限列表
	 */
	private void exportAuthzHTTPMutil(File root,
			Map<String, List<PjGrUsr>> pjGrUsrMap,
			Map<String, List<PjAuth>> resMap) {
		if (root == null) {
			return;
		}
		File outFile = new File(root, "authz");
		StringBuffer contents = new StringBuffer();
		contents.append("[aliases]").append(SEP);
		contents.append("[groups]").append(SEP);

		for (Iterator<String> grIterator = pjGrUsrMap.keySet().iterator(); grIterator
				.hasNext();) {
			String gr = grIterator.next();// 项目ID_组ID see: Issue 4
			contents.append(gr).append("=");
			List<PjGrUsr> pjGrUsrList = pjGrUsrMap.get(gr);
			for (int i = 0; i < pjGrUsrList.size(); i++) {
				PjGrUsr pjGrUsr = pjGrUsrList.get(i);
				if (pjGrUsr.getUsr() == null) {
					continue;
				}
				if (i != 0) {
					contents.append(",");
				}
				contents.append(pjGrUsr.getUsr());
			}
			contents.append(SEP);
		}

		contents.append(SEP);

		for (Iterator<String> resIterator = resMap.keySet().iterator(); resIterator
				.hasNext();) {
			String res = resIterator.next();
			contents.append(res).append(SEP);
			for (PjAuth pjAuth : resMap.get(res)) {
				if (StringUtils.isNotBlank(pjAuth.getGr())) {
					// 项目ID_组ID see: Issue 4
					contents.append("@")
							.append(pjAuth.getPjId() + "_" + pjAuth.getGr())
							.append("=").append(pjAuth.getRw()).append(SEP);
				} else if (StringUtils.isNotBlank(pjAuth.getUsr())) {
					contents.append(pjAuth.getUsr()).append("=")
							.append(pjAuth.getRw()).append(SEP);
				}
			}
			contents.append(SEP);
		}

		this.write(outFile, contents.toString());
	}

	/**
	 * 输出权限配置文件
	 *
	 * @param pj
	 *            项目
	 * @param pjGrUsrMap
	 *            项目的组列表
	 * @param resMap
	 *            项目的权限列表
	 */
	private void exportAuthz(Pj pj, Map<String, List<PjGrUsr>> pjGrUsrMap,
			Map<String, List<PjAuth>> resMap) {
		if (pj == null || StringUtils.isBlank(pj.getPj())) {
			return;
		}
		/*
		 * if(pjGrList == null || pjGrList.size() == 0){ return; } if(pjAuthMap
		 * == null || pjAuthMap.size() == 0){ return; }
		 */
		File outFile = new File(pj.getPath(), "/conf/authz");
		StringBuffer contents = new StringBuffer();
		contents.append("[aliases]").append(SEP);
		contents.append("[groups]").append(SEP);

		for (Iterator<String> grIterator = pjGrUsrMap.keySet().iterator(); grIterator
				.hasNext();) {
			String gr = grIterator.next();
			contents.append(gr).append("=");
			List<PjGrUsr> pjGrUsrList = pjGrUsrMap.get(gr);
			for (int i = 0; i < pjGrUsrList.size(); i++) {
				PjGrUsr pjGrUsr = pjGrUsrList.get(i);
				if (pjGrUsr.getUsr() == null) {
					continue;
				}
				if (i != 0) {
					contents.append(",");
				}
				contents.append(pjGrUsr.getUsr());
			}
			contents.append(SEP);
		}

		contents.append(SEP);

		for (Iterator<String> resIterator = resMap.keySet().iterator(); resIterator
				.hasNext();) {
			String res = resIterator.next();
			contents.append(res).append(SEP);
			for (PjAuth pjAuth : resMap.get(res)) {
				if (StringUtils.isNotBlank(pjAuth.getGr())) {
					contents.append("@").append(pjAuth.getGr()).append("=")
							.append(pjAuth.getRw()).append(SEP);
				} else if (StringUtils.isNotBlank(pjAuth.getUsr())) {
					contents.append(pjAuth.getUsr()).append("=")
							.append(pjAuth.getRw()).append(SEP);
				}
			}
			contents.append(SEP);
		}

		this.write(outFile, contents.toString());
	}

	/**
	 * 输出svn方式的svnserve.conf
	 *
	 * @param pj
	 *            项目
	 */
	private void exportSvnConf(Pj pj) {
		if (pj == null || StringUtils.isBlank(pj.getPj())) {
			return;
		}
		File outFile = new File(pj.getPath(), "/conf/svnserve.conf");

		StringBuffer contents = new StringBuffer();
		contents.append("[general]").append(SEP);
		contents.append("anon-access = none").append(SEP);
		contents.append("auth-access = write").append(SEP);
		contents.append("password-db = passwd").append(SEP);
		contents.append("authz-db = authz").append(SEP);
		contents.append("[sasl]").append(SEP);
		this.write(outFile, contents.toString());

	}

	/**
	 * 输出http单库方式的httpd.conf文件
	 *
	 * @param pj
	 *            项目
	 */
	private void exportSVNPathConf(Pj pj) {
		if (pj == null || StringUtils.isBlank(pj.getPj())) {
			return;
		}
		File outFile = new File(pj.getPath(), "/conf/httpd.conf");
		StringBuffer contents = new StringBuffer();
		contents.append("#Include ").append(pj.getPath())
				.append("/conf/httpd.conf").append(SEP);

		String location = pj.getPj();
		// 例如 http://192.168.1.100/svn/projar/trunk
		if (StringUtils.isNotBlank(pj.getUrl())
				&& pj.getUrl().indexOf("//") != -1) {
			String svnUrl =RepositoryService.parseURL(pj.getUrl());
			location = StringUtils.substringAfter(svnUrl, "//");// 192.168.1.100/svn/projar/trunk
			location = StringUtils.substringAfter(location, "/");// svn/projar/trunk
			location = StringUtils.substringBeforeLast(location, "/trunk");// svn/projar
																			// see:
																			// Issue
																			// 5
		}

		contents.append("<Location /").append(location).append(">").append(SEP);
		contents.append("DAV svn").append(SEP);
		contents.append("SVNPath ").append(pj.getPath()).append(SEP);
		contents.append("AuthType Basic").append(SEP);
		contents.append("AuthName ").append("\"").append(pj.getPj())
				.append("\"").append(SEP);
		contents.append("AuthUserFile ").append(pj.getPath())
				.append("/conf/passwd.http").append(SEP);
		contents.append("AuthzSVNAccessFile ").append(pj.getPath())
				.append("/conf/authz").append(SEP);
		contents.append("Require valid-user").append(SEP);
		contents.append("</Location>").append(SEP);
		this.write(outFile, contents.toString());

	}

	/**
	 * 输出http多库方式的httpd.conf文件
	 *
	 * @param root
	 *            svn root
	 */
	private void exportSVNParentPathConf(File root) {
		String svnRoot = StringUtils.replace(root.getAbsolutePath(), "\\", "/");
		File outFile = new File(root, "httpd.conf");
		StringBuffer contents = new StringBuffer();
		contents.append("#Include ").append(svnRoot).append("/httpd.conf")
				.append(SEP);

		String location = root.getName();

		contents.append("<Location /").append(location).append("/>")
				.append(SEP);
		contents.append("DAV svn").append(SEP);
		contents.append("SVNListParentPath on").append(SEP);
		contents.append("SVNParentPath ").append(svnRoot).append(SEP);
		contents.append("AuthType Basic").append(SEP);
		contents.append("AuthName ").append("\"")
				.append("Subversion repositories").append("\"").append(SEP);
		contents.append("AuthUserFile ").append(svnRoot).append("/passwd.http")
				.append(SEP);
		contents.append("AuthzSVNAccessFile ").append(svnRoot).append("/authz")
				.append(SEP);
		contents.append("Require valid-user").append(SEP);
		contents.append("</Location>").append(SEP);
		contents.append("RedirectMatch ^(/").append(location).append(")$ $1/")
				.append(SEP);
		this.write(outFile, contents.toString());
	}

	/**
	 * 写文件流
	 *
	 * @param outFile
	 *            输出文件
	 * @param contents
	 *            内容
	 */
	private void write(File outFile, String contents) {
		BufferedWriter writer = null;
		try {
			if (contents == null) {
				contents = "";
			}
			if (!outFile.getParentFile().exists()) {
				outFile.getParentFile().mkdirs();
			}
			writer = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream(outFile), "UTF-8"));// UTF-8 without
																// BOM
			writer.write(contents);
			LOG.debug(outFile);
		} catch (Exception e) {
			e.printStackTrace();
			throw new RuntimeException(e.getMessage());
		} finally {
			if (writer != null) {
				try {
					writer.flush();
				} catch (IOException e) {
					e.printStackTrace();
				}
				try {
					writer.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}
}


text:
import { Vector3 } from './../math/Vector3';
import { Euler } from './../math/Euler';
import { Quaternion } from './../math/Quaternion';
import { Matrix4 } from './../math/Matrix4';
import { Matrix3 } from './../math/Matrix3';
import { Layers } from './Layers';
import { WebGLRenderer } from './../renderers/WebGLRenderer';
import { Scene } from './../scenes/Scene';
import { Camera } from './../cameras/Camera';
import { Geometry } from './Geometry';
import { Material } from './../materials/Material';
import { Group } from './../objects/Group';
import { Raycaster } from './Raycaster';
import { EventDispatcher } from './EventDispatcher';
import { BufferGeometry } from './BufferGeometry';
import { Intersection } from './Raycaster';

export let Object3DIdCount: number;

/**
 * Base class for scene graph objects
 */
export class Object3D extends EventDispatcher {
  constructor();

  /**
   * Unique number of this object instance.
   */
  id: number;

  /**
   *
   */
  uuid: string;

  /**
   * Optional name of the object (doesn't need to be unique).
   */
  name: string;

  type: string;

  /**
   * Object's parent in the scene graph.
   */
  parent: Object3D | null;

  /**
   * Array with object's children.
   */
  children: Object3D[];

  /**
   * Up direction.
   */
  up: Vector3;

  /**
   * Object's local position.
   */
  position: Vector3;

  /**
   * Object's local rotation (Euler angles), in radians.
   */
  rotation: Euler;

  /**
   * Global rotation.
   */
  quaternion: Quaternion;

  /**
   * Object's local scale.
   */
  scale: Vector3;

  modelViewMatrix: Matrix4;

  normalMatrix: Matrix3;

  /**
   * Local transform.
   */
  matrix: Matrix4;

  /**
   * The global transform of the object. If the Object3d has no parent, then it's identical to the local transform.
   */
  matrixWorld: Matrix4;

  /**
   * When this is set, it calculates the matrix of position, (rotation or quaternion) and scale every frame and also recalculates the matrixWorld property.
   */
  matrixAutoUpdate: boolean;

  /**
   * When this is set, it calculates the matrixWorld in that frame and resets this property to false.
   */
  matrixWorldNeedsUpdate: boolean;

  layers: Layers;
  /**
   * Object gets rendered if true.
   */
  visible: boolean;

  /**
   * Gets rendered into shadow map.
   */
  castShadow: boolean;

  /**
   * Material gets baked in shadow receiving.
   */
  receiveShadow: boolean;

  /**
   * When this is set, it checks every frame if the object is in the frustum of the camera. Otherwise the object gets drawn every frame even if it isn't visible.
   */
  frustumCulled: boolean;

  /**
   * Overrides the default rendering order of scene graph objects, from lowest to highest renderOrder. Opaque and transparent objects remain sorted independently though. When this property is set for an instance of Group, all descendants objects will be sorted and rendered together.
   */
  renderOrder: number;

  /**
   * An object that can be used to store custom data about the Object3d. It should not hold references to functions as these will not be cloned.
   */
  userData: { [key: string]: any };

  /**
   * Used to check whether this or derived classes are Object3Ds. Default is true.
   * You should not change this, as it is used internally for optimisation.
   */
  isObject3D: true;

  /**
   * Calls before rendering object
   */
  onBeforeRender: (
    renderer: WebGLRenderer,
    scene: Scene,
    camera: Camera,
    geometry: Geometry | BufferGeometry,
    material: Material,
    group: Group
  ) => void;

  /**
   * Calls after rendering object
   */
  onAfterRender: (
    renderer: WebGLRenderer,
    scene: Scene,
    camera: Camera,
    geometry: Geometry | BufferGeometry,
    material: Material,
    group: Group
  ) => void;

  static DefaultUp: Vector3;
  static DefaultMatrixAutoUpdate: boolean;

  /**
   * This updates the position, rotation and scale with the matrix.
   */
  applyMatrix(matrix: Matrix4): void;

  applyQuaternion(quaternion: Quaternion): this;

  /**
   *
   */
  setRotationFromAxisAngle(axis: Vector3, angle: number): void;

  /**
   *
   */
  setRotationFromEuler(euler: Euler): void;

  /**
   *
   */
  setRotationFromMatrix(m: Matrix4): void;

  /**
   *
   */
  setRotationFromQuaternion(q: Quaternion): void;

  /**
   * Rotate an object along an axis in object space. The axis is assumed to be normalized.
   * @param axis  A normalized vector in object space.
   * @param angle  The angle in radians.
   */
  rotateOnAxis(axis: Vector3, angle: number): this;

  /**
   * Rotate an object along an axis in world space. The axis is assumed to be normalized. Method Assumes no rotated parent.
   * @param axis  A normalized vector in object space.
   * @param angle  The angle in radians.
   */
  rotateOnWorldAxis(axis: Vector3, angle: number): this;

  /**
   *
   * @param angle
   */
  rotateX(angle: number): this;

  /**
   *
   * @param angle
   */
  rotateY(angle: number): this;

  /**
   *
   * @param angle
   */
  rotateZ(angle: number): this;

  /**
   * @param axis  A normalized vector in object space.
   * @param distance  The distance to translate.
   */
  translateOnAxis(axis: Vector3, distance: number): this;

  /**
   * Translates object along x axis by distance.
   * @param distance Distance.
   */
  translateX(distance: number): this;

  /**
   * Translates object along y axis by distance.
   * @param distance Distance.
   */
  translateY(distance: number): this;

  /**
   * Translates object along z axis by distance.
   * @param distance Distance.
   */
  translateZ(distance: number): this;

  /**
   * Updates the vector from local space to world space.
   * @param vector A local vector.
   */
  localToWorld(vector: Vector3): Vector3;

  /**
   * Updates the vector from world space to local space.
   * @param vector A world vector.
   */
  worldToLocal(vector: Vector3): Vector3;

  /**
   * Rotates object to face point in space.
   * @param vector A world vector to look at.
   */
  lookAt(vector: Vector3 | number, y?: number, z?: number): void;

  /**
   * Adds object as child of this object.
   */
  add(...object: Object3D[]): this;

  /**
   * Removes object as child of this object.
   */
  remove(...object: Object3D[]): this;

  /**
   * Searches through the object's children and returns the first with a matching id.
   * @param id  Unique number of the object instance
   */
  getObjectById(id: number): Object3D | undefined;

  /**
   * Searches through the object's children and returns the first with a matching name.
   * @param name  String to match to the children's Object3d.name property.
   */
  getObjectByName(name: string): Object3D | undefined;

  getObjectByProperty(name: string, value: string): Object3D | undefined;

  getWorldPosition(target: Vector3): Vector3;
  getWorldQuaternion(target: Quaternion): Quaternion;
  getWorldScale(target: Vector3): Vector3;
  getWorldDirection(target: Vector3): Vector3;

  raycast(raycaster: Raycaster, intersects: Intersection[]): void;

  traverse(callback: (object: Object3D) => any): void;

  traverseVisible(callback: (object: Object3D) => any): void;

  traverseAncestors(callback: (object: Object3D) => any): void;

  /**
   * Updates local transform.
   */
  updateMatrix(): void;

  /**
   * Updates global transform of the object and its children.
   */
  updateMatrixWorld(force?: boolean): void;

  updateWorldMatrix(updateParents: boolean, updateChildren: boolean): void;

  toJSON(meta?: {
    geometries: any;
    materials: any;
    textures: any;
    images: any;
  }): any;

  clone(recursive?: boolean): this;

  /**
   *
   * @param object
   * @param recursive
   */
  copy(source: Object3D, recursive?: boolean): this;
}


text:
package com.exilant.exility.core;

/**
 * add a column to a grid. This is similar to addColumn grid processor, but not
 * exactly. grid processor evaluates the expression for each row while this step
 * evaluates the expression once against the dc and uses the same value for all
 * rows. Hence this is deprecated and we encourage use of grid processor. T
 *
 */
@Deprecated
class AddColumnStep extends AbstractStep {
	/**
	 * name of the grid to which we are adding the column
	 */
	String gridName = null;

	/**
	 * name of the column to be added. This should be in data dictionary. Its
	 * data type should match the resultant data type of the expression
	 */
	String columnName = null;

	/**
	 * value to be added to the column. Expression is evaluated with dc and the
	 * resultant value is used as column value for all rows. If you want the
	 * expression to contains some of the columns as well, use grid processor
	 * instead.
	 */
	Expression expression = null;

	AddColumnStep() {
		this.stepType = StepType.ADDCOLUMNSTEP;
	}

	@Override
	String executeStep(DataCollection dc, DbHandle handle)
			throws ExilityException {
		if (this.gridName == null || this.gridName.equals("")
				|| this.columnName == null || this.columnName.length() == 0) {
			dc.addError(this.stepType + " not set properly.");
			return AbstractStep.STOP;
		}

		Grid grid = dc.getGrid(this.gridName);
		if (grid == null) {
			Spit.out(this.gridName + " not found in dc.");
			return AbstractStep.NEXT;
		}

		Value val = this.expression.evaluate(dc);

		ValueList valueList = new ValueList(grid.getNumberOfRows());
		for (int i = 0; i < valueList.length(); i++) {
			valueList.setValue(val, i);
		}
		grid.addColumn(this.columnName, valueList);

		dc.addGrid(this.gridName, grid);

		return AbstractStep.NEXT;
	}

}


text:
Connected social accounts:

Connect other social accounts: