text:
package org.elasticsearch.xpack.ml.integration;

import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.SecuritySettingsSourceField;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.rollup.job.RollupJob;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.junit.After;
import org.junit.Before;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;

public class DatafeedJobsRestIT extends ESRestTestCase {

    private static final String BASIC_AUTH_VALUE_SUPER_USER =
            basicAuthHeaderValue("x_pack_rest_user", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
    private static final String BASIC_AUTH_VALUE_ML_ADMIN =
            basicAuthHeaderValue("ml_admin", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
    private static final String BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS =
            basicAuthHeaderValue("ml_admin_plus_data", SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);

    @Override
    protected Settings restClientSettings() {
        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE_SUPER_USER).build();
    }

    @Override
    protected boolean preserveTemplatesUponCompletion() {
        return true;
    }

    private static void setupDataAccessRole(String index) throws IOException {
        Request request = new Request("PUT", "/_security/role/test_data_access");
        request.setJsonEntity("{"
                + "  \"indices\" : ["
                + "    { \"names\": [\"" + index + "\"], \"privileges\": [\"read\"] }"
                + "  ]"
                + "}");
        client().performRequest(request);
    }

    private void setupFullAccessRole(String index) throws IOException {
        Request request = new Request("PUT", "/_security/role/test_data_access");
        request.setJsonEntity("{"
            + "  \"indices\" : ["
            + "    { \"names\": [\"" + index + "\"], \"privileges\": [\"all\"] }"
            + "  ]"
            + "}");
        client().performRequest(request);
    }

    private void setupUser(String user, List<String> roles) throws IOException {
        String password = new String(SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING.getChars());

        Request request = new Request("PUT", "/_security/user/" + user);
        request.setJsonEntity("{"
                + "  \"password\" : \"" + password + "\","
                + "  \"roles\" : [ " + roles.stream().map(unquoted -> "\"" + unquoted + "\"").collect(Collectors.joining(", ")) + " ]"
                + "}");
        client().performRequest(request);
    }

    @Before
    public void setUpData() throws Exception {
        setupDataAccessRole("network-data");
        // This user has admin rights on machine learning, but (importantly for the tests) no rights
        // on any of the data indexes
        setupUser("ml_admin", Collections.singletonList("machine_learning_admin"));
        // This user has admin rights on machine learning, and read access to the network-data index
        setupUser("ml_admin_plus_data", Arrays.asList("machine_learning_admin", "test_data_access"));
        addAirlineData();
        addNetworkData("network-data");
    }

    private void addAirlineData() throws IOException {
        StringBuilder bulk = new StringBuilder();

        Request createEmptyAirlineDataRequest = new Request("PUT", "/airline-data-empty");
        createEmptyAirlineDataRequest.setJsonEntity("{"
                + "  \"mappings\": {"
                + "    \"properties\": {"
                + "      \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional
                + "      \"airline\": { \"type\":\"keyword\"},"
                + "      \"responsetime\": { \"type\":\"float\"}"
                + "    }"
                + "  }"
                + "}");
        client().performRequest(createEmptyAirlineDataRequest);

        // Create index with source = enabled, doc_values = enabled, stored = false + multi-field
        Request createAirlineDataRequest = new Request("PUT", "/airline-data");
        createAirlineDataRequest.setJsonEntity("{"
                + "  \"mappings\": {"
                + "    \"properties\": {"
                + "      \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional
                + "      \"airline\": {"
                + "        \"type\":\"text\","
                + "        \"fields\":{"
                + "          \"text\":{\"type\":\"text\"},"
                + "          \"keyword\":{\"type\":\"keyword\"}"
                + "         }"
                + "       },"
                + "      \"responsetime\": { \"type\":\"float\"}"
                + "    }"
                + "  }"
                + "}");
        client().performRequest(createAirlineDataRequest);

        bulk.append("{\"index\": {\"_index\": \"airline-data\", \"_id\": 1}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data\", \"_id\": 2}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}\n");

        // Create index with source = enabled, doc_values = disabled (except time), stored = false
        Request createAirlineDataDisabledDocValues = new Request("PUT", "/airline-data-disabled-doc-values");
        createAirlineDataDisabledDocValues.setJsonEntity("{"
                + "  \"mappings\": {"
                + "    \"properties\": {"
                + "      \"time stamp\": { \"type\":\"date\"},"
                + "      \"airline\": { \"type\":\"keyword\", \"doc_values\":false},"
                + "      \"responsetime\": { \"type\":\"float\", \"doc_values\":false}"
                + "    }"
                + "  }"
                + "}");
        client().performRequest(createAirlineDataDisabledDocValues);

        bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-doc-values\", \"_id\": 1}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-doc-values\", \"_id\": 2}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}\n");

        // Create index with source = disabled, doc_values = enabled (except time), stored = true
        Request createAirlineDataDisabledSource = new Request("PUT", "/airline-data-disabled-source");
        createAirlineDataDisabledSource.setJsonEntity("{"
                + "  \"mappings\": {"
                + "    \"_source\":{\"enabled\":false},"
                + "    \"properties\": {"
                + "      \"time stamp\": { \"type\":\"date\", \"store\":true},"
                + "      \"airline\": { \"type\":\"keyword\", \"store\":true},"
                + "      \"responsetime\": { \"type\":\"float\", \"store\":true}"
                + "    }"
                + "  }"
                + "}");

        bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-source\", \"_id\": 1}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-disabled-source\", \"_id\": 2}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}\n");

        // Create index with nested documents
        Request createAirlineDataNested = new Request("PUT", "/nested-data");
        createAirlineDataNested.setJsonEntity("{"
                + "  \"mappings\": {"
                + "    \"properties\": {"
                + "      \"time\": { \"type\":\"date\"}"
                + "    }"
                + "  }"
                + "}");
        client().performRequest(createAirlineDataNested);

        bulk.append("{\"index\": {\"_index\": \"nested-data\", \"_id\": 1}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:00:00Z\", \"responsetime\":{\"millis\":135.22}, \"airline\":[{\"name\": \"foo\"}]}\n");
        bulk.append("{\"index\": {\"_index\": \"nested-data\", \"_id\": 2}}\n");
        bulk.append("{\"time\":\"2016-06-01T01:59:00Z\", \"responsetime\":{\"millis\":222.00}, \"airline\":[{\"name\": \"bar\"}]}\n");

        // Create index with multiple docs per time interval for aggregation testing
        Request createAirlineDataAggs = new Request("PUT", "/airline-data-aggs");
        createAirlineDataAggs.setJsonEntity("{"
                + "  \"mappings\": {"
                + "    \"properties\": {"
                + "      \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional
                + "      \"airline\": { \"type\":\"keyword\"},"
                + "      \"responsetime\": { \"type\":\"float\"}"
                + "    }"
                + "  }"
                + "}");
        client().performRequest(createAirlineDataAggs);

        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 1}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":100.0}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 2}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"AAA\",\"responsetime\":200.0}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 3}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"BBB\",\"responsetime\":1000.0}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 4}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"BBB\",\"responsetime\":2000.0}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 5}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"AAA\",\"responsetime\":300.0}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 6}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"AAA\",\"responsetime\":400.0}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 7}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"BBB\",\"responsetime\":3000.0}\n");
        bulk.append("{\"index\": {\"_index\": \"airline-data-aggs\", \"_id\": 8}}\n");
        bulk.append("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"BBB\",\"responsetime\":4000.0}\n");

        bulkIndex(bulk.toString());
    }

    private void addNetworkData(String index) throws IOException {
        // Create index with source = enabled, doc_values = enabled, stored = false + multi-field
        Request createIndexRequest = new Request("PUT", index);
        createIndexRequest.setJsonEntity("{"
                + "  \"mappings\": {"
                + "    \"properties\": {"
                + "      \"timestamp\": { \"type\":\"date\"},"
                + "      \"host\": {"
                + "        \"type\":\"text\","
                + "        \"fields\":{"
                + "          \"text\":{\"type\":\"text\"},"
                + "          \"keyword\":{\"type\":\"keyword\"}"
                + "         }"
                + "       },"
                + "      \"network_bytes_out\": { \"type\":\"long\"}"
                + "    }"
                + "  }"
                + "}");
        client().performRequest(createIndexRequest);

        StringBuilder bulk = new StringBuilder();
        String docTemplate = "{\"timestamp\":%d,\"host\":\"%s\",\"network_bytes_out\":%d}";
        Date date = new Date(1464739200735L);
        for (int i = 0; i < 120; i++) {
            long byteCount = randomNonNegativeLong();
            bulk.append("{\"index\": {\"_index\": \"").append(index).append("\"}}\n");
            bulk.append(String.format(Locale.ROOT, docTemplate, date.getTime(), "hostA", byteCount)).append('\n');

            byteCount = randomNonNegativeLong();
            bulk.append("{\"index\": {\"_index\": \"").append(index).append("\"}}\n");
            bulk.append(String.format(Locale.ROOT, docTemplate, date.getTime(), "hostB", byteCount)).append('\n');

            date = new Date(date.getTime() + 10_000);
        }

        bulkIndex(bulk.toString());
    }

    public void testLookbackOnlyWithMixedTypes() throws Exception {
        new LookbackOnlyTestHelper("test-lookback-only-with-mixed-types", "airline-data")
                .setShouldSucceedProcessing(true).execute();
    }

    public void testLookbackOnlyWithKeywordMultiField() throws Exception {
        new LookbackOnlyTestHelper("test-lookback-only-with-keyword-multi-field", "airline-data")
                .setAirlineVariant("airline.keyword").setShouldSucceedProcessing(true).execute();
    }

    public void testLookbackOnlyWithTextMultiField() throws Exception {
        new LookbackOnlyTestHelper("test-lookback-only-with-keyword-multi-field", "airline-data")
                .setAirlineVariant("airline.text").setShouldSucceedProcessing(true).execute();
    }

    public void testLookbackOnlyWithDocValuesDisabled() throws Exception {
        new LookbackOnlyTestHelper("test-lookback-only-with-doc-values-disabled", "airline-data-disabled-doc-values").execute();
    }

    public void testLookbackOnlyWithSourceDisabled() throws Exception {
        new LookbackOnlyTestHelper("test-lookback-only-with-source-disabled", "airline-data-disabled-source").execute();
    }

    public void testLookbackOnlyWithScriptFields() throws Exception {
        new LookbackOnlyTestHelper("test-lookback-only-with-script-fields", "airline-data")
                .setScriptedFields(
                    "{\"scripted_airline\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['airline.keyword'].value\"}}}")
            .setAirlineVariant("scripted_airline")
            .execute();
    }

    public void testLookbackonlyWithNestedFields() throws Exception {
        String jobId = "test-lookback-only-with-nested-fields";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"description\": \"Nested job\",\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"1h\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"responsetime.millis\",\n"
                + "        \"by_field_name\": \"airline.name\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },"
                + "  \"data_description\": {\"time_field\": \"time\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = jobId + "-datafeed";
        new DatafeedBuilder(datafeedId, jobId, "nested-data").build();
        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(
                new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testLookbackWithGeo() throws Exception {
        String jobId = "test-lookback-only-with-geo";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
            + "  \"description\": \"lat_long with geo_point\",\n"
            + "  \"analysis_config\": {\n"
            + "    \"bucket_span\": \"15m\",\n"
            + "    \"detectors\": [\n"
            + "      {\n"
            + "        \"function\": \"lat_long\",\n"
            + "        \"field_name\": \"location\"\n"
            + "      }\n"
            + "    ]\n"
            + "  },"
            + "  \"data_description\": {\"time_field\": \"time\"}\n"
            + "}");
        client().performRequest(createJobRequest);
        String datafeedId = jobId + "-datafeed";
        new DatafeedBuilder(datafeedId, jobId, "geo-data").build();

        StringBuilder bulk = new StringBuilder();

        Request createGeoData = new Request("PUT", "/geo-data");
        createGeoData.setJsonEntity("{"
            + "  \"mappings\": {"
            + "    \"properties\": {"
            + "      \"time\": { \"type\":\"date\"},"
            + "      \"location\": { \"type\":\"geo_point\"}"
            + "    }"
            + "  }"
            + "}");
        client().performRequest(createGeoData);

        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 1}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:00:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 2}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:05:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 3}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:10:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 4}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:15:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 5}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:20:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 6}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:25:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 7}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:30:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 8}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:40:00Z\",\"location\":{\"lat\":90.0,\"lon\":-77.03653}}\n");
        bulk.append("{\"index\": {\"_index\": \"geo-data\", \"_id\": 9}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:41:00Z\",\"location\":{\"lat\":38.897676,\"lon\":-77.03653}}\n");
        bulkIndex(bulk.toString());

        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(
            new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":9"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":9"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testLookbackWithIndicesOptions() throws Exception {
        String jobId = "test-lookback-only-with-indices-options";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
            + "  \"description\": \"custom indices options\",\n"
            + "  \"analysis_config\": {\n"
            + "    \"bucket_span\": \"15m\",\n"
            + "    \"detectors\": [\n"
            + "      {\n"
            + "        \"function\": \"count\"\n"
            + "      }\n"
            + "    ]\n"
            + "  },"
            + "  \"data_description\": {\"time_field\": \"time\"}\n"
            + "}");
        client().performRequest(createJobRequest);
        String datafeedId = jobId + "-datafeed";
        new DatafeedBuilder(datafeedId, jobId, "*hidden-*")
            .setIndicesOptions("{" +
                "\"expand_wildcards\": [\"all\"]," +
                "\"allow_no_indices\": true"+
                "}")
            .build();

        StringBuilder bulk = new StringBuilder();

        Request createGeoData = new Request("PUT", "/.hidden-index");
        createGeoData.setJsonEntity("{"
            + "  \"mappings\": {"
            + "    \"properties\": {"
            + "      \"time\": { \"type\":\"date\"},"
            + "      \"value\": { \"type\":\"long\"}"
            + "    }"
            + "  }, \"settings\": {\"index.hidden\": true} "
            + "}");
        client().performRequest(createGeoData);

        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 1}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:00:00Z\",\"value\": 1000}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 2}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:05:00Z\",\"value\":1500}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 3}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:10:00Z\",\"value\":1600}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 4}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:15:00Z\",\"value\":100}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 5}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:20:00Z\",\"value\":1}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 6}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:25:00Z\",\"value\":1500}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 7}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:30:00Z\",\"value\":1500}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 8}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:40:00Z\",\"value\":2100}\n");
        bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 9}}\n");
        bulk.append("{\"time\":\"2016-06-01T00:41:00Z\",\"value\":0}\n");
        bulkIndex(bulk.toString());

        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(
            new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":9"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":9"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testLookbackOnlyGivenEmptyIndex() throws Exception {
        new LookbackOnlyTestHelper("test-lookback-only-given-empty-index", "airline-data-empty")
                .setShouldSucceedInput(false).setShouldSucceedProcessing(false).execute();
    }

    public void testInsufficientSearchPrivilegesOnPut() throws Exception {
        String jobId = "privs-put-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"description\": \"Aggs job\",\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"1h\",\n "
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"responsetime\",\n"
                + "        \"by_field_name\":\"airline\"\n"
                + "       }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\" : {\"time_field\": \"time stamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        // This should be disallowed, because even though the ml_admin user has permission to
        // create a datafeed they DON'T have permission to search the index the datafeed is
        // configured to read
        ResponseException e = expectThrows(ResponseException.class, () ->
                new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs")
                        .setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN)
                        .build());

        assertThat(e.getMessage(), containsString("Cannot create datafeed"));
        assertThat(e.getMessage(),
                containsString("user ml_admin lacks permissions on the indices"));
    }

    public void testCreationOnPutWithRollup() throws Exception {
        setupDataAccessRole("airline-data-aggs-rollup");
        String jobId = "privs-put-job-rollup";
        String datafeedId = "datafeed-" + jobId;
        final Response response = createJobAndDataFeed(jobId, datafeedId);

        assertEquals(200, response.getStatusLine().getStatusCode());
        assertThat(EntityUtils.toString(response.getEntity()), containsString("\"datafeed_id\":\"" + datafeedId
            + "\",\"job_id\":\"" + jobId + "\""));
    }

    public void testInsufficientSearchPrivilegesOnPreview() throws Exception {
        String jobId = "privs-preview-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"description\": \"Aggs job\",\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"1h\",\n"
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"responsetime\",\n"
                + "        \"by_field_name\": \"airline\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\" : {\"time_field\": \"time stamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").build();

        // This should be disallowed, because ml_admin is trying to preview a datafeed created by
        // by another user (x_pack_rest_user in this case) that will reveal the content of an index they
        // don't have permission to search directly
        Request getFeed = new Request("GET", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_preview");
        RequestOptions.Builder options = getFeed.getOptions().toBuilder();
        options.addHeader("Authorization", BASIC_AUTH_VALUE_ML_ADMIN);
        getFeed.setOptions(options);
        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getFeed));

        assertThat(e.getMessage(),
                containsString("[indices:data/read/field_caps] is unauthorized for user [ml_admin]"));
    }

    public void testSecondaryAuthSearchPrivilegesLookBack() throws Exception {
        setupDataAccessRole("airline-data");
        String jobId = "secondary-privs-put-job";
        createJob(jobId, "airline.keyword");
        String datafeedId = "datafeed-" + jobId;
        // Primary auth header does not have access, but secondary auth does
        new DatafeedBuilder(datafeedId, jobId, "airline-data")
                .setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN)
                .setSecondaryAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS)
                .build();
        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);

        Response jobStatsResponse = client().performRequest(new Request("GET",
            MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testSecondaryAuthSearchPrivilegesOnPreview() throws Exception {
        setupDataAccessRole("airline-data");
        String jobId = "secondary-privs-preview-job";
        createJob(jobId, "airline.keyword");

        String datafeedId = "datafeed-" + jobId;
        new DatafeedBuilder(datafeedId, jobId, "airline-data").build();

        Request getFeed = new Request("GET", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_preview");
        RequestOptions.Builder options = getFeed.getOptions().toBuilder();
        options.addHeader("Authorization", BASIC_AUTH_VALUE_ML_ADMIN);
        options.addHeader("es-secondary-authorization", BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS);
        getFeed.setOptions(options);
        // Should not fail as secondary auth has permissions.
        client().performRequest(getFeed);
    }

    public void testLookbackOnlyGivenAggregationsWithHistogram() throws Exception {
        String jobId = "aggs-histogram-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"description\": \"Aggs job\",\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"1h\",\n"
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"responsetime\",\n"
                + "        \"by_field_name\": \"airline\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        String aggregations = "{\"buckets\":{\"histogram\":{\"field\":\"time stamp\",\"interval\":3600000},"
                + "\"aggregations\":{"
                + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
                + "\"airline\":{\"terms\":{\"field\":\"airline\",\"size\":10},"
                + "  \"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}";
        new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").setAggregations(aggregations).build();
        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
                MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testLookbackOnlyGivenAggregationsWithDateHistogram() throws Exception {
        String jobId = "aggs-date-histogram-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"description\": \"Aggs job\",\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"3600s\",\n"
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"responsetime\",\n"
                + "        \"by_field_name\": \"airline\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        String aggregations = "{\"time stamp\":{\"date_histogram\":{\"field\":\"time stamp\",\"calendar_interval\":\"1h\"},"
                + "\"aggregations\":{"
                + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
                + "\"airline\":{\"terms\":{\"field\":\"airline\",\"size\":10},"
                + "  \"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}";
        new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").setAggregations(aggregations).build();
        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
                MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testLookbackUsingDerivativeAggWithLargerHistogramBucketThanDataRate() throws Exception {
        String jobId = "derivative-agg-network-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"300s\",\n"
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"bytes-delta\",\n"
                + "        \"by_field_name\": \"hostname\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\": {\"time_field\": \"timestamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        String aggregations =
                 "{\"hostname\": {\"terms\" : {\"field\": \"host.keyword\", \"size\":10},"
                    + "\"aggs\": {\"buckets\": {\"date_histogram\":{\"field\":\"timestamp\",\"fixed_interval\":\"60s\"},"
                        + "\"aggs\": {\"timestamp\":{\"max\":{\"field\":\"timestamp\"}},"
                            + "\"bytes-delta\":{\"derivative\":{\"buckets_path\":\"avg_bytes_out\"}},"
                            + "\"avg_bytes_out\":{\"avg\":{\"field\":\"network_bytes_out\"}} }}}}}";
        new DatafeedBuilder(datafeedId, jobId, "network-data")
                .setAggregations(aggregations)
                .setChunkingTimespan("300s")
                .build();

        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
                MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":40"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":40"));
        assertThat(jobStatsResponseAsString, containsString("\"out_of_order_timestamp_count\":0"));
        assertThat(jobStatsResponseAsString, containsString("\"bucket_count\":3"));
        // The derivative agg won't have values for the first bucket of each host
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":2"));
    }

    public void testLookbackUsingDerivativeAggWithSmallerHistogramBucketThanDataRate() throws Exception {
        String jobId = "derivative-agg-network-job";
        Request createJobRequest = new Request("PUT",  MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"300s\",\n"
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"bytes-delta\",\n"
                + "        \"by_field_name\": \"hostname\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\": {\"time_field\": \"timestamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        String aggregations =
                "{\"hostname\": {\"terms\" : {\"field\": \"host.keyword\", \"size\":10},"
                        + "\"aggs\": {\"buckets\": {\"date_histogram\":{\"field\":\"timestamp\",\"fixed_interval\":\"5s\"},"
                        + "\"aggs\": {\"timestamp\":{\"max\":{\"field\":\"timestamp\"}},"
                        + "\"bytes-delta\":{\"derivative\":{\"buckets_path\":\"avg_bytes_out\"}},"
                        + "\"avg_bytes_out\":{\"avg\":{\"field\":\"network_bytes_out\"}} }}}}}";
        new DatafeedBuilder(datafeedId, jobId, "network-data")
                .setAggregations(aggregations)
                .setChunkingTimespan("300s")
                .build();

        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
                MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":240"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":240"));
    }

    public void testLookbackWithoutPermissions() throws Exception {
        String jobId = "permission-test-network-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"300s\",\n"
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"bytes-delta\",\n"
                + "        \"by_field_name\": \"hostname\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\": {\"time_field\": \"timestamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        String aggregations =
                "{\"hostname\": {\"terms\" : {\"field\": \"host.keyword\", \"size\":10},"
                        + "\"aggs\": {\"buckets\": {\"date_histogram\":{\"field\":\"timestamp\",\"fixed_interval\":\"5s\"},"
                        + "\"aggs\": {\"timestamp\":{\"max\":{\"field\":\"timestamp\"}},"
                        + "\"bytes-delta\":{\"derivative\":{\"buckets_path\":\"avg_bytes_out\"}},"
                        + "\"avg_bytes_out\":{\"avg\":{\"field\":\"network_bytes_out\"}} }}}}}";

        // At the time we create the datafeed the user can access the network-data index that we have access to
        new DatafeedBuilder(datafeedId, jobId, "network-data")
                .setAggregations(aggregations)
                .setChunkingTimespan("300s")
                .setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS)
                .build();

        // Change the role so that the user can no longer access network-data
        setupDataAccessRole("some-other-data");

        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId, BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
                MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        // We expect that no data made it through to the job
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":0"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":0"));

        // There should be a notification saying that there was a problem extracting data
        refreshAllIndices();
        Response notificationsResponse = client().performRequest(
                new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
        String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity());
        assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " +
                "action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data]\""));
    }

    public void testLookbackWithPipelineBucketAgg() throws Exception {
        String jobId = "pipeline-bucket-agg-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"1h\",\n"
                + "    \"summary_count_field_name\": \"doc_count\",\n"
                + "    \"detectors\": [\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"percentile95_airlines_count\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
                + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"fixed_interval\":\"15m\"},"
                + "\"aggregations\":{"
                    + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
                    + "\"airlines\":{\"terms\":{\"field\":\"airline.keyword\",\"size\":10}},"
                    + "\"percentile95_airlines_count\":{\"percentiles_bucket\":" +
                        "{\"buckets_path\":\"airlines._count\", \"percents\": [95]}}}}}";
        new DatafeedBuilder(datafeedId, jobId, "airline-data").setAggregations(aggregations).build();

        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
                MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"input_field_count\":4"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_field_count\":4"));
        assertThat(jobStatsResponseAsString, containsString("\"out_of_order_timestamp_count\":0"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testLookbackOnlyGivenAggregationsWithHistogramAndRollupIndex() throws Exception {
        String jobId = "aggs-histogram-rollup-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
            + "  \"description\": \"Aggs job\",\n"
            + "  \"analysis_config\": {\n"
            + "    \"bucket_span\": \"1h\",\n"
            + "    \"summary_count_field_name\": \"doc_count\",\n"
            + "    \"detectors\": [\n"
            + "      {\n"
            + "        \"function\": \"mean\",\n"
            + "        \"field_name\": \"responsetime\",\n"
            + "        \"by_field_name\": \"airline\"\n"
            + "      }\n"
            + "    ]\n"
            + "  },\n"
            + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
            + "}");
        client().performRequest(createJobRequest);

        String rollupJobId = "rollup-" + jobId;
        Request createRollupRequest = new Request("PUT", "/_rollup/job/" + rollupJobId);
        createRollupRequest.setJsonEntity("{\n"
            + "\"index_pattern\": \"airline-data-aggs\",\n"
            + "    \"rollup_index\": \"airline-data-aggs-rollup\",\n"
            + "    \"cron\": \"*/30 * * * * ?\",\n"
            + "    \"page_size\" :1000,\n"
            + "    \"groups\" : {\n"
            + "      \"date_histogram\": {\n"
            + "        \"field\": \"time stamp\",\n"
            + "        \"fixed_interval\": \"2m\",\n"
            + "        \"delay\": \"7d\"\n"
            + "      },\n"
            + "      \"terms\": {\n"
            + "        \"fields\": [\"airline\"]\n"
            + "      }"
            + "    },\n"
            + "    \"metrics\": [\n"
            + "        {\n"
            + "            \"field\": \"responsetime\",\n"
            + "            \"metrics\": [\"avg\",\"min\",\"max\",\"sum\"]\n"
            + "        },\n"
            + "        {\n"
            + "            \"field\": \"time stamp\",\n"
            + "            \"metrics\": [\"min\",\"max\"]\n"
            + "        }\n"
            + "    ]\n"
            + "}");
        client().performRequest(createRollupRequest);
        client().performRequest(new Request("POST", "/_rollup/job/" + rollupJobId + "/_start"));

        assertBusy(() -> {
            Response getRollup = client().performRequest(new Request("GET", "/_rollup/job/" + rollupJobId));
            String body = EntityUtils.toString(getRollup.getEntity());
            assertThat(body, containsString("\"job_state\":\"started\""));
            assertThat(body, containsString("\"rollups_indexed\":4"));
        }, 60, TimeUnit.SECONDS);

        client().performRequest(new Request("POST", "/_rollup/job/" + rollupJobId + "/_stop"));
        assertBusy(() -> {
            Response getRollup = client().performRequest(new Request("GET", "/_rollup/job/" + rollupJobId));
            assertThat(EntityUtils.toString(getRollup.getEntity()), containsString("\"job_state\":\"stopped\""));
        }, 60, TimeUnit.SECONDS);

        final Request refreshRollupIndex = new Request("POST", "airline-data-aggs-rollup/_refresh");
        client().performRequest(refreshRollupIndex);

        String datafeedId = "datafeed-" + jobId;
        String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"fixed_interval\":\"3600000ms\"},"
            + "\"aggregations\":{"
            + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
            + "\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}";
        new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup").setAggregations(aggregations).build();
        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
            MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
    }

    public void testLookbackWithoutPermissionsAndRollup() throws Exception {
        setupFullAccessRole("airline-data-aggs-rollup");
        String jobId = "rollup-permission-test-network-job";
        String datafeedId = "datafeed-" + jobId;
        createJobAndDataFeed(jobId, datafeedId);

        // Change the role so that the user can no longer access network-data
        setupFullAccessRole("some-other-data");

        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId, BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS);
        waitUntilJobIsClosed(jobId);
        // There should be a notification saying that there was a problem extracting data
        refreshAllIndices();
        Response notificationsResponse = client().performRequest(
            new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
        String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity());
        assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " +
            "action [indices:data/read/xpack/rollup/search] is unauthorized for user [ml_admin_plus_data]\""));
    }

    public void testLookbackWithSingleBucketAgg() throws Exception {
        String jobId = "aggs-date-histogram-with-single-bucket-agg-job";
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
            + "  \"description\": \"Aggs job\",\n"
            + "  \"analysis_config\": {\n"
            + "    \"bucket_span\": \"3600s\",\n"
            + "    \"summary_count_field_name\": \"doc_count\",\n"
            + "    \"detectors\": [\n"
            + "      {\n"
            + "        \"function\": \"mean\",\n"
            + "        \"field_name\": \"responsetime\""
            + "      }\n"
            + "    ]\n"
            + "  },\n"
            + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
            + "}");
        client().performRequest(createJobRequest);

        String datafeedId = "datafeed-" + jobId;
        String aggregations = "{\"time stamp\":{\"date_histogram\":{\"field\":\"time stamp\",\"calendar_interval\":\"1h\"},"
            + "\"aggregations\":{"
            + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
            + "\"airlineFilter\":{\"filter\":{\"term\": {\"airline\":\"AAA\"}},"
            + "  \"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}";
        new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs").setAggregations(aggregations).build();
        openJob(client(), jobId);

        startDatafeedAndWaitUntilStopped(datafeedId);
        waitUntilJobIsClosed(jobId);
        Response jobStatsResponse = client().performRequest(new Request("GET",
            MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
        String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
        assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
        assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
    }

    public void testRealtime() throws Exception {
        String jobId = "job-realtime-1";
        createJob(jobId, "airline");
        String datafeedId = jobId + "-datafeed";
        new DatafeedBuilder(datafeedId, jobId, "airline-data").build();
        openJob(client(), jobId);

        Request startRequest = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
        startRequest.addParameter("start", "2016-06-01T00:00:00Z");
        Response response = client().performRequest(startRequest);
        assertThat(EntityUtils.toString(response.getEntity()), containsString("\"started\":true"));
        assertBusy(() -> {
            try {
                Response getJobResponse = client().performRequest(new Request("GET",
                        MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
                String responseAsString = EntityUtils.toString(getJobResponse.getEntity());
                assertThat(responseAsString, containsString("\"processed_record_count\":2"));
                assertThat(responseAsString, containsString("\"state\":\"opened\""));
            } catch (Exception e1) {
                throw new RuntimeException(e1);
            }
        });

        // Model state should be persisted at the end of lookback
        // test a model snapshot is present
        assertBusy(() -> {
            try {
                Response getJobResponse = client().performRequest(new Request("GET",
                        MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/model_snapshots"));
                String responseAsString = EntityUtils.toString(getJobResponse.getEntity());
                assertThat(responseAsString, containsString("\"count\":1"));
            } catch (Exception e1) {
                throw new RuntimeException(e1);
            }
        });

        ResponseException e = expectThrows(ResponseException.class,
                () -> client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId)));
        response = e.getResponse();
        assertThat(response.getStatusLine().getStatusCode(), equalTo(409));
        assertThat(EntityUtils.toString(response.getEntity()),
                containsString("Cannot delete job [" + jobId + "] because the job is opened"));

        response = client().performRequest(new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stop"));
        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
        assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"stopped\":true}"));

        client().performRequest(new Request("POST", "/_ml/anomaly_detectors/" + jobId + "/_close"));

        response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId));
        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
        assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"acknowledged\":true}"));

        response = client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId));
        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
        assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"acknowledged\":true}"));
    }

    public void testForceDeleteWhileDatafeedIsRunning() throws Exception {
        String jobId = "job-realtime-2";
        createJob(jobId, "airline");
        String datafeedId = jobId + "-datafeed";
        new DatafeedBuilder(datafeedId, jobId, "airline-data").build();
        openJob(client(), jobId);

        Request startRequest = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
        startRequest.addParameter("start", "2016-06-01T00:00:00Z");
        Response response = client().performRequest(startRequest);
        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
        assertThat(EntityUtils.toString(response.getEntity()), containsString("\"started\":true"));

        ResponseException e = expectThrows(ResponseException.class,
                () -> client().performRequest(new Request("DELETE", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId)));
        response = e.getResponse();
        assertThat(response.getStatusLine().getStatusCode(), equalTo(409));
        assertThat(EntityUtils.toString(response.getEntity()),
                containsString("Cannot delete datafeed [" + datafeedId + "] while its status is started"));

        Request forceDeleteRequest = new Request("DELETE", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId);
        forceDeleteRequest.addParameter("force", "true");
        response = client().performRequest(forceDeleteRequest);
        assertThat(EntityUtils.toString(response.getEntity()), equalTo("{\"acknowledged\":true}"));

        expectThrows(ResponseException.class,
                () -> client().performRequest(new Request("GET", "/_ml/datafeeds/" + datafeedId)));
    }

    private class LookbackOnlyTestHelper {
        private String jobId;
        private String airlineVariant;
        private String dataIndex;
        private String scriptedFields;
        private boolean shouldSucceedInput;
        private boolean shouldSucceedProcessing;

        LookbackOnlyTestHelper(String jobId, String dataIndex) {
            this.jobId = jobId;
            this.dataIndex = dataIndex;
            this.shouldSucceedInput = true;
            this.shouldSucceedProcessing = true;
            this.airlineVariant = "airline";
        }

        public LookbackOnlyTestHelper setScriptedFields(String scriptFields) {
            this.scriptedFields = scriptFields;
            return this;
        }

        public LookbackOnlyTestHelper setAirlineVariant(String airlineVariant) {
            this.airlineVariant = airlineVariant;
            return this;
        }

        public LookbackOnlyTestHelper setShouldSucceedInput(boolean value) {
            shouldSucceedInput = value;
            return this;
        }

        public LookbackOnlyTestHelper setShouldSucceedProcessing(boolean value) {
            shouldSucceedProcessing = value;
            return this;
        }

        public void execute() throws Exception {
            createJob(jobId, airlineVariant);
            String datafeedId = "datafeed-" + jobId;
            new DatafeedBuilder(datafeedId, jobId, dataIndex).setScriptedFields(scriptedFields).build();
            openJob(client(), jobId);

            startDatafeedAndWaitUntilStopped(datafeedId);
            waitUntilJobIsClosed(jobId);

            Response jobStatsResponse = client().performRequest(new Request("GET",
                    MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
            String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity());
            if (shouldSucceedInput) {
                assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2"));
            } else {
                assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":0"));
            }
            if (shouldSucceedProcessing) {
                assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2"));
            } else {
                assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":0"));
            }
            assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0"));
        }
    }
    private void startDatafeedAndWaitUntilStopped(String datafeedId) throws Exception {
        startDatafeedAndWaitUntilStopped(datafeedId, BASIC_AUTH_VALUE_SUPER_USER);
    }

    private void startDatafeedAndWaitUntilStopped(String datafeedId, String authHeader) throws Exception {
        Request request = new Request("POST", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_start");
        request.addParameter("start", "2016-06-01T00:00:00Z");
        request.addParameter("end", "2016-06-02T00:00:00Z");
        RequestOptions.Builder options = request.getOptions().toBuilder();
        options.addHeader("Authorization", authHeader);
        request.setOptions(options);
        Response startDatafeedResponse = client().performRequest(request);
        assertThat(EntityUtils.toString(startDatafeedResponse.getEntity()), containsString("\"started\":true"));
        assertBusy(() -> {
            try {
                Response datafeedStatsResponse = client().performRequest(new Request("GET",
                        MachineLearning.BASE_PATH + "datafeeds/" + datafeedId + "/_stats"));
                assertThat(EntityUtils.toString(datafeedStatsResponse.getEntity()),
                        containsString("\"state\":\"stopped\""));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }, 60, TimeUnit.SECONDS);
    }

    private void waitUntilJobIsClosed(String jobId) throws Exception {
        assertBusy(() -> {
            try {
                Response jobStatsResponse = client().performRequest(new Request("GET",
                        MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"));
                assertThat(EntityUtils.toString(jobStatsResponse.getEntity()), containsString("\"state\":\"closed\""));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
    }

    private Response createJob(String id, String airlineVariant) throws Exception {
        Request request = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + id);
        request.setJsonEntity("{\n"
                + "  \"description\": \"Analysis of response time by airline\",\n"
                + "  \"analysis_config\": {\n"
                + "    \"bucket_span\": \"1h\",\n"
                + "    \"detectors\" :[\n"
                + "      {\n"
                + "        \"function\": \"mean\",\n"
                + "        \"field_name\": \"responsetime\",\n"
                + "        \"by_field_name\": \"" + airlineVariant + "\"\n"
                + "      }\n"
                + "    ]\n"
                + "  },\n"
                + "  \"data_description\": {\n"
                + "    \"format\": \"xcontent\",\n"
                + "    \"time_field\": \"time stamp\",\n"
                + "    \"time_format\": \"yyyy-MM-dd'T'HH:mm:ssX\"\n"
                + "  }\n"
                + "}");
        return client().performRequest(request);
    }

    public static void openJob(RestClient client, String jobId) throws IOException {
        client.performRequest(new Request("POST", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_open"));
    }

    @After
    public void clearMlState() throws Exception {
        new MlRestTestStateCleaner(logger, adminClient()).clearMlMetadata();
        // Don't check rollup jobs because we clear them in the superclass.
        // Don't check analytics jobs as they are independent of anomaly detection jobs and should not be created by this test.
        waitForPendingTasks(
            adminClient(),
            taskName -> taskName.startsWith(RollupJob.NAME) || taskName.contains(MlTasks.DATA_FRAME_ANALYTICS_TASK_NAME));
    }

    private static class DatafeedBuilder {
        String datafeedId;
        String jobId;
        String index;
        boolean source;
        String scriptedFields;
        String aggregations;
        String authHeader = BASIC_AUTH_VALUE_SUPER_USER;
        String secondaryAuthHeader = null;
        String chunkingTimespan;
        String indicesOptions;

        DatafeedBuilder(String datafeedId, String jobId, String index) {
            this.datafeedId = datafeedId;
            this.jobId = jobId;
            this.index = index;
        }

        DatafeedBuilder setSource(boolean enableSource) {
            this.source = enableSource;
            return this;
        }

        DatafeedBuilder setScriptedFields(String scriptedFields) {
            this.scriptedFields = scriptedFields;
            return this;
        }

        DatafeedBuilder setAggregations(String aggregations) {
            this.aggregations = aggregations;
            return this;
        }

        DatafeedBuilder setAuthHeader(String authHeader) {
            this.authHeader = authHeader;
            return this;
        }

        DatafeedBuilder setSecondaryAuthHeader(String authHeader) {
            this.secondaryAuthHeader = authHeader;
            return this;
        }

        DatafeedBuilder setChunkingTimespan(String timespan) {
            chunkingTimespan = timespan;
            return this;
        }

        DatafeedBuilder setIndicesOptions(String indicesOptions) {
            this.indicesOptions = indicesOptions;
            return this;
        }

        Response build() throws IOException {
            Request request = new Request("PUT", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId);
            request.setJsonEntity("{"
                    + "\"job_id\": \"" + jobId + "\",\"indexes\":[\"" + index + "\"]"
                    + (source ? ",\"_source\":true" : "")
                    + (scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields)
                    + (aggregations == null ? "" : ",\"aggs\":" + aggregations)
                    + (indicesOptions == null ? "" : ",\"indices_options\":" + indicesOptions)
                    + (chunkingTimespan == null ? "" :
                            ",\"chunking_config\":{\"mode\":\"MANUAL\",\"time_span\":\"" + chunkingTimespan + "\"}")
                    + "}");
            RequestOptions.Builder options = request.getOptions().toBuilder();
            options.addHeader("Authorization", authHeader);
            if (this.secondaryAuthHeader != null) {
                options.addHeader("es-secondary-authorization", secondaryAuthHeader);
            }
            request.setOptions(options);
            return client().performRequest(request);
        }
    }

    private void bulkIndex(String bulk) throws IOException {
        Request bulkRequest = new Request("POST", "/_bulk");
        bulkRequest.setJsonEntity(bulk);
        bulkRequest.addParameter("refresh", "true");
        bulkRequest.addParameter("pretty", null);
        String bulkResponse = EntityUtils.toString(client().performRequest(bulkRequest).getEntity());
        assertThat(bulkResponse, not(containsString("\"errors\": false")));
    }

    private Response createJobAndDataFeed(String jobId, String datafeedId) throws IOException {
        Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId);
        createJobRequest.setJsonEntity("{\n"
            + "  \"description\": \"Aggs job\",\n"
            + "  \"analysis_config\": {\n"
            + "    \"bucket_span\": \"1h\",\n"
            + "    \"summary_count_field_name\": \"doc_count\",\n"
            + "    \"detectors\": [\n"
            + "      {\n"
            + "        \"function\": \"mean\",\n"
            + "        \"field_name\": \"responsetime\",\n"
            + "        \"by_field_name\": \"airline\"\n"
            + "      }\n"
            + "    ]\n"
            + "  },\n"
            + "  \"data_description\": {\"time_field\": \"time stamp\"}\n"
            + "}");
        client().performRequest(createJobRequest);

        String rollupJobId = "rollup-" + jobId;
        Request createRollupRequest = new Request("PUT", "/_rollup/job/" + rollupJobId);
        createRollupRequest.setJsonEntity("{\n"
            + "\"index_pattern\": \"airline-data-aggs\",\n"
            + "    \"rollup_index\": \"airline-data-aggs-rollup\",\n"
            + "    \"cron\": \"*/30 * * * * ?\",\n"
            + "    \"page_size\" :1000,\n"
            + "    \"groups\" : {\n"
            + "      \"date_histogram\": {\n"
            + "        \"field\": \"time stamp\",\n"
            + "        \"fixed_interval\": \"2m\",\n"
            + "        \"delay\": \"7d\"\n"
            + "      },\n"
            + "      \"terms\": {\n"
            + "        \"fields\": [\"airline\"]\n"
            + "      }"
            + "    },\n"
            + "    \"metrics\": [\n"
            + "        {\n"
            + "            \"field\": \"responsetime\",\n"
            + "            \"metrics\": [\"avg\",\"min\",\"max\",\"sum\"]\n"
            + "        },\n"
            + "        {\n"
            + "            \"field\": \"time stamp\",\n"
            + "            \"metrics\": [\"min\",\"max\"]\n"
            + "        }\n"
            + "    ]\n"
            + "}");
        client().performRequest(createRollupRequest);

        String aggregations = "{\"buckets\":{\"date_histogram\":{\"field\":\"time stamp\",\"fixed_interval\":\"3600000ms\"},"
            + "\"aggregations\":{"
            + "\"time stamp\":{\"max\":{\"field\":\"time stamp\"}},"
            + "\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}";

        return new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs-rollup")
            .setAggregations(aggregations)
            .setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS)
            .build();
    }
}

redpajama_set_name: RedPajamaGithub


text:
Currently Selected: • 1930s	• Disaster	• Jungle	• Thriller
Aviation(1)
Creature Features(1)
KING KONG (2005) (2005)
A movie in which a lovelorn supersize gorilla and lonely human belle make heavy eyes at each other, not Lord of the Rings here, but Beauty and the Beast! The film is in three chapters; a movie producer sets sail with cast and crew for the legendary Skull Island; where they are attacked by human sacrificing natives, ravenous dinosaurs, cockroaches as big as beer barrels and scorpions the size of cars, to name but a few; what's left of our less than heroic crew return to New York with the greates	find out more...

redpajama_set_name: RedPajamaCommonCrawl


text:
Maine universities reassure students after harassment is reported nationwide
There have been no major incidents on the seven UMaine System campuses in the wake of the presidential election, but several alleged racist threats on U.S. college campuses are being investigated.
By Noel K. GallagherPortland Press Herald
MACHIAS — Amid tensions and protests on college campuses nationwide over the election of Donald Trump as president, the leaders of Maine's public university system are telling students to report hate crimes and are reassuring them that they are safe on campus.
"While horrendous acts of hate have occurred across the country, and we know people are experiencing anxiety and fear within our community, we all, including the UMaine police, unequivocally stand together and assure that harassment or intimidation of any sort will not be tolerated," University of Maine President Susan Hunter wrote in an email to those on the Orono campus.
Several other presidents sent out similar emails, officials said.
"It's an opportunity to be proactive, while respecting free speech," said Ray Rice, interim president and provost of the University of Maine at Presque Isle. "These concerns are real and their anxiety is real. It is a particularly difficult time."
So far, no major incidents have been reported at any of the seven University of Maine System campuses, campus presidents and system officials said Monday at the board of trustees meeting in Machias.
The Associated Press and other media outlets identified several reports of racist incidents at schools since the election last Tuesday, including a University of Oklahoma student who was identified as having threatened – via text messages – black freshmen at the University of Pennsylvania, Trump's alma mater. At the University of Michigan in Ann Arbor, police are looking into a report of a man who threatened to set a Muslim student on fire with a lighter Friday if she didn't remove her hijab.
In Maine, student representatives to the board of trustees said Monday that tensions are high on campuses and praised leaders for being pro-active in reassuring students and reminding them of what services and resources are available to them on campus.
"It's a hard election for the millennium generation in particular," said Jay Knowlton, the University of Southern Maine graduate student representative to the board.
At a meeting on Sunday, the student reps asked the trustees "to recognize there is some unrest and it would be very wise to restate our policies around that – that hate is not OK," Knowlton said.
Several campuses were also planning forums to discuss the election, each emphasizing that the sessions were an opportunity to consider all views.
At USM, the College of Arts, Humanities and Social Sciences is sponsoring a moderated discussion that will "go beyond the punditry, the mudslinging, and the ridicule to try to understand what is happening in our country."
"If this election indicated anything, it is that we are living in one country, but culturally, we occupy parallel universes," read a flier for the event, to be held on Tuesday.
Rice attached a copy of the UMaine Presque Isle conduct code and freedom of speech and assembly policies to his email.
"The subsequent intolerant incidents that have occurred on some campuses around the country provide an opportunity for the University of Maine at Presque Isle to reaffirm its commitment to respectful dialogue, human dignity and social justice. It is also crucial that we continue to support free speech on our campus," Rice wrote.
Hunter also said any "illegal, abusive or incivil behavior" should be reported to the UMaine police or the Dean of Students Office.
"Our policies and our focus on inclusion and compassion ARE NOT changing. We are not retreating here at UMaine," Hunter wrote in her email.
The University of Maine at Farmington is also planning a forum "for 'election reflection' and giving public voice to emotions," President Kathryn Foster said in an email. "Your thoughtfulness, your understanding of and appreciation for the range of views on campus … and, perhaps most of all, your forthright honesty about your fears, anxieties, apprehensions and hopes reinforce core UMF values of diversity, openness, safety and security, and community well-being," Foster wrote.
Students at multiple Maine colleges – from Southern Maine Community College to Bates College in Lewiston – were planning a walkout at noon Tuesday, part of a coordinated "Stop Trump" campaign nationwide by a group called Student Action that was spreading via Facebook late Monday.
Noel K. Gallagher can be contacted at 791-6387 or at:
Twitter: noelinmaine


text:
UFO Crash Retrievals - Status Report II
The UFO Crash Retrieval Syndrome - New Sources New Data
By: Leonard H. Stringfield
Narrated by: Pete Ferrand
Categories: Nonfiction, Unexplained Phenomena
UFO Crash Retrievals - Status Report I
Retrievals of the Third Kind
Narrated by: Peter Ferrand
Leonard Stringfield was born in 1920. He was director of CRIFO (Civilian Research, Interplanetary Flying Objects) - one of the world's largest research groups during the mid-'50s, and publisher of its newsletter, ORBIT (1953-1957). He also worked in cooperation with the United States Air Force (1953-1957), investigating and reporting UFO activity, having been assigned a special code number for reporting to the Air Defense Command in Columbus, Ohio.
UFO Crash Retrievals - Status Report III: Amassing the Evidence
This is the third entry in Stringfield's influential 7-part UFO Crash Retrieval Status Report series, in which he lays the groundwork for the theories that made him famous.
UFO Crash Retrievals - Status Report V
Is the Cover-Up Lid Lifting?
Good book and great content
By nathan on 09-27-16
UFO Crash Retrievals - Status Report IV
The Fatal Encounter at Ft. Dix-McGuire - A Case Study
Good and informative
The CIA UFO Papers
50 Years of Government Secrets and Cover-Ups
By: Dan Wright, Jan Harzan - foreword
Narrated by: Kevin Kenerly
In autumn 2016, the CIA sent to its website a cache of electronic files previously released under the Freedom of Information Act but housed at the National Archives. Among a variety of subjects were "unidentified flying objects." Finally, a stockpile of reports and correspondences were available for serious UFO researchers to examine at home. This book consists of selections from those secret files.
Unexpected surprise, as no reviews existed yet. Excellent book!
By Boom Depleter on 10-06-19
UFO Crash Retrievals - Status Report VII: Search for Proof in a Hall of Mirrors
This is the finale in Leonard Stringfield's influential seven-part UFO Crash Retrieval Status Report series, in which he summarizes the groundbreaking theories that made him famous. Stringfield, the first researcher to exclusively focus on UFO crashes, shares many intriguing, eye-opening details about a number of major UFO cases. This extremely rare volume has been lovingly reproduced exactly as it was first published 35 years ago. The material inside, which was shocking to many observers in those days, is finally gaining acceptance.
UFO Crash Retrievals: The Inner Sanctum
Status Report VI
Leonard Stringfield was born in 1920. He was director of CRIFO (Civilian Research, Interplanetary Flying Objects) - one of the world's largest research groups during the mid-50s, and publisher of its newsletter, ORBIT (1953-1957).
Witness to Roswell
Unmasking the Government's Biggest Cover-Up
By: Thomas J. Carey, Donald R. Schmitt
Narrated by: Eric Michael Summerer
The best-selling UFO book of 2007/2008 is revised and updated with even more hard-hitting eyewitness testimony of one of the most important events of all time: the actual recovery of a UFO outside of Roswell, New Mexico, in 1947. For more than 60 years, government authorities have led us to believe the wreckage was merely a very conventional weather balloon - but the witnesses who were there continue to tell a different story.
The best book on the world's most famous UFO case.
By Noe on 01-13-11
An Alien Enterprise: The Shocking Truth Behind the Greatest Cover-Up in Human History
By: Timothy Good
Narrated by: Stephen Bel Davies
This fascinating new volume tells the story of contact between aliens and humans from all across the globe, dating back to 1932, including meetings with military personnel and American presidents such as Eisenhower and Kennedy. For the first time, a former member of MI6 reveals her conversation with Neil Armstrong at a NASA conference, when she confirmed that there were "other" spacecraft on the moon when Apollo 11 landed in 1969. Armstrong also confirmed that the CIA was behind the cover-up.
This is a book!
By Janalyn on 12-10-19
Our Haunted Planet
By: John Keel
Narrated by: Michael Hacker
In the classic Our Haunted Planet, John A. Keel brings into chilling focus strange truths about the Earth and its mysterious inhabitants. Could an unseen, prehuman race have taken careful measures to remain hidden from surface dwellers? Are they still watching us from their secret hiding places, manipulating and misleading us, using us for their own entertainment, and controlling our actions?
Keel's 2nd best work
The Worldwide UFO Cover-Up
Narrated by: Victor Talmadge
Renowned UFO researcher Timothy Good has gathered reports from all over the world to show that UFO sightings are an international phenomenon. In Above Top Secret, Good argues that U.S. government agencies, in collaboration with the Defense Intelligence Agency, the CIA, and NASA, have conspired to keep people from the truth about UFOs. The author presents persuasive evidence for his argument in this well-documented work.
Good but not the best
By Val on 03-12-12
The Eighth Tower
On Ultraterrestrials and the Superspectrum
By: John A. Keel
Is there a single intelligent force behind all religious, occult, and UFO phenomena? Strange manifestations have haunted humans since prehistoric times. Beams of light, voices from the heavens, the "little people", gods and devils, ghosts and monsters, and UFOs have all had a prominent place in our history and legends. In this dark work, John Keel explores these phenomena, and in doing so reveals the shocking truth about our present position and future destiny in the cosmic scheme of things.
Great book, really makes you think
By Cameron on 04-03-19
Incident at Devil's Den
By: Terry Lovelace Esq.
Narrated by: Terry J. Lovelace
I'm a 64-year-old retired lawyer and former Assistant Attorney General with an unusual story to tell. Every word of it is true. It happened in a state park known as Devil's Den. For fear of losing my job and damaging my reputation in the legal community, I never spoke about the incident. If it hadn't happened to me, I'd be skeptical too. In 1977, I was an active duty NCO in the USAF. A friend and I planned a two-night camping trip to explore the wilderness. We drove deep into the forest until we came to a high plateau. There on the horizon sat a perfect triangle of three very bright stars....
Breathtaking facts you can't unknow.
By Daniel Martin on 12-08-18
Situation Red
The UFO Siege
In 1973, a virtual epidemic of flying saucer sightings occurred. What was most disturbing was that they seemed to be different from earlier sightings. For instance, there was an increasing incidence of physical injury, dangerous interference, and outright abduction of observers by mysterious beings associated with the phenomena.
Absolutely horrible narrator
By Isaac McDonald on 08-03-16
The Aztec UFO Incident
The Case, Evidence, and Elaborate Cover-Up of One of the Most Perplexing Crashes in History
By: Scott Ramsey, Suzanne Ramsey, Frank Thayer
Narrated by: Paul Woodson
The Aztec UFO Incident - the first ever widely publicized report of a recovered flying saucer - was derided as a hoax for decades. But now the Ramseys and Frank Thayer reveal the exact spot where the craft landed and show how the 100-foot-diameter saucer was moved to a secret laboratory. Witnesses to the incident who were interviewed by the authors affirm that they were sworn to secrecy by the military. The authors also reveal the names of scientists who worked on the craft after its recovery.
Hard research only, with little speculation.
By jebidia24 on 07-25-16
Hunt for the Skinwalker
Science Confronts the Unexplained at a Remote Ranch in Utah
By: Colm A. Kelleher Ph.D
Narrated by: David Bendena
For more than 50 years, bizarre events at a remote Utah ranch have ranged from the perplexing to the wholly terrifying: vanishing and mutilated cattle, huge otherworldly creatures, invisible objects emitting magnetic fields, and more. For the family living at Skinwalker Ranch, life was under siege, and no one had been able to explain the horrors that surrounded them...But maybe science could. Leading a first-class team of research scientists, Colm Kelleher spent hundreds of days and nights on the Skinwalker property.
Buy this! No BS!
By Neil S. on 11-14-17
An Autobiography
By: Bob Lazar, George Knapp - foreword
Narrated by: Barry Abrams
Bob Lazar was a brilliant young physicist that found himself employed at a top-secret facility in the middle of the desert outside Las Vegas. Under the watchful eye of the government elite, he is tasked with understanding an exotic propulsion system being used by an advanced aerospace vehicle he is told came from outer space. The stressful work and long, odd hours start to wear on Bob and he becomes concerned for his safety. He tells his wife and a couple close friends about what he's doing in the desert, and his employers find out and are furious.
Great summary of Bob Lazar's story
By AA on 11-03-19
Operation Trojan Horse
The Classic Breakthrough Study of UFOs
When this work was first published in 1970, it shredded the then-trendy nuts-and-bolts extraterrestrial hypothesis for UFOs in favor of one that linked UFOs to a variety of paranormal and supernatural phenomena that have taken place throughout history. Keel was one of the first to note that the UFO phenomenon appears in different disguises - and that one could not begin to decipher this great mystery without first taking into account its many and varied deliberate deceptions.
It's a stretch
By Carol on 02-21-19
Leonard Stringfield was born in 1920. He was director of CRIFO (Civilian Research, Interplanetary Flying Objects) - one of the world's largest research groups during the mid-50s, and publisher of its newsletter, ORBIT (1953-1957). He also worked in cooperation with the United States Air Force (1953-1957), investigating and reporting UFO activity, having been assigned a special code number for reporting to the Air Defense Command in Columbus, Ohio. For over 30 years, Stringfield served in several of the major UFO organizations in a public relations capacity. From 1957 to 1970, he was public relations adviser with the National Investigations Committee on Aerial Phenomena (NICAP). Later on, he was Director of Public Relations and a board member of the Mutual UFO Network (MUFON). He was also a regional investigator for the Center for UFO Studies, directed by Dr. J. Allen Hynek. Stringfield was an executive with DuBois Chemicals, an international manufacturer. Stringfield retired in 1981 as Director of Public Relations and Marketing Services for DuBois Chemicals, a division of Chemed Corp., after 31 years of service with the company. From 1967-1968 he served as Early Warning Coordinator for the University of Colorado UFO Project, screening UFO reports for possible scientific study. Stringfield was also an advisor to Sir Eric Gairy, former Prime Minister of Grenada, 1977-78, during his efforts to establish a UFO research agency within the framework of the United Nations. Stringfield passed away in 1994, leaving a string of fascinating publications behind, one of which is Crash Retrievals: Status Report II: The UFO Crash Retrieval Syndrome - New Sources, New Data, based on a talk Stringfield gave at a MUFON convention in 1978.
©1982 Saucerian Press (P)2015 New Saucerian Press
Dave Ramsey's The Total Money Makeover | Summary & Review
Motivation: Stories on Life and Success from Brazilian Jiu-Jitsu Black Belts


Use styles to structure your document: headings, lists, bold etc.
Save document as plain text, in a format that preserves styles.
Writage supports Microsoft Office 2010/2013/2016 on Windows.
Writage is in active development now and available for free trial.
The main idea of Writage is providing opportunity to write in Markdown to those who like Microsoft Word. There are lots of Markdown editors such as iA Writer, Texts, WriteMonkey or Byword and online Markdown writing services, like StackEdit, Dillinger or Markable for people who like to have minimalistic, distraction-free interface. And there is nothing for those who prefer to use rich interface and features of Microsoft Word.
So Writage is designed for everyone who likes to start writing well-structured documents and don't have time or don't want to drill into details Markdown syntax or prefer to use Word as a text editor.
Writage enables Microsoft Word to create and edit Markdown files (with .md .markdown or .text extensions). You could also save your .doc file as Markdown file.
The installation process is very simple and takes just a few minutes.
Double-click on the file «Writage.msi» and follow the instructions. Please, note that you should have «Administrator» rights on your computer.
After finishing installation your Writage plugin is ready to use.
Uninstaller for Writage can be invoked via Control Panel.
Writage turns Word into your Markdown WYSIWYG editor, so you will be able to open a Markdown file and edit it like you normally edit any document in Microsoft Word. Also it will be possible to save your Word document as a Markdown file without any other converters.
On receiving Markdown document from other people or downloading from the web you don't have to install any additional software to be able to read it as a normal document (without formatting tags which might be destructive for readers).
Choose «Open» command and choose Markdown document you like.
File will be shown like any other document.
If you don't like the idea of using Microsoft Word for writing you could simply save any Word document which you receive from you colleague as Markdown file and continue to work with your favorite Markdown tool.
Launch Microsoft Word and create sample document.
Choose «Save as…» command and choose «Markdown» from «Save as type» field.
To be able to use rich formatting functions of Microsoft Word or simply to share your Markdown file with people who use Microsoft Word you should convert a Markdown file into a Word document.
Choose «Save as…» command and choose «Word Document» from «Save as type» field.
Writage also adds commands for copying selected text as Markdown, and for pasting Markdown as rich text.
Zero knowledge of Markdown is required in order to write well-structured documents with Writage. No need to remember markup for hyperlink or footnote, or how to format a table.
Microsoft Word interface is rich, offering document outline, spell checking, search and lots of other features important for writing.
Please send emails to support@writage.com.
Receive email when new version of Writage becomes available.