text:
package org.elasticsearch.xpack.sql.qa.rest;

import com.fasterxml.jackson.core.io.JsonStringEncoder;

import org.apache.http.HttpEntity;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.test.NotEqualMessageBuilder;
import org.elasticsearch.xpack.sql.proto.Mode;
import org.elasticsearch.xpack.sql.proto.StringUtils;
import org.elasticsearch.xpack.sql.qa.ErrorsTestCase;
import org.hamcrest.Matcher;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.sql.JDBCType;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static java.util.Collections.unmodifiableMap;
import static org.elasticsearch.xpack.ql.TestUtils.getNumberOfSearchContexts;
import static org.hamcrest.Matchers.containsString;

/**
 * Integration test for the rest sql action. The one that speaks json directly to a
 * user rather than to the JDBC driver or CLI.
 */
public abstract class RestSqlTestCase extends BaseRestSqlTestCase implements ErrorsTestCase {

    public static String SQL_QUERY_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_QUERY_REST_ENDPOINT;
    private static String SQL_TRANSLATE_REST_ENDPOINT = org.elasticsearch.xpack.sql.proto.Protocol.SQL_TRANSLATE_REST_ENDPOINT;

    /**
     * Builds that map that is returned in the header for each column.
     */
    public static Map<String, Object> columnInfo(String mode, String name, String type, JDBCType jdbcType, int size) {
        Map<String, Object> column = new HashMap<>();
        column.put("name", name);
        column.put("type", type);
        if ("jdbc".equals(mode)) {
            column.put("display_size", size);
        }
        return unmodifiableMap(column);
    }

    public void testBasicQuery() throws IOException {
        index("{\"test\":\"test\"}", "{\"test\":\"test\"}");

        Map<String, Object> expected = new HashMap<>();
        String mode = randomMode();
        boolean columnar = randomBoolean();

        expected.put("columns", singletonList(columnInfo(mode, "test", "text", JDBCType.VARCHAR, Integer.MAX_VALUE)));
        if (columnar) {
            expected.put("values", singletonList(Arrays.asList("test", "test")));
        } else {
            expected.put("rows", Arrays.asList(singletonList("test"), singletonList("test")));
        }
        assertResponse(expected, runSql(mode, "SELECT * FROM test", columnar));
    }

    public void testNextPage() throws IOException {
        Request request = new Request("POST", "/test/_bulk");
        request.addParameter("refresh", "true");
        String mode = randomMode();
        StringBuilder bulk = new StringBuilder();
        for (int i = 0; i < 20; i++) {
            bulk.append("{\"index\":{\"_id\":\"" + i + "\"}}\n");
            bulk.append("{\"text\":\"text" + i + "\", \"number\":" + i + "}\n");
        }
        request.setJsonEntity(bulk.toString());
        client().performRequest(request);

        boolean columnar = randomBoolean();
        String sqlRequest = query("SELECT text, number, SQRT(number) AS s, SCORE()" + "     FROM test" + " ORDER BY number, SCORE()").mode(
            mode
        ).fetchSize(2).columnar(columnarValue(columnar)).toString();

        Number value = xContentDependentFloatingNumberValue(mode, 1f);
        String cursor = null;
        for (int i = 0; i < 20; i += 2) {
            Map<String, Object> response;
            if (i == 0) {
                response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "", mode);
            } else {
                columnar = randomBoolean();
                response = runSql(
                    new StringEntity(cursor(cursor).mode(mode).columnar(columnarValue(columnar)).toString(), ContentType.APPLICATION_JSON),
                    StringUtils.EMPTY,
                    mode
                );
            }

            Map<String, Object> expected = new HashMap<>();
            if (i == 0) {
                expected.put(
                    "columns",
                    Arrays.asList(
                        columnInfo(mode, "text", "text", JDBCType.VARCHAR, Integer.MAX_VALUE),
                        columnInfo(mode, "number", "long", JDBCType.BIGINT, 20),
                        columnInfo(mode, "s", "double", JDBCType.DOUBLE, 25),
                        columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15)
                    )
                );
            }

            if (columnar) {
                expected.put(
                    "values",
                    Arrays.asList(
                        Arrays.asList("text" + i, "text" + (i + 1)),
                        Arrays.asList(i, i + 1),
                        Arrays.asList(Math.sqrt(i), Math.sqrt(i + 1)),
                        Arrays.asList(value, value)
                    )
                );
            } else {
                expected.put(
                    "rows",
                    Arrays.asList(
                        Arrays.asList("text" + i, i, Math.sqrt(i), value),
                        Arrays.asList("text" + (i + 1), i + 1, Math.sqrt(i + 1), value)
                    )
                );
            }
            cursor = (String) response.remove("cursor");
            assertResponse(expected, response);
            assertNotNull(cursor);
        }
        Map<String, Object> expected = new HashMap<>();
        columnar = randomBoolean();
        if (columnar) {
            expected.put("values", emptyList());
        } else {
            expected.put("rows", emptyList());
        }
        assertResponse(
            expected,
            runSql(
                new StringEntity(cursor(cursor).mode(mode).columnar(columnarValue(columnar)).toString(), ContentType.APPLICATION_JSON),
                StringUtils.EMPTY,
                mode
            )
        );
    }

    public void testNextPageWithDatetimeAndTimezoneParam() throws IOException {
        Request request = new Request("PUT", "/test_date_timezone");
        XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
        createIndex.startObject("mappings");
        {
            createIndex.startObject("properties");
            {
                createIndex.startObject("date").field("type", "date").field("format", "epoch_millis");
                createIndex.endObject();
            }
            createIndex.endObject();
        }
        createIndex.endObject().endObject();
        request.setJsonEntity(Strings.toString(createIndex));
        client().performRequest(request);

        request = new Request("PUT", "/test_date_timezone/_bulk");
        request.addParameter("refresh", "true");
        StringBuilder bulk = new StringBuilder();
        long[] datetimes = new long[] { 1_000, 10_000, 100_000, 1_000_000, 10_000_000 };
        for (long datetime : datetimes) {
            bulk.append("{\"index\":{}}\n");
            bulk.append("{\"date\":").append(datetime).append("}\n");
        }
        request.setJsonEntity(bulk.toString());
        assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode());

        ZoneId zoneId = randomZone();
        String mode = randomMode();
        String sqlRequest = query("SELECT DATE_PART('TZOFFSET', date) AS tz FROM test_date_timezone ORDER BY date").timeZone(zoneId.getId())
            .mode(mode)
            .fetchSize(2)
            .toString();

        String cursor = null;
        for (int i = 0; i <= datetimes.length; i += 2) {
            Map<String, Object> expected = new HashMap<>();
            Map<String, Object> response;

            if (i == 0) {
                expected.put("columns", singletonList(columnInfo(mode, "tz", "integer", JDBCType.INTEGER, 11)));
                response = runSql(new StringEntity(sqlRequest, ContentType.APPLICATION_JSON), "", mode);
            } else {
                response = runSql(
                    new StringEntity(cursor(cursor).mode(mode).toString(), ContentType.APPLICATION_JSON),
                    StringUtils.EMPTY,
                    mode
                );
            }

            List<Object> values = new ArrayList<>(2);
            for (int j = 0; j < (i < datetimes.length - 1 ? 2 : 1); j++) {
                values.add(
                    singletonList(
                        ZonedDateTime.ofInstant(Instant.ofEpochMilli(datetimes[i + j]), zoneId).getOffset().getTotalSeconds() / 60
                    )
                );
            }
            expected.put("rows", values);
            cursor = (String) response.remove("cursor");
            assertResponse(expected, response);
            assertNotNull(cursor);
        }
        Map<String, Object> expected = new HashMap<>();
        expected.put("rows", emptyList());
        assertResponse(
            expected,
            runSql(new StringEntity(cursor(cursor).mode(mode).toString(), ContentType.APPLICATION_JSON), StringUtils.EMPTY, mode)
        );
    }

    @AwaitsFix(bugUrl = "Unclear status, https://github.com/elastic/x-pack-elasticsearch/issues/2074")
    public void testTimeZone() throws IOException {
        String mode = randomMode();
        boolean columnar = randomBoolean();
        index("{\"test\":\"2017-07-27 00:00:00\"}", "{\"test\":\"2017-07-27 01:00:00\"}");

        Map<String, Object> expected = new HashMap<>();
        expected.put("columns", singletonMap("test", singletonMap("type", "text")));
        if (columnar) {
            expected.put("values", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
        } else {
            // TODO: what exactly is this test suppossed to do. We need to check the 2074 issue above.
            expected.put("rows", Arrays.asList(singletonMap("test", "test"), singletonMap("test", "test")));
        }
        expected.put("size", 2);

        // Default TimeZone is UTC
        assertResponse(expected, runSql(mode, "SELECT DAY_OF_YEAR(test), COUNT(*) FROM test", columnar));
    }

    public void testScoreWithFieldNamedScore() throws IOException {
        Request request = new Request("POST", "/test/_bulk");
        request.addParameter("refresh", "true");
        String mode = randomMode();
        StringBuilder bulk = new StringBuilder();
        bulk.append("{\"index\":{\"_id\":\"1\"}}\n");
        bulk.append("{\"name\":\"test\", \"score\":10}\n");
        request.setJsonEntity(bulk.toString());
        client().performRequest(request);

        Map<String, Object> expected = new HashMap<>();
        boolean columnar = randomBoolean();
        expected.put(
            "columns",
            Arrays.asList(
                columnInfo(mode, "name", "text", JDBCType.VARCHAR, Integer.MAX_VALUE),
                columnInfo(mode, "score", "long", JDBCType.BIGINT, 20),
                columnInfo(mode, "SCORE()", "float", JDBCType.REAL, 15)
            )
        );
        Number value = xContentDependentFloatingNumberValue(mode, 1f);
        if (columnar) {
            expected.put("values", Arrays.asList(singletonList("test"), singletonList(10), singletonList(value)));
        } else {
            expected.put("rows", singletonList(Arrays.asList("test", 10, value)));
        }

        assertResponse(expected, runSql(mode, "SELECT *, SCORE() FROM test ORDER BY SCORE()", columnar));
        assertResponse(expected, runSql(mode, "SELECT name, \\\"score\\\", SCORE() FROM test ORDER BY SCORE()", columnar));
    }

    public void testSelectWithJoinFails() throws Exception {
        // Normal join not supported
        expectBadRequest(
            () -> runSql(randomMode(), "SELECT * FROM test JOIN other"),
            containsString("line 1:21: Queries with JOIN are not yet supported")
        );
        // Neither is a self join
        expectBadRequest(
            () -> runSql(randomMode(), "SELECT * FROM test JOIN test"),
            containsString("line 1:21: Queries with JOIN are not yet supported")
        );
        // Nor fancy stuff like CTEs
        expectBadRequest(
            () -> runSql(
                randomMode(),
                "    WITH evil" + "  AS (SELECT *" + "        FROM foo)" + "SELECT *" + "  FROM test" + "  JOIN evil"
            ),
            containsString("line 1:67: Queries with JOIN are not yet supported")
        );
    }

    public void testSelectGroupByAllFails() throws Exception {
        index("{\"foo\":1}", "{\"foo\":2}");
        expectBadRequest(
            () -> runSql(randomMode(), "SELECT foo FROM test GROUP BY ALL foo"),
            containsString("line 1:32: GROUP BY ALL is not supported")
        );
    }

    @Override
    public void testSelectInvalidSql() {
        String mode = randomFrom("jdbc", "plain");
        expectBadRequest(() -> runSql(mode, "SELECT * FRO"), containsString("1:8: Cannot determine columns for [*]"));
    }

    @Override
    public void testSelectFromMissingIndex() {
        String mode = randomFrom("jdbc", "plain");
        expectBadRequest(() -> runSql(mode, "SELECT * FROM missing"), containsString("1:15: Unknown index [missing]"));
    }

    @Override
    public void testSelectColumnFromMissingIndex() throws Exception {
        String mode = randomFrom("jdbc", "plain");
        expectBadRequest(() -> runSql(mode, "SELECT abc FROM missing"), containsString("1:17: Unknown index [missing]"));
    }

    @Override
    public void testSelectFromEmptyIndex() throws Exception {
        // Create an index without any types
        Request request = new Request("PUT", "/test");
        request.setJsonEntity("{}");
        client().performRequest(request);
        String mode = randomFrom("jdbc", "plain");
        expectBadRequest(() -> runSql(mode, "SELECT * FROM test"), containsString("1:8: Cannot determine columns for [*]"));
    }

    @Override
    public void testSelectColumnFromEmptyIndex() throws Exception {
        Request request = new Request("PUT", "/test");
        request.setJsonEntity("{}");
        client().performRequest(request);
        String mode = randomFrom("jdbc", "plain");
        expectBadRequest(() -> runSql(mode, "SELECT abc FROM test"), containsString("1:8: Unknown column [abc]"));
    }

    @Override
    public void testSelectMissingField() throws IOException {
        index("{\"test\":\"test\"}");
        String mode = randomFrom("jdbc", "plain");
        expectBadRequest(() -> runSql(mode, "SELECT foo FROM test"), containsString("1:8: Unknown column [foo]"));
    }

    @Override
    public void testSelectMissingFunction() throws Exception {
        index("{\"foo\":1}");
        expectBadRequest(() -> runSql(randomMode(), "SELECT missing(foo) FROM test"), containsString("1:8: Unknown function [missing]"));
    }

    @Override
    public void testSelectProjectScoreInAggContext() throws Exception {
        index("{\"foo\":1}");
        expectBadRequest(
            () -> runSql(randomMode(), "     SELECT foo, SCORE(), COUNT(*)" + "     FROM test" + " GROUP BY foo"),
            containsString("Cannot use non-grouped column [SCORE()], expected [foo]")
        );
    }

    @Override
    public void testSelectOrderByScoreInAggContext() throws Exception {
        index("{\"foo\":1}");
        expectBadRequest(
            () -> runSql(randomMode(), "     SELECT foo, COUNT(*)" + "     FROM test" + " GROUP BY foo" + " ORDER BY SCORE()"),
            containsString("Cannot order by non-grouped column [SCORE()], expected [foo]")
        );
    }

    @Override
    public void testSelectGroupByScore() throws Exception {
        index("{\"foo\":1}");
        expectBadRequest(
            () -> runSql(randomMode(), "SELECT COUNT(*) FROM test GROUP BY SCORE()"),
            containsString("Cannot use [SCORE()] for grouping")
        );
    }

    public void testCountAndCountDistinct() throws IOException {
        String mode = randomMode();
        index(
            "test",
            "{\"gender\":\"m\", \"langs\": 1}",
            "{\"gender\":\"m\", \"langs\": 1}",
            "{\"gender\":\"m\", \"langs\": 2}",
            "{\"gender\":\"m\", \"langs\": 3}",
            "{\"gender\":\"m\", \"langs\": 3}",
            "{\"gender\":\"f\", \"langs\": 1}",
            "{\"gender\":\"f\", \"langs\": 2}",
            "{\"gender\":\"f\", \"langs\": 2}",
            "{\"gender\":\"f\", \"langs\": 2}",
            "{\"gender\":\"f\", \"langs\": 3}",
            "{\"gender\":\"f\", \"langs\": 3}"
        );

        Map<String, Object> expected = new HashMap<>();
        boolean columnar = randomBoolean();
        expected.put(
            "columns",
            Arrays.asList(
                columnInfo(mode, "gender", "text", JDBCType.VARCHAR, Integer.MAX_VALUE),
                columnInfo(mode, "cnt", "long", JDBCType.BIGINT, 20),
                columnInfo(mode, "cnt_dist", "long", JDBCType.BIGINT, 20)
            )
        );
        if (columnar) {
            expected.put("values", Arrays.asList(Arrays.asList("f", "m"), Arrays.asList(6, 5), Arrays.asList(3, 3)));
        } else {
            expected.put("rows", Arrays.asList(Arrays.asList("f", 6, 3), Arrays.asList("m", 5, 3)));
        }

        Map<String, Object> response = runSql(
            mode,
            "SELECT gender, COUNT(langs) AS cnt, COUNT(DISTINCT langs) AS cnt_dist " + "FROM test GROUP BY gender ORDER BY gender",
            columnar
        );

        String cursor = (String) response.remove("cursor");
        assertNotNull(cursor);
        assertResponse(expected, response);
    }

    @Override
    public void testSelectScoreSubField() throws Exception {
        index("{\"foo\":1}");
        expectBadRequest(
            () -> runSql(randomMode(), "SELECT SCORE().bar FROM test"),
            containsString("line 1:15: extraneous input '.' expecting {<EOF>, ','")
        );
    }

    @Override
    public void testHardLimitForSortOnAggregate() throws Exception {
        index("{\"a\": 1, \"b\": 2}");
        expectBadRequest(
            () -> runSql(randomMode(), "SELECT max(a) max FROM test GROUP BY b ORDER BY max LIMIT 120000"),
            containsString("The maximum LIMIT for aggregate sorting is [65536], received [120000]")
        );
    }

    public void testUseColumnarForUnsupportedFormats() throws Exception {
        String format = randomFrom("txt", "csv", "tsv");
        index("{\"foo\":1}");

        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
        request.addParameter("error_trace", "true");
        request.addParameter("pretty", "true");
        request.addParameter("format", format);
        request.setEntity(
            new StringEntity(
                query("SELECT * FROM test").mode(randomValueOtherThan(Mode.JDBC.toString(), BaseRestSqlTestCase::randomMode))
                    .columnar(true)
                    .toString(),
                ContentType.APPLICATION_JSON
            )
        );
        expectBadRequest(() -> {
            client().performRequest(request);
            return emptyMap();
        }, containsString("Invalid use of [columnar] argument: cannot be used in combination with txt, csv or tsv formats"));
    }

    public void testUseColumnarForTranslateRequest() throws IOException {
        index("{\"test\":\"test\"}", "{\"test\":\"test\"}");

        String mode = randomMode();
        Request request = new Request("POST", SQL_TRANSLATE_REST_ENDPOINT);
        request.setEntity(new StringEntity(query("SELECT * FROM test").mode(mode).columnar(true).toString(), ContentType.APPLICATION_JSON));
        expectBadRequest(() -> {
            client().performRequest(request);
            return emptyMap();
        }, containsString("unknown field [columnar]"));
    }

    public void testValidateRuntimeMappingsInSqlQuery() throws IOException {
        testValidateRuntimeMappingsInQuery(SQL_QUERY_REST_ENDPOINT);

        String mode = randomMode();
        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
        index("{\"test\":true}", "{\"test\":false}");
        String runtimeMappings = "{\"bool_as_long\": {\"type\":\"long\", \"script\": {\"source\":\"if(doc['test'].value == true) emit(1);"
            + "else emit(0);\"}}}";
        request.setEntity(
            new StringEntity(
                query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(),
                ContentType.APPLICATION_JSON
            )
        );
        Map<String, Object> expected = new HashMap<>();
        expected.put(
            "columns",
            Arrays.asList(
                columnInfo(mode, "bool_as_long", "long", JDBCType.BIGINT, 20),
                columnInfo(mode, "test", "boolean", JDBCType.BOOLEAN, 1)
            )
        );
        expected.put("rows", Arrays.asList(Arrays.asList(1, true), Arrays.asList(0, false)));
        assertResponse(
            expected,
            runSql(
                new StringEntity(
                    query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(),
                    ContentType.APPLICATION_JSON
                ),
                StringUtils.EMPTY,
                mode
            )
        );
    }

    public void testValidateRuntimeMappingsInTranslateQuery() throws IOException {
        testValidateRuntimeMappingsInQuery(SQL_TRANSLATE_REST_ENDPOINT);

        index("{\"test\":true}", "{\"test\":false}");
        String runtimeMappings = "{\"bool_as_long\": {\"type\":\"long\", \"script\": {\"source\":\"if(doc['test'].value == true) emit(1);"
            + "else emit(0);\"}}}";
        Map<String, Object> response = runTranslateSql(query("SELECT * FROM test").runtimeMappings(runtimeMappings).toString());
        assertEquals(response.get("size"), 1000);
        assertFalse((Boolean) response.get("_source"));
        @SuppressWarnings("unchecked")
        List<Map<String, Object>> source = (List<Map<String, Object>>) response.get("fields");
        assertEquals(Arrays.asList(singletonMap("field", "bool_as_long"), singletonMap("field", "test")), source);

        assertNull(response.get("query"));

        @SuppressWarnings("unchecked")
        List<Map<String, Object>> sort = (List<Map<String, Object>>) response.get("sort");
        assertEquals(singletonList(singletonMap("_doc", singletonMap("order", "asc"))), sort);
    }

    private static void testValidateRuntimeMappingsInQuery(String queryTypeEndpoint) {
        String mode = randomMode();
        String runtimeMappings = "{\"address\": {\"script\": \"return\"}}";
        Request request = new Request("POST", queryTypeEndpoint);
        request.setEntity(
            new StringEntity(
                query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(),
                ContentType.APPLICATION_JSON
            )
        );
        expectBadRequest(() -> {
            client().performRequest(request);
            return emptyMap();
        }, containsString("No type specified for runtime field [address]"));

        runtimeMappings = "{\"address\": [{\"script\": \"return\"}]}";
        request.setEntity(
            new StringEntity(
                query("SELECT * FROM test").mode(mode).runtimeMappings(runtimeMappings).toString(),
                ContentType.APPLICATION_JSON
            )
        );
        expectBadRequest(() -> {
            client().performRequest(request);
            return emptyMap();
        }, containsString("Expected map for runtime field [address] definition but got [String]"));
    }

    public static void expectBadRequest(CheckedSupplier<Map<String, Object>, Exception> code, Matcher<String> errorMessageMatcher) {
        try {
            Map<String, Object> result = code.get();
            fail("expected ResponseException but got " + result);
        } catch (ResponseException e) {
            if (400 != e.getResponse().getStatusLine().getStatusCode()) {
                String body;
                try {
                    body = Streams.copyToString(new InputStreamReader(e.getResponse().getEntity().getContent(), StandardCharsets.UTF_8));
                } catch (IOException bre) {
                    throw new RuntimeException("error reading body after remote sent bad status", bre);
                }
                fail("expected [400] response but get [" + e.getResponse().getStatusLine().getStatusCode() + "] with body:\n" + body);
            }
            assertThat(e.getMessage(), errorMessageMatcher);
        } catch (Exception e) {
            throw new AssertionError("expected ResponseException but got [" + e.getClass() + "]", e);
        }
    }

    private Map<String, Object> runSql(String mode, String sql) throws IOException {
        return runSql(mode, sql, StringUtils.EMPTY, randomBoolean());
    }

    private Map<String, Object> runSql(String mode, String sql, boolean columnar) throws IOException {
        return runSql(mode, sql, StringUtils.EMPTY, columnar);
    }

    private Map<String, Object> runSql(String mode, String sql, String suffix, boolean columnar) throws IOException {
        // put an explicit "columnar": false parameter or omit it altogether, it should make no difference
        return runSql(
            new StringEntity(query(sql).mode(mode).columnar(columnarValue(columnar)).toString(), ContentType.APPLICATION_JSON),
            suffix,
            mode
        );
    }

    protected Map<String, Object> runTranslateSql(String sql) throws IOException {
        return runSql(new StringEntity(sql, ContentType.APPLICATION_JSON), "/translate/", Mode.PLAIN.toString());
    }

    private static Boolean columnarValue(boolean columnar) {
        return columnar ? Boolean.TRUE : (randomBoolean() ? null : Boolean.FALSE);
    }

    protected Map<String, Object> runSql(HttpEntity sql, String suffix, String mode) throws IOException {
        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT + suffix);
        request.addParameter("error_trace", "true");   // Helps with debugging in case something crazy happens on the server.
        request.addParameter("pretty", "true");        // Improves error reporting readability
        if (randomBoolean()) {
            // We default to JSON but we force it randomly for extra coverage
            request.addParameter("format", "json");
        }
        if (randomBoolean()) {
            // JSON is the default but randomly set it sometime for extra coverage
            RequestOptions.Builder options = request.getOptions().toBuilder();
            options.addHeader("Accept", randomFrom("*/*", "application/json"));
            request.setOptions(options);
        }
        request.setEntity(sql);
        return toMap(client().performRequest(request), mode);
    }

    public void testPrettyPrintingEnabled() throws IOException {
        boolean columnar = randomBoolean();
        String expected = "";
        if (columnar) {
            expected = "{\n"
                + "  \"columns\" : [\n"
                + "    {\n"
                + "      \"name\" : \"test1\",\n"
                + "      \"type\" : \"text\"\n"
                + "    }\n"
                + "  ],\n"
                + "  \"values\" : [\n"
                + "    [\n"
                + "      \"test1\",\n"
                + "      \"test2\"\n"
                + "    ]\n"
                + "  ]\n"
                + "}\n";
        } else {
            expected = "{\n"
                + "  \"columns\" : [\n"
                + "    {\n"
                + "      \"name\" : \"test1\",\n"
                + "      \"type\" : \"text\"\n"
                + "    }\n"
                + "  ],\n"
                + "  \"rows\" : [\n"
                + "    [\n"
                + "      \"test1\"\n"
                + "    ],\n"
                + "    [\n"
                + "      \"test2\"\n"
                + "    ]\n"
                + "  ]\n"
                + "}\n";
        }
        executeAndAssertPrettyPrinting(expected, "true", columnar);
    }

    public void testPrettyPrintingDisabled() throws IOException {
        boolean columnar = randomBoolean();
        String expected = "";
        if (columnar) {
            expected = "{\"columns\":[{\"name\":\"test1\",\"type\":\"text\"}],\"values\":[[\"test1\",\"test2\"]]}";
        } else {
            expected = "{\"columns\":[{\"name\":\"test1\",\"type\":\"text\"}],\"rows\":[[\"test1\"],[\"test2\"]]}";
        }
        executeAndAssertPrettyPrinting(expected, randomFrom("false", null), columnar);
    }

    private void executeAndAssertPrettyPrinting(String expectedJson, String prettyParameter, boolean columnar) throws IOException {
        index("{\"test1\":\"test1\"}", "{\"test1\":\"test2\"}");

        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
        if (prettyParameter != null) {
            request.addParameter("pretty", prettyParameter);
        }
        if (randomBoolean()) {
            // We default to JSON but we force it randomly for extra coverage
            request.addParameter("format", "json");
        }
        if (randomBoolean()) {
            // JSON is the default but randomly set it sometime for extra coverage
            RequestOptions.Builder options = request.getOptions().toBuilder();
            options.addHeader("Accept", randomFrom("*/*", "application/json"));
            request.setOptions(options);
        }
        request.setEntity(
            new StringEntity(
                query("SELECT * FROM test").mode(Mode.PLAIN).columnar(columnarValue(columnar)).toString(),
                ContentType.APPLICATION_JSON
            )
        );

        Response response = client().performRequest(request);
        try (InputStream content = response.getEntity().getContent()) {
            String actualJson = new BytesArray(content.readAllBytes()).utf8ToString();
            assertEquals(expectedJson, actualJson);
        }
    }

    public void testBasicTranslateQuery() throws IOException {
        index("{\"test\":\"test\"}", "{\"test\":\"test\"}");

        Map<String, Object> response = runTranslateSql(query("SELECT * FROM test").toString());
        assertEquals(1000, response.get("size"));
        assertFalse((Boolean) response.get("_source"));
        @SuppressWarnings("unchecked")
        List<Map<String, Object>> source = (List<Map<String, Object>>) response.get("fields");
        assertEquals(singletonList(singletonMap("field", "test")), source);
    }

    public void testBasicQueryWithFilter() throws IOException {
        String mode = randomMode();
        index("{\"test\":\"foo\"}", "{\"test\":\"bar\"}");

        Map<String, Object> expected = new HashMap<>();
        expected.put("columns", singletonList(columnInfo(mode, "test", "text", JDBCType.VARCHAR, Integer.MAX_VALUE)));
        expected.put("rows", singletonList(singletonList("foo")));
        assertResponse(
            expected,
            runSql(
                new StringEntity(
                    query("SELECT * FROM test").mode(mode).filter("{\"match\": {\"test\": \"foo\"}}").toString(),
                    ContentType.APPLICATION_JSON
                ),
                StringUtils.EMPTY,
                mode
            )
        );
    }

    public void testBasicQueryWithParameters() throws IOException {
        String mode = randomMode();
        boolean columnar = randomBoolean();
        index("{\"test\":\"foo\"}", "{\"test\":\"bar\"}");

        Map<String, Object> expected = new HashMap<>();
        expected.put(
            "columns",
            Arrays.asList(
                columnInfo(mode, "test", "text", JDBCType.VARCHAR, Integer.MAX_VALUE),
                columnInfo(mode, "param", "integer", JDBCType.INTEGER, 11)
            )
        );
        if (columnar) {
            expected.put("values", Arrays.asList(singletonList("foo"), singletonList(10)));
        } else {
            expected.put("rows", Arrays.asList(Arrays.asList("foo", 10)));
        }

        String params = mode.equals("jdbc")
            ? "{\"type\": \"integer\", \"value\": 10}, {\"type\": \"keyword\", \"value\": \"foo\"}"
            : "10, \"foo\"";
        assertResponse(
            expected,
            runSql(
                new StringEntity(
                    query("SELECT test, ? param FROM test WHERE test = ?").mode(mode)
                        .columnar(columnarValue(columnar))
                        .params("[" + params + "]")
                        .toString(),
                    ContentType.APPLICATION_JSON
                ),
                StringUtils.EMPTY,
                mode
            )
        );
    }

    public void testBasicTranslateQueryWithFilter() throws IOException {
        index("{\"test\":\"foo\"}", "{\"test\":\"bar\"}");

        Map<String, Object> response = runTranslateSql(query("SELECT * FROM test").filter("{\"match\": {\"test\": \"foo\"}}").toString());

        assertEquals(response.get("size"), 1000);
        assertFalse((Boolean) response.get("_source"));
        @SuppressWarnings("unchecked")
        List<Map<String, Object>> source = (List<Map<String, Object>>) response.get("fields");
        assertEquals(singletonList(singletonMap("field", "test")), source);

        @SuppressWarnings("unchecked")
        Map<String, Object> query = (Map<String, Object>) response.get("query");
        assertNotNull(query);

        @SuppressWarnings("unchecked")
        Map<String, Object> bool = (Map<String, Object>) query.get("bool");
        assertNotNull(bool);

        @SuppressWarnings("unchecked")
        List<Object> filter = (List<Object>) bool.get("filter");
        assertNotNull(filter);

        @SuppressWarnings("unchecked")
        Map<String, Object> map = (Map<String, Object>) filter.get(0);
        assertNotNull(map);

        @SuppressWarnings("unchecked")
        Map<String, Object> matchQ = (Map<String, Object>) map.get("match");

        @SuppressWarnings("unchecked")
        Map<String, Object> matchQuery = (Map<String, Object>) matchQ.get("test");

        assertNotNull(matchQuery);
        assertEquals("foo", matchQuery.get("query"));
    }

    public void testTranslateQueryWithGroupByAndHaving() throws IOException {
        index("{\"salary\":100}", "{\"age\":20}");

        Map<String, Object> response = runTranslateSql(
            query("SELECT avg(salary) FROM test GROUP BY abs(age) HAVING avg(salary) > 50 LIMIT 10").toString()
        );

        assertEquals(response.get("size"), 0);
        assertEquals(false, response.get("_source"));
        assertNull(response.get("stored_fields"));

        @SuppressWarnings("unchecked")
        Map<String, Object> aggregations = (Map<String, Object>) response.get("aggregations");
        assertEquals(1, aggregations.size());
        assertNotNull(aggregations);

        @SuppressWarnings("unchecked")
        Map<String, Object> groupby = (Map<String, Object>) aggregations.get("groupby");
        assertEquals(2, groupby.size());

        @SuppressWarnings("unchecked")
        Map<String, Object> composite = (Map<String, Object>) groupby.get("composite");
        assertEquals(2, composite.size());
        assertEquals(10, composite.get("size"));

        @SuppressWarnings("unchecked")
        List<Object> sources = (List<Object>) composite.get("sources");
        assertEquals(1, sources.size());

        @SuppressWarnings("unchecked")
        Map<String, Object> sourcesListMap = (Map<String, Object>) ((Map<String, Object>) sources.get(0)).values().iterator().next();
        assertEquals(1, sourcesListMap.size());

        @SuppressWarnings("unchecked")
        Map<String, Object> terms = (Map<String, Object>) sourcesListMap.get("terms");
        assertEquals(4, terms.size());
        assertEquals("long", terms.get("value_type"));
        assertEquals(true, terms.get("missing_bucket"));
        assertEquals("asc", terms.get("order"));

        @SuppressWarnings("unchecked")
        Map<String, Object> termsScript = (Map<String, Object>) terms.get("script");
        assertEquals(3, termsScript.size());
        assertEquals("InternalSqlScriptUtils.abs(InternalQlScriptUtils.docValue(doc,params.v0))", termsScript.get("source"));
        assertEquals("painless", termsScript.get("lang"));

        @SuppressWarnings("unchecked")
        Map<String, Object> termsScriptParams = (Map<String, Object>) termsScript.get("params");
        assertEquals(1, termsScriptParams.size());
        assertEquals("age", termsScriptParams.get("v0"));

        @SuppressWarnings("unchecked")
        Map<String, Object> aggregations2 = (Map<String, Object>) groupby.get("aggregations");
        assertEquals(2, aggregations2.size());

        List<String> aggKeys = new ArrayList<>(2);
        String aggFilterKey = null;
        for (Map.Entry<String, Object> entry : aggregations2.entrySet()) {
            String key = entry.getKey();
            if (key.startsWith("having")) {
                aggFilterKey = key;
            } else {
                aggKeys.add(key);
                @SuppressWarnings("unchecked")
                Map<String, Object> aggr = (Map<String, Object>) entry.getValue();
                assertEquals(1, aggr.size());
                @SuppressWarnings("unchecked")
                Map<String, Object> avg = (Map<String, Object>) aggr.get("avg");
                assertEquals(1, avg.size());
                assertEquals("salary", avg.get("field"));
            }
        }
        Collections.sort(aggKeys);
        assertEquals("having." + aggKeys.get(0), aggFilterKey);

        @SuppressWarnings("unchecked")
        Map<String, Object> having = (Map<String, Object>) aggregations2.get(aggFilterKey);
        assertEquals(1, having.size());

        @SuppressWarnings("unchecked")
        Map<String, Object> bucketSelector = (Map<String, Object>) having.get("bucket_selector");
        assertEquals(3, bucketSelector.size());
        assertEquals("skip", bucketSelector.get("gap_policy"));

        @SuppressWarnings("unchecked")
        Map<String, Object> bucketsPath = (Map<String, Object>) bucketSelector.get("buckets_path");
        assertEquals(1, bucketsPath.size());
        assertEquals(aggKeys.get(0).toString(), bucketsPath.get("a0"));

        @SuppressWarnings("unchecked")
        Map<String, Object> filterScript = (Map<String, Object>) bucketSelector.get("script");
        assertEquals(3, filterScript.size());
        assertEquals("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(params.a0,params.v0))", filterScript.get("source"));
        assertEquals("painless", filterScript.get("lang"));
        @SuppressWarnings("unchecked")
        Map<String, Object> filterScriptParams = (Map<String, Object>) filterScript.get("params");
        assertEquals(1, filterScriptParams.size());
        assertEquals(50, filterScriptParams.get("v0"));
    }

    public void testBasicQueryText() throws IOException {
        index("{\"test\":\"test\"}", "{\"test\":\"test\"}");

        String expected = "     test      \n" + "---------------\n" + "test           \n" + "test           \n";
        Tuple<String, String> response = runSqlAsText("SELECT * FROM test", "text/plain");
        assertEquals(expected, response.v1());
    }

    public void testNextPageText() throws IOException {
        executeQueryWithNextPage("text/plain", "     text      |    number     |      sum      \n", "%-15s|%-15d|%-15d\n");
    }

    // CSV/TSV tests

    private static String toJson(String value) {
        return "\"" + new String(JsonStringEncoder.getInstance().quoteAsString(value)) + "\"";
    }

    public void testDefaultQueryInCSV() throws IOException {
        index(
            "{\"name\":" + toJson("first") + ", \"number\" : 1 }",
            "{\"name\":" + toJson("second\t") + ", \"number\": 2 }",
            "{\"name\":" + toJson("\"third,\"") + ", \"number\": 3 }"
        );

        String expected = "name,number\r\n" + "first,1\r\n" + "second\t,2\r\n" + "\"\"\"third,\"\"\",3\r\n";

        String query = "SELECT * FROM test ORDER BY number";
        Tuple<String, String> response = runSqlAsText(query, "text/csv");
        assertEquals(expected, response.v1());

        response = runSqlAsTextFormat(query, "csv");
        assertEquals(expected, response.v1());
    }

    public void testQueryWithoutHeaderInCSV() throws IOException {
        index(
            "{\"name\":" + toJson("first") + ", \"number\" : 1 }",
            "{\"name\":" + toJson("second\t") + ", \"number\": 2 }",
            "{\"name\":" + toJson("\"third,\"") + ", \"number\": 3 }"
        );

        String expected = "first,1\r\n" + "second\t,2\r\n" + "\"\"\"third,\"\"\",3\r\n";

        String query = "SELECT * FROM test ORDER BY number";
        Tuple<String, String> response = runSqlAsText(query, "text/csv; header=absent");
        assertEquals(expected, response.v1());
    }

    public void testNextPageCSV() throws IOException {
        executeQueryWithNextPage("text/csv; header=present", "text,number,sum\r\n", "%s,%d,%d\r\n");
    }

    public void testCSVWithDelimiterParameter() throws IOException {
        String format = randomFrom("txt", "tsv", "json", "yaml", "smile", "cbor");
        String query = "SELECT * FROM test";
        index("{\"foo\":1}");

        Request badRequest = new Request("POST", SQL_QUERY_REST_ENDPOINT);
        badRequest.addParameter("format", format);
        badRequest.addParameter("delimiter", ";");
        badRequest.setEntity(
            new StringEntity(
                query(query).mode(randomValueOtherThan(Mode.JDBC.toString(), BaseRestSqlTestCase::randomMode)).toString(),
                ContentType.APPLICATION_JSON
            )
        );
        expectBadRequest(() -> {
            client().performRequest(badRequest);
            return emptyMap();
        }, containsString("request [/_sql] contains unrecognized parameter: [delimiter]"));

        Request csvRequest = new Request("POST", SQL_QUERY_REST_ENDPOINT + "?format=csv&delimiter=%3B");
        csvRequest.setEntity(
            new StringEntity(
                query(query).mode(randomValueOtherThan(Mode.JDBC.toString(), BaseRestSqlTestCase::randomMode)).toString(),
                ContentType.APPLICATION_JSON
            )
        );
        assertOK(client().performRequest(csvRequest));
    }

    public void testQueryInTSV() throws IOException {
        index(
            "{\"name\":" + toJson("first") + ", \"number\" : 1 }",
            "{\"name\":" + toJson("second\t") + ", \"number\": 2 }",
            "{\"name\":" + toJson("\"third,\"") + ", \"number\": 3 }"
        );

        String expected = "name\tnumber\n" + "first\t1\n" + "second\\t\t2\n" + "\"third,\"\t3\n";

        String query = "SELECT * FROM test ORDER BY number";
        Tuple<String, String> response = runSqlAsText(query, "text/tab-separated-values");
        assertEquals(expected, response.v1());
        response = runSqlAsTextFormat(query, "tsv");
        assertEquals(expected, response.v1());
    }

    public void testNextPageTSV() throws IOException {
        executeQueryWithNextPage("text/tab-separated-values", "text\tnumber\tsum\n", "%s\t%d\t%d\n");
    }

    public void testBinaryFieldFiltering() throws IOException {
        XContentBuilder createIndex = JsonXContent.contentBuilder().startObject();
        createIndex.startObject("mappings");
        {
            createIndex.startObject("properties");
            {
                createIndex.startObject("id").field("type", "long").endObject();
                createIndex.startObject("binary").field("type", "binary").field("doc_values", true).endObject();
            }
            createIndex.endObject();
        }
        createIndex.endObject().endObject();

        Request request = new Request("PUT", "/test_binary");
        request.setJsonEntity(Strings.toString(createIndex));
        assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode());

        long nonNullId = randomLong();
        long nullId = randomLong();
        StringBuilder bulk = new StringBuilder();
        bulk.append("{").append(toJson("index")).append(":{}}\n");
        bulk.append("{");
        {
            bulk.append(toJson("id")).append(":").append(nonNullId);
            bulk.append(",");
            bulk.append(toJson("binary")).append(":").append(toJson("U29tZSBiaW5hcnkgYmxvYg=="));
        }
        bulk.append("}\n");
        bulk.append("{").append(toJson("index")).append(":{}}\n");
        bulk.append("{");
        {
            bulk.append(toJson("id")).append(":").append(nullId);
        }
        bulk.append("}\n");

        request = new Request("PUT", "/test_binary/_bulk?refresh=true");
        request.setJsonEntity(bulk.toString());
        assertEquals(200, client().performRequest(request).getStatusLine().getStatusCode());

        String mode = randomMode();
        Map<String, Object> expected = new HashMap<>();
        expected.put("columns", singletonList(columnInfo(mode, "id", "long", JDBCType.BIGINT, 20)));
        expected.put("rows", singletonList(singletonList(nonNullId)));
        assertResponse(expected, runSql(mode, "SELECT id FROM test_binary WHERE binary IS NOT NULL", false));

        expected.put("rows", singletonList(singletonList(nullId)));
        assertResponse(expected, runSql(mode, "SELECT id FROM test_binary WHERE binary IS NULL", false));
    }

    private void executeQueryWithNextPage(String format, String expectedHeader, String expectedLineFormat) throws IOException {
        int size = 20;
        String[] docs = new String[size];
        for (int i = 0; i < size; i++) {
            docs[i] = "{\"text\":\"text" + i + "\", \"number\":" + i + "}\n";
        }
        index(docs);

        String request = query("SELECT text, number, number + 5 AS sum FROM test ORDER BY number").fetchSize(2).toString();

        String cursor = null;
        for (int i = 0; i < 20; i += 2) {
            Tuple<String, String> response;
            if (i == 0) {
                response = runSqlAsText(StringUtils.EMPTY, new StringEntity(request, ContentType.APPLICATION_JSON), format);
            } else {
                response = runSqlAsText(
                    StringUtils.EMPTY,
                    new StringEntity(cursor(cursor).toString(), ContentType.APPLICATION_JSON),
                    format
                );
            }

            StringBuilder expected = new StringBuilder();
            if (i == 0) {
                expected.append(expectedHeader);
                if (format == "text/plain") {
                    expected.append("---------------+---------------+---------------\n");
                }
            }
            expected.append(String.format(Locale.ROOT, expectedLineFormat, "text" + i, i, i + 5));
            expected.append(String.format(Locale.ROOT, expectedLineFormat, "text" + (i + 1), i + 1, i + 6));
            cursor = response.v2();
            assertEquals(expected.toString(), response.v1());
            assertNotNull(cursor);
        }
        Map<String, Object> expected = new HashMap<>();
        expected.put("rows", emptyList());
        assertResponse(
            expected,
            runSql(new StringEntity(cursor(cursor).toString(), ContentType.APPLICATION_JSON), StringUtils.EMPTY, Mode.PLAIN.toString())
        );

        Map<String, Object> response = runSql(
            new StringEntity(cursor(cursor).toString(), ContentType.APPLICATION_JSON),
            "/close",
            Mode.PLAIN.toString()
        );
        assertEquals(true, response.get("succeeded"));

        assertEquals(0, getNumberOfSearchContexts(client(), "test"));
    }

    private Tuple<String, String> runSqlAsText(String sql, String accept) throws IOException {
        return runSqlAsText(StringUtils.EMPTY, new StringEntity(query(sql).toString(), ContentType.APPLICATION_JSON), accept);
    }

    /**
     * Run SQL as text using the {@code Accept} header to specify the format
     * rather than the {@code format} parameter.
     */
    private Tuple<String, String> runSqlAsText(String suffix, HttpEntity entity, String accept) throws IOException {
        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT + suffix);
        request.addParameter("error_trace", "true");
        request.setEntity(entity);
        RequestOptions.Builder options = request.getOptions().toBuilder();
        options.addHeader("Accept", accept);
        request.setOptions(options);
        Response response = client().performRequest(request);
        return new Tuple<>(
            Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8)),
            response.getHeader("Cursor")
        );
    }

    /**
     * Run SQL as text using the {@code format} parameter to specify the format
     * rather than an {@code Accept} header.
     */
    private Tuple<String, String> runSqlAsTextFormat(String sql, String format) throws IOException {
        Request request = new Request("POST", SQL_QUERY_REST_ENDPOINT);
        request.addParameter("error_trace", "true");
        request.addParameter("format", format);
        request.setJsonEntity(query(sql).toString());

        Response response = client().performRequest(request);
        return new Tuple<>(
            Streams.copyToString(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8)),
            response.getHeader("Cursor")
        );
    }

    public static void assertResponse(Map<String, Object> expected, Map<String, Object> actual) {
        if (false == expected.equals(actual)) {
            NotEqualMessageBuilder message = new NotEqualMessageBuilder();
            message.compareMaps(actual, expected);
            fail("Response does not match:\n" + message.toString());
        }
    }
}


text:
layout: post
title: "Python versus Shell scripting : from experience"
date: 2015-11-08 14:24:13 +0100
permalink: articles/python-versus-shell-scripting-experience
tags:
  - python
  - scripting
  - shell
maturity: stable
---
![Python vs Shell](/assets/blog/Selection_008.png)

A quick "pros & cons" to choose between Python or Shell scripting, from what I've observed through the years.

This could probably apply to other high level script languages* vs shell.

\* *Like Ruby, but not like Perl, as it shares more with shells from my point of view, like unreadable syntax and execution speed.*

## Choose Python for :

- **its simple syntax** : can be read even by non Python-speaking people. End users may not be scared to change scripts to fit their need.
- **portability** : Python is actually more portable than shell because it's easy to install on any OS. Shell is theoretically portable but it's complicated to explain end users how to install the runtime (e.g. see cygwin). A Python script can be packaged into an executable program. There may be ways to bundle a script with the shell runtime.

## Choose Shell for :

- **speed** : except in case of bad code logic, shell scripts are usually a lot faster than Python's equivalents.
- **its ease to deal with external programs and system** : shell's syntax is already made to test/pipe/chain easily existing programs, whereas Python requires some extra boilerplate. Shell also has built-in functions to deal directly with files and the system, whereas Python uses an object-oriented model, which may not be as handy.

redpajama_set_name: RedPajamaGithub


text:
namespace apps {

IntentHandlingMetrics::IntentHandlingMetrics() {}

void IntentHandlingMetrics::RecordIntentPickerMetrics(
    Source source,
    bool should_persist,
    AppsNavigationThrottle::PickerAction action,
    AppsNavigationThrottle::Platform platform) {
  // TODO(crbug.com/985233) For now External Protocol Dialog is only querying
  // ARC apps.
  if (source == Source::kExternalProtocol) {
    UMA_HISTOGRAM_ENUMERATION("ChromeOS.Apps.ExternalProtocolDialog", action);
  } else {
    UMA_HISTOGRAM_ENUMERATION("ChromeOS.Apps.IntentPickerAction", action);

    UMA_HISTOGRAM_ENUMERATION("ChromeOS.Apps.IntentPickerDestinationPlatform",
                              platform);
  }
}

void IntentHandlingMetrics::RecordIntentPickerUserInteractionMetrics(
    const std::string& selected_app_package,
    PickerEntryType entry_type,
    IntentPickerCloseReason close_reason,
    Source source,
    bool should_persist) {
  if (entry_type == PickerEntryType::kArc &&
      (close_reason == IntentPickerCloseReason::PREFERRED_APP_FOUND ||
       close_reason == IntentPickerCloseReason::OPEN_APP)) {
    UMA_HISTOGRAM_ENUMERATION("Arc.UserInteraction",
                              arc::UserInteractionType::APP_STARTED_FROM_LINK);
  }
  AppsNavigationThrottle::PickerAction action =
      AppsNavigationThrottle::GetPickerAction(entry_type, close_reason,
                                              should_persist);
  AppsNavigationThrottle::Platform platform =
      AppsNavigationThrottle::GetDestinationPlatform(selected_app_package,
                                                     action);
  RecordIntentPickerMetrics(source, should_persist, action, platform);
}

void IntentHandlingMetrics::RecordExternalProtocolMetrics(
    arc::Scheme scheme,
    PickerEntryType entry_type,
    bool accepted,
    bool persisted) {
  arc::ProtocolAction action =
      arc::GetProtocolAction(scheme, entry_type, accepted, persisted);
  if (accepted) {
    UMA_HISTOGRAM_ENUMERATION("ChromeOS.Apps.ExternalProtocolDialog.Accepted",
                              action);
  } else {
    UMA_HISTOGRAM_ENUMERATION("ChromeOS.Apps.ExternalProtocolDialog.Rejected",
                              action);
  }
}

void IntentHandlingMetrics::RecordOpenBrowserMetrics(AppType type) {
  UMA_HISTOGRAM_ENUMERATION("ChromeOS.Apps.OpenBrowser", type);
}

}  // namespace apps


text:
package org.apache.hadoop.hdfs;

import junit.framework.TestCase;
import java.io.*;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.TreeMap;
import java.util.zip.CRC32;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
import org.apache.hadoop.util.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * This tests data transfer protocol handling in the Datanode. It sends
 * various forms of wrong data and verifies that Datanode handles it well.
 *
 * This test uses the following two file from src/test/.../dfs directory :
 *   1) hadoop-version-dfs-dir.tgz : contains DFS directories.
 *   2) hadoop-dfs-dir.txt : checksums that are compared in this test.
 * Please read hadoop-dfs-dir.txt for more information.
 */
public class TestDFSUpgradeFromImage extends TestCase {

  private static final Log LOG = LogFactory.getLog(
                    "org.apache.hadoop.hdfs.TestDFSUpgradeFromImage");
  private static File TEST_ROOT_DIR =
                      new File(MiniDFSCluster.getBaseDirectory());

  public int numDataNodes = 4;

  private static class ReferenceFileInfo {
    String path;
    long checksum;
  }

  LinkedList<ReferenceFileInfo> refList = new LinkedList<ReferenceFileInfo>();
  Iterator<ReferenceFileInfo> refIter;

  boolean printChecksum = false;

  protected void setUp() throws IOException {
    unpackStorage();
  }

  public void unpackStorage() throws IOException {
    String tarFile = System.getProperty("test.cache.data", "build/test/cache") +
                     "/hadoop-14-dfs-dir.tgz";
    String dataDir = System.getProperty("test.build.data", "build/test/data");
    File dfsDir = new File(dataDir, "dfs");
    if ( dfsDir.exists() && !FileUtil.fullyDelete(dfsDir) ) {
      throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
    }
    FileUtil.unTar(new File(tarFile), new File(dataDir));
    //Now read the reference info

    BufferedReader reader = new BufferedReader(
                        new FileReader(System.getProperty("test.cache.data", "build/test/cache") +
                                       "/hadoop-dfs-dir.txt"));
    String line;
    while ( (line = reader.readLine()) != null ) {

      line = line.trim();
      if (line.length() <= 0 || line.startsWith("#")) {
        continue;
      }
      String[] arr = line.split("\\s+\t\\s+");
      if (arr.length < 1) {
        continue;
      }
      if (arr[0].equals("printChecksums")) {
        printChecksum = true;
        break;
      }
      if (arr.length < 2) {
        continue;
      }
      ReferenceFileInfo info = new ReferenceFileInfo();
      info.path = arr[0];
      info.checksum = Long.parseLong(arr[1]);
      refList.add(info);
    }
    reader.close();
  }

  private void verifyChecksum(String path, long checksum) throws IOException {
    if ( refIter == null ) {
      refIter = refList.iterator();
    }

    if ( printChecksum ) {
      LOG.info("CRC info for reference file : " + path + " \t " + checksum);
    } else {
      if ( !refIter.hasNext() ) {
        throw new IOException("Checking checksum for " + path +
                              "Not enough elements in the refList");
      }
      ReferenceFileInfo info = refIter.next();
      // The paths are expected to be listed in the same order
      // as they are traversed here.
      assertEquals(info.path, path);
      assertEquals("Checking checksum for " + path, info.checksum, checksum);
    }
  }

  CRC32 overallChecksum = new CRC32();

  private void verifyDir(DistributedFileSystem dfs, Path dir)
                                           throws IOException {

    FileStatus[] fileArr = dfs.listStatus(dir);
    TreeMap<Path, Boolean> fileMap = new TreeMap<Path, Boolean>();

    for(FileStatus file : fileArr) {
      fileMap.put(file.getPath(), Boolean.valueOf(file.isDirectory()));
    }

    for(Iterator<Path> it = fileMap.keySet().iterator(); it.hasNext();) {
      Path path = it.next();
      boolean isDir = fileMap.get(path);

      String pathName = path.toUri().getPath();
      overallChecksum.update(pathName.getBytes());

      if ( isDir ) {
        verifyDir(dfs, path);
      } else {
        // this is not a directory. Checksum the file data.
        CRC32 fileCRC = new CRC32();
        FSInputStream in = dfs.dfs.open(pathName);
        byte[] buf = new byte[4096];
        int nRead = 0;
        while ( (nRead = in.read(buf, 0, buf.length)) > 0 ) {
          fileCRC.update(buf, 0, nRead);
        }

        verifyChecksum(pathName, fileCRC.getValue());
      }
    }
  }

  private void verifyFileSystem(DistributedFileSystem dfs) throws IOException {

    verifyDir(dfs, new Path("/"));

    verifyChecksum("overallCRC", overallChecksum.getValue());

    if ( printChecksum ) {
      throw new IOException("Checksums are written to log as requested. " +
                            "Throwing this exception to force an error " +
                            "for this test.");
    }
  }

  public void testUpgradeFromImage() throws IOException {
    MiniDFSCluster cluster = null;
    try {
      Configuration conf = new HdfsConfiguration();
      if (System.getProperty("test.build.data") == null) { // to allow test to be run outside of Ant
        System.setProperty("test.build.data", "build/test/data");
      }
      conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); // block scanning off
      cluster = new MiniDFSCluster.Builder(conf)
                                  .numDataNodes(numDataNodes)
                                  .format(false)
                                  .startupOption(StartupOption.UPGRADE)
                                  .clusterId("testClusterId")
                                  .build();
      cluster.waitActive();
      DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem();
      DFSClient dfsClient = dfs.dfs;
      //Safemode will be off only after upgrade is complete. Wait for it.
      while ( dfsClient.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET) ) {
        LOG.info("Waiting for SafeMode to be OFF.");
        try {
          Thread.sleep(1000);
        } catch (InterruptedException ignored) {}
      }

      verifyFileSystem(dfs);
    } finally {
      if (cluster != null) { cluster.shutdown(); }
    }
  }

  /**
   * Test that sets up a fake image from Hadoop 0.3.0 and tries to start a
   * NN, verifying that the correct error message is thrown.
   */
  public void testFailOnPreUpgradeImage() throws IOException {
    Configuration conf = new HdfsConfiguration();

    File namenodeStorage = new File(TEST_ROOT_DIR, "nnimage-0.3.0");
    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeStorage.toString());

    // Set up a fake NN storage that looks like an ancient Hadoop dir circa 0.3.0
    FileUtil.fullyDelete(namenodeStorage);
    assertTrue("Make " + namenodeStorage, namenodeStorage.mkdirs());
    File imageDir = new File(namenodeStorage, "image");
    assertTrue("Make " + imageDir, imageDir.mkdirs());

    // Hex dump of a formatted image from Hadoop 0.3.0
    File imageFile = new File(imageDir, "fsimage");
    byte[] imageBytes = StringUtils.hexStringToByte(
      "fffffffee17c0d2700000000");
    FileOutputStream fos = new FileOutputStream(imageFile);
    try {
      fos.write(imageBytes);
    } finally {
      fos.close();
    }

    // Now try to start an NN from it

    try {
      new MiniDFSCluster.Builder(conf).numDataNodes(0)
        .format(false)
        .manageDataDfsDirs(false)
        .manageNameDfsDirs(false)
        .startupOption(StartupOption.REGULAR)
        .build();
      fail("Was able to start NN from 0.3.0 image");
    } catch (IOException ioe) {
      LOG.info("Got expected exception", ioe);
      assertTrue(ioe.toString().contains("Old layout version is 'too old'"));
    }
  }
}


text:
FILE(REMOVE_RECURSE
  "CMakeFiles/schur_complement_solver_test.dir/schur_complement_solver_test.cc.o"
  "../../bin/schur_complement_solver_test.pdb"
  "../../bin/schur_complement_solver_test"
)

# Per-language clean rules from dependency scanning.
FOREACH(lang CXX)
  INCLUDE(CMakeFiles/schur_complement_solver_test.dir/cmake_clean_${lang}.cmake OPTIONAL)
ENDFOREACH(lang)


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef INTEGRATION_H
#define INTEGRATION_H

#include <iostream>
#include <sstream>

#include "../include/data/constructs/KeyValue.h"
#include "../include/data/constructs/security/Authorizations.h"
#include "../include/scanner/constructs/Results.h"
#include "../include/scanner/impl/Scanner.h"
#include "../include/writer/impl/SinkImpl.h"
#include "../include/data/constructs/client/zookeeperinstance.h"
#include "../include/interconnect/Master.h"
#include "../include/interconnect/tableOps/TableOperations.h"
#include "../include/interconnect/securityOps/SecurityOperations.h"

#include "../include/data/constructs/rfile/RFile.h"
#include "../include/data/constructs/compressor/compressor.h"
#include "../include/data/constructs/compressor/zlibCompressor.h"

#include <assert.h>

#define BOOST_IOSTREAMS_NO_LIB 1

class IntegrationTest
{
protected:
  interconnect::MasterConnect master;
  public:
    IntegrationTest(interconnect::MasterConnect otherMaster) : master(otherMaster)
  {
  }

  virtual std::string testName() = 0;

  virtual int run(std::string table) = 0;
};

#endif


text:
{% extends "_internal/base.html" %}
{% block title %}{{ title }}{% endblock %}
{% block description %}{% endblock %}
{% block content %}

{{ _('Update of model: %s' % model) }}

{{ message }}

      {% call form(class_='form-horizontal',role='form') %}
        {% for f in form %}

            {{ form[f.name].label(class_='col-lg-2 control-label')|safe }}

            {{ form[f.name].render(class_='form-control')|safe }}

        {% endfor %}

          <button class="btn btn-lg btn-block btn-primary" type="submit">{{_('Save')}}</button>

      {% endcall %}

{% endblock %}


text:
from concurrent.futures.thread import ThreadPoolExecutor

import logging

log = logging.getLogger(__name__)

# This implementation is functionally identical to using the ThreadPoolExecutor directly,
# but it adds the callback, to log uncaught exceptions.
class ActionQueue:
    def __init__(self):
        self.thread_pool_executor = ThreadPoolExecutor()

    def submit(self, function, *args, **kwargs):
        future = self.thread_pool_executor.submit(function, *args, **kwargs)
        future.add_done_callback(self._on_future_done)
        return future

    def _on_future_done(self, future):
        exc = future.exception()
        if exc is not None:
            log.exception("Logging an uncaught exception (ActionQueue)", exc_info=exc)


text:
import {Component} from 'angular2/core';
import WebGLRenderer = THREE.WebGLRenderer;
import Vector3 = THREE.Vector3;
import Object3D = THREE.Object3D;
//vt-x add hello
//import {THREE} from 'three'
//import Quaternion = THREE.Quaternion;
//vt-x end
import Scene = THREE.Scene;
import PerspectiveCamera = THREE.PerspectiveCamera;
import Mesh = THREE.Mesh;
import VRControls = THREE.VRControls;
import VREffect = THREE.VREffect;
import {Base} from '../base/base';
import {CameraKeypressEvents} from '../camera-keypress-events/camera-keypress-events'
import {VRScene} from '../vrscene/vrscene';
import {VRSceneProvider} from '../vrscene/vrscene';
import {Injectable} from 'angular2/core';

@Component({
  selector: 'cube-scene',
  templateUrl: './app/cube-scene/cube-scene.html',
  //templateUrl: './app/cpp-scenes.html',

  styleUrls: ['app//cube-scene/cube-scene.css'],
  providers: [VRSceneProvider],
  //directives: [],
  directives: [CameraKeypressEvents],
  pipes: []
})

@Injectable()
export class CubeScene {
  private canvas: HTMLElement;
  private renderer: WebGLRenderer;
  private width: number;
  private height: number;
  private scene: Scene;
  private camera: PerspectiveCamera;
  private dolly: Object3D;
  //private controls: TrackballControls;
  private vrControls: VRControls;
  private vrEffect: VREffect;
  //private webVrManager: WebVRManager;
  private webVrManager;
  private sphere: Mesh;
  private cube: Mesh;
  private BaseRotation = new THREE.Quaternion();
  //private BaseRotation = new Quaternion();

  //private scene: Scene;
  //private base = new Base()
  private cubeQuat = new THREE.Quaternion().setFromAxisAngle( new THREE.Vector3(0,1,0), Base.ONE_DEG * 0.2 );
  // private vrScene: VRScene;

  constructor() {
  // constructor(private vrScene: VRScene) {
    // this.vrScene = vrScene;
  };

  initWebGl() {
    console.log('initWebGl: window.width=' + window.innerWidth)
    this.width = window.innerWidth;
    this.height = window.innerHeight;

    try {
      this.canvas = document.getElementById('scene-view');
      console.log('initWebGl: this.canvas=' + this.canvas)

      var glParms = new Object();

      glParms['antialias'] = true;
      glParms['canvas'] = this.canvas;

      this.renderer = new THREE.WebGLRenderer(glParms);
    }
    catch (e) {
      alert('This application needs WebGL enabled! error=' + e);
      return false;
    }

    // this controls the ambient background color e.g of the the "sky"
    this.renderer.setClearColor(0xf31313, 1.0);

    this.renderer.setSize(this.width, this.height);

    //this.renderer.r
    //this.container = document.getElementById('canvas-container');
    this.canvas.focus();
  }

  initScene() {
    this.scene = new THREE.Scene;

    this.camera = new THREE.PerspectiveCamera(75, this.width / this.height);
    this.camera.position.set(0, 1.5, 100);
    this.dolly = new THREE.Object3D();
    this.dolly.position.z = 50;
    this.scene.add(this.dolly);

    this.dolly.add(this.camera);

    this.vrControls = new THREE.VRControls(this.camera);

    this.vrEffect = new THREE.VREffect(this.renderer);
    this.vrEffect.setSize(this.width, this.height);
    this.webVrManager = new (<any>window).WebVRManager(this.renderer, this.vrEffect);
    console.log('vtClass.initScene: this.webVrManager=' + this.webVrManager);
    this.camera.quaternion.copy(this.BaseRotation);

    var geometry = new THREE.BoxGeometry(25, 25, 25);
    var meshParms = new Object();

    //meshParms['color'] = 0xffff00;
    meshParms['color'] = 0xff8000;

    //var material = new THREE.MeshNormalMaterial(meshParms);
    var material = new THREE.MeshBasicMaterial(meshParms);
    //material = new THREE.MeshBasicMaterial( {color: 0xffff00, side: THREE.DoubleSide} );
    this.cube = new THREE.Mesh(geometry, material);
    this.scene.add(this.cube);

    // draw!
    this.renderer.render(this.scene, this.camera);
    //window.requestAnimationFrame(this.mainLoop);
    //window.requestAnimationFrame(vtClass.prototype.mainLoop);

  }

  mainLoop() {
    window.requestAnimationFrame(CubeScene.prototype.mainLoop.bind(this));

    this.cube.quaternion.multiply(this.cubeQuat);

    this.vrControls.update();

    this.webVrManager.render(this.scene, this.camera);
  }

  canvasKeyHandler (event) {
    console.log('cube-scene.canvasKeyHandler: event.keyCode=' + event.keyCode);

    CameraKeypressEvents.keyHandler(event, this.dolly)
  }

}


text:
package ch.dissem.bitmessage.utils;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

public class StringsTest {
    @Test
    public void testHexString() {
        assertEquals("48656c6c6f21", Strings.hex("Hello!".getBytes()).toString());
        assertEquals("0001", Strings.hex(new byte[]{0, 1}).toString());
    }
}


text:
const neo4jDriver = require('neo4j-driver').v1;

const logger = require('./../../applogger');

const config = require('./../../config');
const graphConsts = require('./../common/graphConstants');

let getPublishIntent = function(domainObj) {
   let domainName=domainObj.domain;
   let intentName=domainObj.intent;
   logger.debug(domainName);
   let promise = new Promise(function(resolve, reject) {
       logger.debug(
           "Now proceeding to publish the concepts for domain name: ",
           domainName);
       let driver = neo4jDriver.driver(config.NEO4J.neo4jURL,
           neo4jDriver.auth.basic(config.NEO4J.usr, config.NEO4J.pwd), {
               encrypted: false
           }
       );

       let session = driver.session();

       logger.debug("obtained connection with neo4j");

       let query = 'merge (c:' + graphConsts.NODE_INTENT + '{name:{intentName}})'
       query += 'merge(d:' + graphConsts.NODE_DOMAIN + '{name:{domainName}})'
       query += 'merge(c)-[r:' + graphConsts.REL_INTENT_OF + ']->(d)'

       let params={
         intentName: intentName,
         domainName: domainName
       };

       session.run(query,params).then(function(result) {
               if (result) {
                   logger.debug(result);
               }
               session.close();
               resolve(intentName);
           })
           .catch(function(error) {
               logger.error("Error in NODE_CONCEPT query: ", error, ' query is: ', query);
               reject(error);
           });
   });
   return promise;
};

let getPublishIntentCallback = function(domainObj, callback) {
   logger.debug("from the callback : " +domainObj.domain);
   getPublishIntent(domainObj).then(function(intentDetails) {
       callback(null, intentDetails);
   }, function(err) {
       callback(err, null);
   });
};

module.exports = {
   getPublishIntentCallback: getPublishIntentCallback
};


text:
package core.geom.tools;

import core.Model;
import core.shape_interface.PadInterface;
import java.awt.Graphics2D;
import java.awt.event.MouseEvent;

/**
 *
 * @author Mithusayel Murmu
 */
public class EraserInterface extends PadInterface {

    private Eraser eraser;

    public EraserInterface(Model model) {
        this.model = model;
    }

    @Override
    public void clearShape() {
        eraser = null;
    }

    @Override
    public void paint(Graphics2D g2) {}

    @Override
    public void mouseDown(MouseEvent e) {
        if (Model.getMode() == Model.IDLE_MODE) {
            model.setMode(Model.DRAW_MODE);

            model.createNewElement(e.getPoint(), e.getPoint());
            eraser = (Eraser) model.getElement();
        }
    }

    @Override
    public void mouseDrag(MouseEvent e) {
        if(eraser == null)
            return;

        pad.paintTemp();

        if(Model.getMode() == Model.DRAW_MODE)
            eraser.modify(null, e.getPoint());

        pad.paintTemp();
    }

    @Override
    public void mouseUp(MouseEvent e) {
        if(Model.getMode() == Model.DRAW_MODE) {
            eraser.done();
            model.registerShape(eraser);
            model.setMode(Model.IDLE_MODE);
        }
        pad.repaint();
    }
}


text:
package org.elasticsearch.index;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.TieredMergePolicy;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;

/**
 * A shard in elasticsearch is a Lucene index, and a Lucene index is broken
 * down into segments. Segments are internal storage elements in the index
 * where the index data is stored, and are immutable up to delete markers.
 * Segments are, periodically, merged into larger segments to keep the
 * index size at bay and expunge deletes.
 *
 *

 * Merges select segments of approximately equal size, subject to an allowed
 * number of segments per tier. The merge policy is able to merge
 * non-adjacent segments, and separates how many segments are merged at once from how many
 * segments are allowed per tier. It also does not over-merge (i.e., cascade merges).
 *
 *

 * All merge policy settings are dynamic and can be updated on a live index.
 * The merge policy has the following settings:
 *
 *

 *
index.merge.policy.expunge_deletes_allowed:
 *
 *     When expungeDeletes is called, we only merge away a segment if its delete
 *     percentage is over this threshold. Default is 10.
 *
 *
index.merge.policy.floor_segment:
 *
 *     Segments smaller than this are "rounded up" to this size, i.e. treated as
 *     equal (floor) size for merge selection. This is to prevent frequent
 *     flushing of tiny segments, thus preventing a long tail in the index. Default
 *     is 2mb.
 *
 *
index.merge.policy.max_merge_at_once:
 *
 *     Maximum number of segments to be merged at a time during "normal" merging.
 *     Default is 10.
 *
 *
index.merge.policy.max_merge_at_once_explicit:
 *
 *     Maximum number of segments to be merged at a time, during force merge or
 *     expungeDeletes. Default is 30.
 *
 *
index.merge.policy.max_merged_segment:
 *
 *     Maximum sized segment to produce during normal merging (not explicit
 *     force merge). This setting is approximate: the estimate of the merged
 *     segment size is made by summing sizes of to-be-merged segments
 *     (compensating for percent deleted docs). Default is 5gb.
 *
 *
index.merge.policy.segments_per_tier:
 *
 *     Sets the allowed number of segments per tier. Smaller values mean more
 *     merging but fewer segments. Default is 10. Note, this value needs to be
 *     >= than the max_merge_at_once otherwise you'll force too many merges to
 *     occur.
 *
 *
index.merge.policy.reclaim_deletes_weight:
 *
 *     Controls how aggressively merges that reclaim more deletions are favored.
 *     Higher values favor selecting merges that reclaim deletions. A value of
 *     0.0 means deletions don't impact merge selection. Defaults to 2.0.
 *

 *
 *

 * For normal merging, the policy first computes a "budget" of how many
 * segments are allowed to be in the index. If the index is over-budget,
 * then the policy sorts segments by decreasing size (proportionally considering percent
 * deletes), and then finds the least-cost merge. Merge cost is measured by
 * a combination of the "skew" of the merge (size of largest seg divided by
 * smallest seg), total merge size and pct deletes reclaimed, so that
 * merges with lower skew, smaller size and those reclaiming more deletes,
 * are favored.
 *
 *

 * If a merge will produce a segment that's larger than
 * max_merged_segment then the policy will merge fewer segments (down to
 * 1 at once, if that one has deletions) to keep the segment size under
 * budget.
 *
 *

 * Note, this can mean that for large shards that holds many gigabytes of
 * data, the default of max_merged_segment (5gb) can cause for many
 * segments to be in an index, and causing searches to be slower. Use the
 * indices segments API to see the segments that an index has, and
 * possibly either increase the max_merged_segment or issue an optimize
 * call for the index (try and aim to issue it on a low traffic time).
 */

public final class MergePolicyConfig {
    private final TieredMergePolicy mergePolicy = new TieredMergePolicy();
    private final Logger logger;
    private final boolean mergesEnabled;

    public static final double          DEFAULT_EXPUNGE_DELETES_ALLOWED     = 10d;
    public static final ByteSizeValue   DEFAULT_FLOOR_SEGMENT               = new ByteSizeValue(2, ByteSizeUnit.MB);
    public static final int             DEFAULT_MAX_MERGE_AT_ONCE           = 10;
    public static final int             DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT  = 30;
    public static final ByteSizeValue   DEFAULT_MAX_MERGED_SEGMENT          = new ByteSizeValue(5, ByteSizeUnit.GB);
    public static final double          DEFAULT_SEGMENTS_PER_TIER           = 10.0d;
    public static final double          DEFAULT_RECLAIM_DELETES_WEIGHT      = 2.0d;
    public static final Setting<Double> INDEX_COMPOUND_FORMAT_SETTING       =
        new Setting<>("index.compound_format", Double.toString(TieredMergePolicy.DEFAULT_NO_CFS_RATIO), MergePolicyConfig::parseNoCFSRatio,
            Property.Dynamic, Property.IndexScope);

    public static final Setting<Double> INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING =
        Setting.doubleSetting("index.merge.policy.expunge_deletes_allowed", DEFAULT_EXPUNGE_DELETES_ALLOWED, 0.0d,
            Property.Dynamic, Property.IndexScope);
    public static final Setting<ByteSizeValue> INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING =
        Setting.byteSizeSetting("index.merge.policy.floor_segment", DEFAULT_FLOOR_SEGMENT,
            Property.Dynamic, Property.IndexScope);
    public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING =
        Setting.intSetting("index.merge.policy.max_merge_at_once", DEFAULT_MAX_MERGE_AT_ONCE, 2,
            Property.Dynamic, Property.IndexScope);
    public static final Setting<Integer> INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING =
        Setting.intSetting("index.merge.policy.max_merge_at_once_explicit", DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT, 2,
            Property.Dynamic, Property.IndexScope);
    public static final Setting<ByteSizeValue> INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING =
        Setting.byteSizeSetting("index.merge.policy.max_merged_segment", DEFAULT_MAX_MERGED_SEGMENT,
            Property.Dynamic, Property.IndexScope);
    public static final Setting<Double> INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING =
        Setting.doubleSetting("index.merge.policy.segments_per_tier", DEFAULT_SEGMENTS_PER_TIER, 2.0d,
            Property.Dynamic, Property.IndexScope);
    public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
        Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
            Property.Dynamic, Property.IndexScope);
    public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin

    MergePolicyConfig(Logger logger, IndexSettings indexSettings) {
        this.logger = logger;
        double forceMergeDeletesPctAllowed = indexSettings.getValue(INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING); // percentage
        ByteSizeValue floorSegment = indexSettings.getValue(INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING);
        int maxMergeAtOnce = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING);
        int maxMergeAtOnceExplicit = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING);
        // TODO is this really a good default number for max_merge_segment, what happens for large indices, won't they end up with many segments?
        ByteSizeValue maxMergedSegment = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING);
        double segmentsPerTier = indexSettings.getValue(INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING);
        double reclaimDeletesWeight = indexSettings.getValue(INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING);
        this.mergesEnabled = indexSettings.getSettings().getAsBoolean(INDEX_MERGE_ENABLED, true);
        if (mergesEnabled == false) {
            logger.warn("[{}] is set to false, this should only be used in tests and can cause serious problems in production environments", INDEX_MERGE_ENABLED);
        }
        maxMergeAtOnce = adjustMaxMergeAtOnceIfNeeded(maxMergeAtOnce, segmentsPerTier);
        mergePolicy.setNoCFSRatio(indexSettings.getValue(INDEX_COMPOUND_FORMAT_SETTING));
        mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
        mergePolicy.setFloorSegmentMB(floorSegment.getMbFrac());
        mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
        mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
        mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
        mergePolicy.setSegmentsPerTier(segmentsPerTier);
        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
        if (logger.isTraceEnabled()) {
            logger.trace("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]",
                forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight);
        }
    }

    void setReclaimDeletesWeight(Double reclaimDeletesWeight) {
        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
    }

    void setSegmentsPerTier(Double segmentsPerTier) {
        mergePolicy.setSegmentsPerTier(segmentsPerTier);
    }

    void setMaxMergedSegment(ByteSizeValue maxMergedSegment) {
        mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
    }

    void setMaxMergesAtOnceExplicit(Integer maxMergeAtOnceExplicit) {
        mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
    }

    void setMaxMergesAtOnce(Integer maxMergeAtOnce) {
        mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
    }

    void setFloorSegmentSetting(ByteSizeValue floorSegementSetting) {
        mergePolicy.setFloorSegmentMB(floorSegementSetting.getMbFrac());
    }

    void setExpungeDeletesAllowed(Double value) {
        mergePolicy.setForceMergeDeletesPctAllowed(value);
    }

    void setNoCFSRatio(Double noCFSRatio) {
        mergePolicy.setNoCFSRatio(noCFSRatio);
    }

    private int adjustMaxMergeAtOnceIfNeeded(int maxMergeAtOnce, double segmentsPerTier) {
        // fixing maxMergeAtOnce, see TieredMergePolicy#setMaxMergeAtOnce
        if (!(segmentsPerTier >= maxMergeAtOnce)) {
            int newMaxMergeAtOnce = (int) segmentsPerTier;
            // max merge at once should be at least 2
            if (newMaxMergeAtOnce <= 1) {
                newMaxMergeAtOnce = 2;
            }
            logger.debug("changing max_merge_at_once from [{}] to [{}] because segments_per_tier [{}] has to be higher or equal to it", maxMergeAtOnce, newMaxMergeAtOnce, segmentsPerTier);
            maxMergeAtOnce = newMaxMergeAtOnce;
        }
        return maxMergeAtOnce;
    }

    MergePolicy getMergePolicy() {
        return mergesEnabled ? mergePolicy : NoMergePolicy.INSTANCE;
    }

    private static double parseNoCFSRatio(String noCFSRatio) {
        noCFSRatio = noCFSRatio.trim();
        if (noCFSRatio.equalsIgnoreCase("true")) {
            return 1.0d;
        } else if (noCFSRatio.equalsIgnoreCase("false")) {
            return 0.0;
        } else {
            try {
                double value = Double.parseDouble(noCFSRatio);
                if (value < 0.0 || value > 1.0) {
                    throw new IllegalArgumentException("NoCFSRatio must be in the interval [0..1] but was: [" + value + "]");
                }
                return value;
            } catch (NumberFormatException ex) {
                throw new IllegalArgumentException("Expected a boolean or a value in the interval [0..1] but was: [" + noCFSRatio + "]", ex);
            }
        }
    }
}


text:
import React from 'react'

import './VideoMessage.css'

import { FaCloudDownloadAlt, FaExclamationTriangle } from 'react-icons/fa'

import classNames from 'classnames'
import ProgressCircle from '../Circle/Circle'
import { IProgressOptions, IVideoMessageProps } from '../type'

const VideoMessage: React.FC<IVideoMessageProps> = props => {
  var progressOptions = {
    strokeWidth: 2.3,
    color: '#efe',
    trailColor: '#aaa',
    trailWidth: 1,
    step: (
      state: IProgressOptions,
      circle: {
        path: { setAttribute: (arg0: string, arg1: string) => void }
        value: () => number
        setText: (arg0: string | number) => void
      }
    ) => {
      circle.path.setAttribute('trail', state.state.color)
      circle.path.setAttribute('trailwidth-width', state.state.width)

      var value = Math.round(circle?.value() * 100)
      if (value === 0) circle?.setText('')
      else circle?.setText(value)
    },
  }

  const error = props?.data?.status && props?.data?.status.error === true
  const downloaded = props?.data?.status && props?.data?.status.download

  return (

        {!downloaded && (

        )}

        {downloaded && (
          <video controls controlsList={props.controlsList}>
            <source src={props?.data.videoURL} type='video/mp4' />
            Your browser does not support HTML video.
          </video>
        )}

        {error && (

              <FaExclamationTriangle />

        )}
        {!error && props?.data?.status && !downloaded && (

            {!props.data.status.click && (
              <button onClick={props.onDownload} className='rce-mbox-video--video__block-item rce-mbox-video--download'>
                <FaCloudDownloadAlt />
              </button>
            )}
            {typeof props.data.status.loading === 'number' && props.data.status.loading !== 0 && (
              <ProgressCircle
                animate={props.data.status.loading}
                className='rce-mbox-video--video__block-item'
                progressOptions={progressOptions}
              />
            )}

        )}

      {props?.text &&
{props.text}
}

  )
}

export default VideoMessage


text:
{% extends "layout.html" %}

{% block page_title %}
  2FA - GOV.UK prototype
{% endblock %}

{% block service_name %}

      Bob Thomas Arctor (AEDM)

Home

Your profile

Sign out

{% endblock %}

{% block content %}

<link href="https://ci.test.mot-testing.service.gov.uk/assets/stylesheets/dvsa.css" media="screen" rel="stylesheet" />

        BETA
        This is a new service – your feedback will help us to improve it.

                Your home

                            AE information

                            Site information

                            User search

                            Replacement/duplicate certificate

                            MOT tests
                         (comparison of tests available)

                            Demo test requests

                        Special notices

                    0

                        unread

                    0

                        overdue

                        Notifications

                        0
                        unread

                        View all notifications

Training

Practice on any class of vehicle without using slots or recording the outcome.

                        Start training test

Resources on GOV.UK

                            Motorcycle inspection checklist for class 1 and 2 (VT29M)

                            Vehicle inspection checklist for class 3, 4, 5 and 7 (VT29)

                            In service exhaust emission standards for road vehicles

                            Special notices

MOT manuals and guide

                                Manual for class 1 and 2 vehicles

                                Manual for class 3, 4, 5, and 7 vehicles

                                MOT testing guide

{% endblock %}