text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package app.metatron.discovery.domain.dataprep.teddy;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;

import app.metatron.discovery.domain.dataprep.teddy.exceptions.TeddyException;
import java.io.IOException;
import org.junit.BeforeClass;
import org.junit.Test;

public class ReplaceExtractTest extends TeddyTest {

  @BeforeClass
  public static void setUp() throws Exception {
    loadGridCsv("sample", "teddy/sample.csv");
    loadGridCsv("contract", "teddy/contract.csv");
    loadGridCsv("crime", "teddy/crime.csv");
    loadGridCsv("sales_regex_test", "teddy/sales_regex_test.csv");

    // Original values of sales desc column:
    // "Nortel Networks T7316 \"E Nt8 B27\""
    // "SM TSP800 TSP847IIU Receipt Printer"
    // "SM \"TSP100 TSP143LAN Receipt\" Printer"
  }

  @Test
  public void test_replace() throws IOException, TeddyException {
    DataFrame sales = new DataFrame();
    sales.setByGrid(grids.get("sales_regex_test"));
    sales = prepare_sales_regex_test(sales);

    String ruleString = "replace col: `desc` on: 'Star Micronics' with: 'SM' global: false";
    DataFrame newDf = apply_rule(sales, ruleString);
    newDf.show();

    assertEquals("Nortel Networks T7316 \"E Nt8 B27\"", newDf.rows.get(0).get("desc"));
    assertEquals("SM TSP800 TSP847IIU Receipt Printer", newDf.rows.get(1).get("desc"));
    assertEquals("SM \"TSP100 TSP143LAN Receipt\" Printer", newDf.rows.get(2).get("desc"));
  }

  @Test
  public void test_replace_multi() throws IOException, TeddyException {
    DataFrame crime = new DataFrame();
    crime.setByGrid(grids.get("crime"));
    crime.show();

    String ruleString = "replace col: column3, column5, column7, column8, column9, column10, column11, column12,"
            + " column13 on: '_' with: '' global: true";

    DataFrame newDf = apply_rule(crime, ruleString);
    newDf.show();
  }

  @Test
  public void test_replace_regex() throws IOException, TeddyException {
    DataFrame contract = new DataFrame();
    contract.setByGrid(grids.get("contract"));
    contract = prepare_contract(contract);
    contract.show(100);

    String ruleString = "extract col: cdate on: /\\w+/ quote: '\"' limit: 3";

    DataFrame newDf = apply_rule(contract, ruleString);
    newDf.show();
  }

  @Test
  public void test_extract() throws IOException, TeddyException {
    DataFrame df = new DataFrame();
    df.setByGrid(grids.get("sample"));
    df = prepare_sample(df);
    df.show();

    String ruleString = "extract col: name on: 'e' limit: 3";

    DataFrame newDf = apply_rule(df, ruleString);
    newDf.show();

    assertEquals("e", newDf.rows.get(0).get("extract_name1"));
    assertNull(newDf.rows.get(0).get("extract_name2"));

    assertNull(newDf.rows.get(1).get("extract_name1"));
    assertNull(newDf.rows.get(1).get("extract_name2"));

    assertEquals("e", newDf.rows.get(2).get("extract_name1"));
    assertEquals("e", newDf.rows.get(2).get("extract_name2"));
    assertEquals("e", newDf.rows.get(2).get("extract_name3"));
  }

  @Test
  public void test_extract_regex() throws IOException, TeddyException {
    DataFrame contract = new DataFrame();
    contract.setByGrid(grids.get("contract"));
    contract = prepare_contract(contract);
    contract.show();

    String ruleString = "extract col: cdate on: /\\w+/ limit: 3";

    DataFrame newDf = apply_rule(contract, ruleString);
    newDf.show();

    assertEquals("2012", newDf.rows.get(0).get("extract_cdate1"));
  }

  @Test
  public void test_extract_extra() throws IOException, TeddyException {
    DataFrame sales = new DataFrame();
    sales.setByGrid(grids.get("sales_regex_test"));
    sales = prepare_sales_regex_test(sales);

    String ruleString = "extract col: desc on: /[a-zA-Z]+/ quote:'\"' limit: 8 global:true";
    DataFrame newDf = apply_rule(sales, ruleString);
    newDf.show();

    assertNull(newDf.rows.get(0).get("extract_desc4"));
    assertEquals("TSP", newDf.rows.get(1).get("extract_desc4"));
    assertNull(newDf.rows.get(2).get("extract_desc4"));
    assertEquals("TSP", newDf.rows.get(3).get("extract_desc4"));
  }

  @Test
  public void test_extract_quote_literal() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM TSP800 TSP847IIU Receipt Printer"},
            {"SM \"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"TSP100\" TSP143LAN Receipt Printer"},
            {}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // When the pattern is "TSP":
    // Nortel Networks T7316 "E Nt8 B27" -> None
    // "SM TSP800 TSP847IIU Receipt Printer" -> [TSP], [TSP], null
    // "SM \"TSP100 TSP143LAN Receipt\" Printer" -> null, null, null
    // "SM \"TSP100\" TSP143LAN Receipt Printer" -> [TSP], null, null

    df = apply_rule(df, "extract col: desc on: 'TSP' quote: '\"' limit: 3");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", null, null, null});
    assertRow(df.rows.get(1), new Object[]{"SM TSP800 TSP847IIU Receipt Printer", "TSP", "TSP", null});
    assertRow(df.rows.get(2), new Object[]{"SM \"TSP100 TSP143LAN Receipt\" Printer", null, null, null});
    assertRow(df.rows.get(3), new Object[]{"SM \"TSP100\" TSP143LAN Receipt Printer", "TSP", null, null});
    assertRow(df.rows.get(4), new Object[]{null, null, null, null});
  }

  @Test
  public void test_extract_unmatched_quote_literal() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM \"TSP800 TSP847IIU Receipt Printer"},
            {"SM \"\"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"\"TSP100\" TSP143LAN Receipt Printer"}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // When the pattern is "TSP":
    // Nortel Networks T7316 "E Nt8 B27" -> None
    // "SM \"TSP800 TSP847IIU Receipt Printer" -> null, null, null
    // "SM \"\"TSP100 TSP143LAN Receipt\" Printer" -> [TSP], [TSP], null
    // "SM \"\"TSP100\" TSP143LAN Receipt Printer" -> [TSP], null, null

    df = apply_rule(df, "extract col: desc on: 'TSP' quote: '\"' limit: 3");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", null, null, null});
    assertRow(df.rows.get(1), new Object[]{"SM \"TSP800 TSP847IIU Receipt Printer", null, null, null});
    assertRow(df.rows.get(2), new Object[]{"SM \"\"TSP100 TSP143LAN Receipt\" Printer", "TSP", "TSP", null});
    assertRow(df.rows.get(3), new Object[]{"SM \"\"TSP100\" TSP143LAN Receipt Printer", "TSP", null, null});
  }

  @Test
  public void test_extract_quote_regex() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM TSP800 TSP847IIU Receipt Printer"},
            {"SM \"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"TSP100\" TSP143LAN Receipt Printer"},
            {}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // When the pattern is "/[\s\W\\"]+[^\s\d\W]+\d+/" (Words start with alphabets, end numeric chars after):
    // Nortel Networks T7316 "E Nt8 B27" -> [T7316], null, null
    // "SM TSP800 TSP847IIU Receipt Printer" -> [TSP800], [TSP847IIU], null
    // "SM \"TSP100 TSP143LAN Receipt\" Printer" -> null, null, null
    // "SM \"TSP100\" TSP143LAN Receipt Printer" -> [TSP143LAN], null, null

    df = apply_rule(df, "extract col: desc on: /[^\\s\\W]\\w+\\d+\\w*/ quote: '\"' limit: 3");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", "T7316", null, null});
    assertRow(df.rows.get(1), new Object[]{"SM TSP800 TSP847IIU Receipt Printer", "TSP800", "TSP847IIU", null});
    assertRow(df.rows.get(2), new Object[]{"SM \"TSP100 TSP143LAN Receipt\" Printer", null, null, null});
    assertRow(df.rows.get(3), new Object[]{"SM \"TSP100\" TSP143LAN Receipt Printer", "TSP143LAN", null, null});
  }

  @Test
  public void test_extract_weird_quote_regex() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM \"TSP800 TSP847IIU Receipt Printer"},
            {"SM \"\"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"\"TSP100\" TSP143LAN Receipt Printer"},
            {}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // Nortel Networks T7316 "E Nt8 B27" -> [T7316], null, null
    // "SM \"TSP800 TSP847IIU Receipt Printer" -> null, null, null
    // "SM \"\"TSP100 TSP143LAN Receipt\" Printer" -> [TSP100], [TSP143LAN], null
    // "SM \"\"TSP100\" TSP143LAN Receipt Printer" -> [TSP143LAN], null, null

    df = apply_rule(df, "extract col: desc on: /[^\\s\\W]\\w+\\d+\\w*/ quote: '\"' limit: 3");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", "T7316", null, null});
    assertRow(df.rows.get(1), new Object[]{"SM \"TSP800 TSP847IIU Receipt Printer", null, null, null});
    assertRow(df.rows.get(2), new Object[]{"SM \"\"TSP100 TSP143LAN Receipt\" Printer", "TSP100", "TSP143LAN", null});
    assertRow(df.rows.get(3), new Object[]{"SM \"\"TSP100\" TSP143LAN Receipt Printer", "TSP100", null, null});
  }

  // Countpattern test cases below are almost the same to extract test cases above.

  @Test
  public void test_countpattern() throws IOException, TeddyException {
    DataFrame df = new DataFrame();
    df.setByGrid(grids.get("sample"));
    df = prepare_sample(df);
    df.show();

    String ruleString = "countpattern col: name on: 'e'";

    DataFrame newDf = apply_rule(df, ruleString);
    newDf.show();

    assertEquals(1L, newDf.rows.get(0).get("countpattern_name"));
    assertEquals(0L, newDf.rows.get(1).get("countpattern_name"));
    assertEquals(3L, newDf.rows.get(2).get("countpattern_name"));
    assertEquals(0L, newDf.rows.get(3).get("countpattern_name"));
    assertEquals(0L, newDf.rows.get(4).get("countpattern_name"));
  }

  @Test
  public void test_countpattern_regex() throws IOException, TeddyException {
    DataFrame contract = new DataFrame();
    contract.setByGrid(grids.get("contract"));
    contract = prepare_contract(contract);
    contract.show();

    String ruleString = "countpattern col: cdate on: /\\w+/";

    DataFrame newDf = apply_rule(contract, ruleString);
    newDf.show();

    assertEquals(3L, newDf.rows.get(0).get("countpattern_cdate"));
    assertEquals(3L, newDf.rows.get(1).get("countpattern_cdate"));
    assertEquals(3L, newDf.rows.get(2).get("countpattern_cdate"));
    assertEquals(3L, newDf.rows.get(3).get("countpattern_cdate"));
    assertEquals(3L, newDf.rows.get(4).get("countpattern_cdate"));
  }

  @Test
  public void test_countpattern_quote_literal() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM TSP800 TSP847IIU Receipt Printer"},
            {"SM \"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"TSP100\" TSP143LAN Receipt Printer"},
            {}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // When the pattern is "TSP":
    // Nortel Networks T7316 "E Nt8 B27" -> None
    // "SM TSP800 TSP847IIU Receipt Printer" -> [TSP], [TSP], null
    // "SM \"TSP100 TSP143LAN Receipt\" Printer" -> null, null, null
    // "SM \"TSP100\" TSP143LAN Receipt Printer" -> [TSP], null, null
    // Null -> 0

    df = apply_rule(df, "countpattern col: desc on: 'TSP' quote: '\"'");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", 0L});
    assertRow(df.rows.get(1), new Object[]{"SM TSP800 TSP847IIU Receipt Printer", 2L});
    assertRow(df.rows.get(2), new Object[]{"SM \"TSP100 TSP143LAN Receipt\" Printer", 0L});
    assertRow(df.rows.get(3), new Object[]{"SM \"TSP100\" TSP143LAN Receipt Printer", 1L});
    assertRow(df.rows.get(4), new Object[]{null, 0L});
  }

  @Test
  public void test_countpattern_unmatched_quote_literal() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM \"TSP800 TSP847IIU Receipt Printer"},
            {"SM \"\"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"\"TSP100\" TSP143LAN Receipt Printer"}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // When the pattern is "TSP":
    // Nortel Networks T7316 "E Nt8 B27" -> None
    // "SM \"TSP800 TSP847IIU Receipt Printer" -> null, null, null
    // "SM \"\"TSP100 TSP143LAN Receipt\" Printer" -> [TSP], [TSP], null
    // "SM \"\"TSP100\" TSP143LAN Receipt Printer" -> [TSP], null, null

    df = apply_rule(df, "countpattern col: desc on: 'TSP' quote: '\"' limit: 3");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", 0L});
    assertRow(df.rows.get(1), new Object[]{"SM \"TSP800 TSP847IIU Receipt Printer", 0L});
    assertRow(df.rows.get(2), new Object[]{"SM \"\"TSP100 TSP143LAN Receipt\" Printer", 2L});
    assertRow(df.rows.get(3), new Object[]{"SM \"\"TSP100\" TSP143LAN Receipt Printer", 1L});
  }

  @Test
  public void test_countpattern_quote_regex() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM TSP800 TSP847IIU Receipt Printer"},
            {"SM \"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"TSP100\" TSP143LAN Receipt Printer"},
            {}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // When the pattern is "/[\s\W\\"]+[^\s\d\W]+\d+/" (Words start with alphabets, end numeric chars after):
    // Nortel Networks T7316 "E Nt8 B27" -> [T7316], null, null
    // "SM TSP800 TSP847IIU Receipt Printer" -> [TSP800], [TSP847IIU], null
    // "SM \"TSP100 TSP143LAN Receipt\" Printer" -> null, null, null
    // "SM \"TSP100\" TSP143LAN Receipt Printer" -> [TSP143LAN], null, null

    df = apply_rule(df, "countpattern col: desc on: /[^\\s\\W]\\w+\\d+\\w*/ quote: '\"'");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", 1L});
    assertRow(df.rows.get(1), new Object[]{"SM TSP800 TSP847IIU Receipt Printer", 2L});
    assertRow(df.rows.get(2), new Object[]{"SM \"TSP100 TSP143LAN Receipt\" Printer", 0L});
    assertRow(df.rows.get(3), new Object[]{"SM \"TSP100\" TSP143LAN Receipt Printer", 1L});
    assertRow(df.rows.get(4), new Object[]{null, 0L});
  }

  @Test
  public void test_countpattern_weird_quote_regex() throws IOException, TeddyException {
    String[][] strGrid = new String[][]{
            {"Nortel Networks T7316 \"E Nt8 B27\""},
            {"SM \"TSP800 TSP847IIU Receipt Printer"},
            {"SM \"\"TSP100 TSP143LAN Receipt\" Printer"},
            {"SM \"\"TSP100\" TSP143LAN Receipt Printer"},
            {}
    };
    DataFrame df = createByGrid(strGrid, new String[]{"desc"});

    // Nortel Networks T7316 "E Nt8 B27" -> [T7316], null, null
    // "SM \"TSP800 TSP847IIU Receipt Printer" -> null, null, null
    // "SM \"\"TSP100 TSP143LAN Receipt\" Printer" -> [TSP100], [TSP143LAN], null
    // "SM \"\"TSP100\" TSP143LAN Receipt Printer" -> [TSP143LAN], null, null

    df = apply_rule(df, "countpattern col: desc on: /[^\\s\\W]\\w+\\d+\\w*/ quote: '\"'");
    assertRow(df.rows.get(0), new Object[]{"Nortel Networks T7316 \"E Nt8 B27\"", 1L});
    assertRow(df.rows.get(1), new Object[]{"SM \"TSP800 TSP847IIU Receipt Printer", 0L});
    assertRow(df.rows.get(2), new Object[]{"SM \"\"TSP100 TSP143LAN Receipt\" Printer", 2L});
    assertRow(df.rows.get(3), new Object[]{"SM \"\"TSP100\" TSP143LAN Receipt Printer", 1L});
  }

  @Test
  public void test_countpattern_extra() throws IOException, TeddyException {
    DataFrame sales = new DataFrame();
    sales.setByGrid(grids.get("sales_regex_test"));
    sales = prepare_sales_regex_test(sales);

    String ruleString = "countpattern col: desc on: /[a-zA-Z]+/ quote:'\"' global:true";
    DataFrame newDf = apply_rule(sales, ruleString);
    newDf.show();

    assertEquals(3L, newDf.rows.get(0).get("countpattern_desc"));
    assertEquals(3L, newDf.rows.get(0).get("countpattern_desc"));
    assertEquals(3L, newDf.rows.get(0).get("countpattern_desc"));
    assertEquals(3L, newDf.rows.get(0).get("countpattern_desc"));
  }

  // Old test cases

  @Test
  public void test_countpattern_2_columns() throws IOException, TeddyException {
    DataFrame contract = new DataFrame();
    contract.setByGrid(grids.get("contract"));
    contract = prepare_contract(contract);
    contract.show();

    String ruleString = "countpattern col: cdate, customer_id on: '08'";

    DataFrame newDf = apply_rule(contract, ruleString);
    newDf.show();
  }

  @Test
  public void test_countpattern_3_columns_with_casting() throws IOException, TeddyException {
    DataFrame contract = new DataFrame();
    contract.setByGrid(grids.get("contract"));
    contract = prepare_contract(contract);
    contract.show();

    contract = apply_rule(contract, "settype col: detail_store_code type: string");

    String ruleString = "countpattern col: cdate, customer_id, detail_store_code on: '08'";
    DataFrame newDf = apply_rule(contract, ruleString);

    ruleString = "sort order: countpattern_cdate_customer_id_detail_store_code type: 'desc'";

    newDf = apply_rule(newDf, ruleString);

    newDf.show(200);
  }

  @Test
  public void test_countpattern_ignorecase() throws IOException, TeddyException {
    DataFrame df = new DataFrame();
    df.setByGrid(grids.get("sample"));
    df = prepare_sample(df);
    df.show();

    String ruleString = "countpattern col: name on: 'm'";

    DataFrame newDf = apply_rule(df, ruleString);
    newDf.show();

    ruleString = "countpattern col: name on: 'm' ignoreCase: true";

    newDf = apply_rule(newDf, ruleString);
    newDf.show();
  }

}


text:
<?php

/* RedbearShopBundle:Blog:blog.html.twig */
class __TwigTemplate_7cabfb1c13a50b9c3936f395e30157bcf6219f050adcff55017146318bc92f40 extends Twig_Template
{
    public function __construct(Twig_Environment $env)
    {
        parent::__construct($env);

        $this->parent = $this->env->loadTemplate("RedbearShopBundle::layout.html.twig");

        $this->blocks = array(
        );
    }

    protected function doGetParent(array $context)
    {
        return "RedbearShopBundle::layout.html.twig";
    }

    protected function doDisplay(array $context, array $blocks = array())
    {
        $this->parent->display($context, array_merge($this->blocks, $blocks));
    }

    public function getTemplateName()
    {
        return "RedbearShopBundle:Blog:blog.html.twig";
    }

    public function isTraitable()
    {
        return false;
    }

    public function getDebugInfo()
    {
        return array ();
    }
}

redpajama_set_name: RedPajamaGithub


text:
Alerts & Query Matching
=======================

In the following tutorial we'll learn how to generate time window metrics on some http traffic (apache log records) and
how to raise custom alerts based on lucene matching query criterion.

We assume that you already know how to parse and ingest Apache logs into logisland.
If it's not the case please refer to the previous `Apache logs indexing tutorial <index-apache-logs.html>`_.
We will first add an `SQLAggregator </plugins.html#kafkarecordstreamsqlaggregator>`_  Stream
to compute some metrics and then add a `MatchQuery </plugins.html#matchquery>`_ Processor.

.. note::

    You can download the `latest release <https://github.com/Hurence/logisland/releases>`_ of logisland and the `YAML configuration file <https://github.com/Hurence/logisland/blob/master/logisland-framework/logisland-resources/src/main/resources/conf/query-matching.yml>`_ for this tutorial which can be also found under `$LOGISLAND_HOME/conf` directory.

1. Setup SQL Aggregation Stream
-------------------------------
Our application will be composed of 2 streams, the first one use a `KafkaRecordStreamSQLAggregator </plugins.html#kafkarecordstreamsqlaggregator>`_.
This stream defines input/output topics names as well as Serializers, avro schema.

.. note::

    The `Avro <http://avro.apache.org/docs/1.7.7/spec.html>`_ schema is set for the input topic and must be same as the avro schema of the output topic for the stream that
    produces the records (please refer to `Index Apache logs tutorial <index-apache-logs.html>`_

The most important part of the `KafkaRecordStreamSQLAggregator` is its `sql.query` property which defines
a query to apply on the incoming records for the given time window.

The following SQL query will be applied

.. code-block:: sql

    SELECT count(*) AS connections_count, avg(bytes_out) AS avg_bytes_out, src_ip, first(record_time) as record_time
    FROM logisland_events
    GROUP BY src_ip
    ORDER BY connections_count DESC
    LIMIT 20

which will consider ``logisland_events`` topic as SQL table and create 20 output Record with the fields avg_bytes_out, src_ip & record_time.
the statement with record_time will ensure that the created Records will correspond to the effective input event time (not the actual time).

.. code-block:: yaml

    - stream: metrics_by_host
      component: com.hurence.logisland.stream.spark.KafkaRecordStreamSQLAggregator
      type: stream
      documentation: a processor that links
      configuration:
        kafka.input.topics: logisland_events
        kafka.output.topics: logisland_aggregations
        kafka.error.topics: logisland_errors
        kafka.input.topics.serializer: com.hurence.logisland.serializer.KryoSerializer
        kafka.output.topics.serializer: com.hurence.logisland.serializer.KryoSerializer
        kafka.error.topics.serializer: com.hurence.logisland.serializer.JsonSerializer
        kafka.metadata.broker.list: sandbox:9092
        kafka.zookeeper.quorum: sandbox:2181
        kafka.topic.autoCreate: true
        kafka.topic.default.partitions: 2
        kafka.topic.default.replicationFactor: 1
        avro.input.schema: >
          {  "version":1,
             "type": "record",
             "name": "com.hurence.logisland.record.apache_log",
             "fields": [
               { "name": "record_errors",   "type": [ {"type": "array", "items": "string"},"null"] },
               { "name": "record_raw_key", "type": ["string","null"] },
               { "name": "record_raw_value", "type": ["string","null"] },
               { "name": "record_id",   "type": ["string"] },
               { "name": "record_time", "type": ["long"] },
               { "name": "record_type", "type": ["string"] },
               { "name": "src_ip",      "type": ["string","null"] },
               { "name": "http_method", "type": ["string","null"] },
               { "name": "bytes_out",   "type": ["long","null"] },
               { "name": "http_query",  "type": ["string","null"] },
               { "name": "http_version","type": ["string","null"] },
               { "name": "http_status", "type": ["string","null"] },
               { "name": "identd",      "type": ["string","null"] },
               { "name": "user",        "type": ["string","null"] }    ]}
        sql.query: >
          SELECT count(*) AS connections_count, avg(bytes_out) AS avg_bytes_out, src_ip
          FROM logisland_events
          GROUP BY src_ip
          ORDER BY event_count DESC
          LIMIT 20
        max.results.count: 1000
        output.record.type: top_client_metrics

Here we will compute every x seconds, the top twenty `src_ip` for connections count.
The result of the query will be pushed into to `logisland_aggregations` topic as new `top_client_metrics` Record containing `connections_count` and `avg_bytes_out` fields.

2. Setup Query matching Stream on log Records
---------------------------------------------
The second stream makes use of the `KafkaRecordStreamParallelProcessing <../plugins.html#kafkarecordstreamparallelprocessing>`_ Stream with a
`MatchQuery <plugins.html#matchquery>`_ Processor. This processor provides user with dynamic query registration.
This queries are expressed in the Lucene syntax.

.. note::

    Please read the `Lucene syntax guide <https://lucene.apache.org/core/5_5_0/queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description>`_ for supported operations.

We'll use 2 streams for query matching because we will handle 2 kind of Records.
The first one will send an alert when a particular host (src_ip:199.0.2.27) will make a connection
and anywhen someone from *.edu domain makes a connection (src_ip:*.edu).

.. code-block:: yaml

    # match threshold queries
    - stream: query_matching_stream
      component: com.hurence.logisland.stream.spark.KafkaRecordStreamParallelProcessing
      type: stream
      documentation: a processor that match query in parrallel
      configuration:
        kafka.input.topics: logisland_events
        kafka.output.topics: logisland_alerts
        kafka.error.topics: logisland_errors
        kafka.input.topics.serializer: com.hurence.logisland.serializer.KryoSerializer
        kafka.output.topics.serializer: com.hurence.logisland.serializer.KryoSerializer
        kafka.error.topics.serializer: com.hurence.logisland.serializer.JsonSerializer
        kafka.metadata.broker.list: sandbox:9092
        kafka.zookeeper.quorum: sandbox:2181
        kafka.topic.autoCreate: true
        kafka.topic.default.partitions: 2
        kafka.topic.default.replicationFactor: 1
      processorConfigurations:
        - processor: match_query
          component: com.hurence.logisland.processor.MatchQuery
          type: processor
          documentation: a parser that produce events from an apache log REGEX
          configuration:
            blacklisted_host: src_ip:slip-5.io.com
            edu_host: src_ip:edu
            output.record.type: connection_alert

3. Setup Query matching Stream
------------------------------
The third one will match numeric fields on sql aggregates computed in the very first stream in this tutorial.

.. code-block:: yaml

    # match threshold queries
    - stream: query_matching_stream
      component: com.hurence.logisland.stream.spark.KafkaRecordStreamParallelProcessing
      type: stream
      documentation: a processor that match query in parrallel
      configuration:
        kafka.input.topics: logisland_aggregations
        kafka.output.topics: logisland_alerts
        kafka.error.topics: logisland_errors
        kafka.input.topics.serializer: com.hurence.logisland.serializer.KryoSerializer
        kafka.output.topics.serializer: com.hurence.logisland.serializer.KryoSerializer
        kafka.error.topics.serializer: com.hurence.logisland.serializer.JsonSerializer
        kafka.metadata.broker.list: sandbox:9092
        kafka.zookeeper.quorum: sandbox:2181
        kafka.topic.autoCreate: true
        kafka.topic.default.partitions: 2
        kafka.topic.default.replicationFactor: 1
      processorConfigurations:
        - processor: match_query
          component: com.hurence.logisland.processor.MatchQuery
          type: processor
          documentation: a parser that produce events from an apache log REGEX
          configuration:
            numeric.fields: bytes_out,connections_count
            too_much_bandwidth: average_bytes:[100 TO 50000]
            too_many_connections: connections_count:[500 TO 1000000]
            output.record.type: threshold_alert

4. Start logisland application
------------------------------
Connect a shell to your logisland container to launch the following stream processing job previously defined.

.. code-block:: sh

    docker exec -ti logisland bash

    #launch logisland streams
    cd $LOGISLAND_HOME
    bin/logisland.sh --conf conf/index-apache-logs.yml
    bin/logisland.sh --conf conf/query-matching.yml

    # send logs to kafka
    head 500000 NASA_access_log_Jul95 | kafkacat -b sandbox:9092 -t logisland_raw

5. Check your alerts with Kibana
--------------------------------
Open up your browser and go to `http://sandbox:5601/ <http://sandbox:5601/app/kibana#/discover?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'1995-05-08T12:14:53.216Z',mode:absolute,to:'1995-11-25T05:30:52.010Z'))&_a=(columns:!(_source),filters:!(),index:'li-*',interval:auto,query:(query_string:(analyze_wildcard:!t,query:usa)),sort:!('@timestamp',desc),vis:(aggs:!((params:(field:host,orderBy:'2',size:20),schema:segment,type:terms),(id:'2',schema:metric,type:count)),type:histogram))&indexPattern=li-*&type=histogram>`_ and you should be able to explore your apache logs.

As we explore data logs from july 1995 we'll have to select an absolute time filter from 1995-06-30 to 1995-07-08 to see the events.

.. image:: /_static/kibana-apache-logs.png

you can filter your events with ``record_type:connection_alert`` to get 71733 connections alerts matching your query

.. image:: /_static/kibana-connection-alerts.png

by adding another filter on ``alert_match_name:blacklisted_host`` you'll only get request from ``slip-5.io.com`` which is a host we where monitoring.

.. image:: /_static/kibana-blacklisted-host.png

if we filter now on threshold alerts whith ``record_type:threshold_alert`` you'll get the 13 src_ip that have been catched by the threshold query.

.. image:: /_static/kibana-threshold-alerts.png


text:
/*
 * Copyright (C) 1990,91   Silicon Graphics, Inc.
 *
 _______________________________________________________________________
 ______________  S I L I C O N   G R A P H I C S   I N C .  ____________
 |
 |   $Revision: 1.1.1.1 $
 |
 |   Classes:
 |	SoSearchAction
 |
 |   Author(s)		: Paul S. Strauss
 |
 ______________  S I L I C O N   G R A P H I C S   I N C .  ____________
 _______________________________________________________________________
 */

#include <Inventor/actions/SoSearchAction.h>
#include <Inventor/errors/SoDebugError.h>

SO_ACTION_SOURCE(SoSearchAction);

SbBool SoSearchAction::duringSearchAll = FALSE;

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Constructor. Sets up default values: search for first node, no
//    exact match necessary.
//
// Use: public

SoSearchAction::SoSearchAction()
//
////////////////////////////////////////////////////////////////////////
{
    SO_ACTION_CONSTRUCTOR(SoSearchAction);

    retPath = NULL;
    reset();
}

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Destructor.
//
// Use: public

SoSearchAction::~SoSearchAction()
//
////////////////////////////////////////////////////////////////////////
{
    if (retPath != NULL)
	retPath->unref();
}

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Reset everything back to just-constructed state.
//
// Use: public

void
SoSearchAction::reset()
//
////////////////////////////////////////////////////////////////////////
{
    if (retPath)
	retPath->unref();

    retPath = NULL;
    retPaths.truncate(0);

    setType(SoType::badType());
    setNode(NULL);
    setFind(0);
    setInterest(FIRST);
    setSearchingAll(FALSE);
}

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Sets the node to search for.
//
// Use: public

void
SoSearchAction::setNode(SoNode *n)
//
////////////////////////////////////////////////////////////////////////
{
    node = n;

    if (n == NULL)
	lookingFor &= ~NODE;
    else
	lookingFor |= NODE;
}

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Sets the search to look for nodes of a specific type, or to look
//    for no type if BadType is passed in.  The derivedOk flag sets
//    whether or not the types need to match exactly.
//
// Use: public

void
SoSearchAction::setType(SoType t, SbBool dOk)
//
////////////////////////////////////////////////////////////////////////
{
    derivedOk = dOk;
    type = t;

    if (t.isBad())
	lookingFor &= ~TYPE;

    else {
#ifdef DEBUG
	if (! t.isDerivedFrom(SoNode::getClassTypeId()))
	    SoDebugError::postWarning("SoSearchAction::setType",
				      "Type %s is not derived from SoNode",
				      t.getName().getString());
#endif /* DEBUG */

	lookingFor |= TYPE;
    }
}

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Sets the name of the node to look for.
//
// Use: public

void
SoSearchAction::setName(const SbName &n)
//
////////////////////////////////////////////////////////////////////////
{
    name = n;
    lookingFor |= NAME;
}

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Sets the path to return (interest == FIRST or LAST) or adds to
//    the list of paths to return (interest == ALL).
//
// Use: extender

void
SoSearchAction::addPath(SoPath *path)
//
////////////////////////////////////////////////////////////////////////
{
    if (interest == ALL)
	retPaths.append(path);

    else {
	if (path != NULL)
	    path->ref();

	if (retPath != NULL)
	    retPath->unref();

	retPath = path;
    }
}

////////////////////////////////////////////////////////////////////////
//
// Description:
//    Initiates action on a graph.
//
// Use: protected

void
SoSearchAction::beginTraversal(SoNode *node)
//
////////////////////////////////////////////////////////////////////////
{
#ifdef DEBUG
    if (getFind() == 0) {
	SoDebugError::post("SoSearchAction::apply",
			   "No node type, node instance, or node name "
			   "specified for search");
	return;
    }
#endif /* DEBUG */

    // Empty things so we can tell when we have a match
    if (interest == ALL)
	retPaths.truncate(0);
    else
	addPath(NULL);

    // Set duringSearchAll flag (being careful to preserve prev value
    // in case of nested searches) so Switches are traversed properly:
    SbBool prevDuring = duringSearchAll;
    duringSearchAll = searchingAll;

    traverse(node);

    duringSearchAll = prevDuring;
}


text:
@interface InitViewController ()

@end

@implementation InitViewController

- (id)initWithNibName:(NSString *)nibNameOrNil bundle:(NSBundle *)nibBundleOrNil
{
    self = [super initWithNibName:nibNameOrNil bundle:nibBundleOrNil];
    if (self) {
        // Custom initialization
    }
    return self;
}

- (void)viewDidLoad
{
    [super viewDidLoad];
    self.topViewController = [self.storyboard instantiateViewControllerWithIdentifier:@"Remote"];

}

- (void)didReceiveMemoryWarning
{
    [super didReceiveMemoryWarning];
    // Dispose of any resources that can be recreated.
}

@end


text:
'use strict';
var fs = require('fs');

var isDocker;

function hasDockerEnv() {
	try {
		fs.statSync('/.dockerenv');
		return true;
	} catch (err) {
		return false;
	}
}

function hasDockerCGroup() {
	try {
		return fs.readFileSync('/proc/self/cgroup', 'utf8').indexOf('docker') !== -1;
	} catch (err) {
		return false;
	}
}

function check() {
	return hasDockerEnv() || hasDockerCGroup();
}

module.exports = function () {
	if (isDocker === undefined) {
		isDocker = check();
	}

	return isDocker;
};