text:
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "arrow/flight/sql/example/sqlite_server.h"

#define BOOST_NO_CXX98_FUNCTION_BASE  // ARROW-17805
#include <boost/algorithm/string.hpp>
#include <mutex>
#include <random>
#include <sstream>
#include <unordered_map>
#include <utility>

#include <sqlite3.h>

#include "arrow/array/builder_binary.h"
#include "arrow/flight/sql/example/sqlite_sql_info.h"
#include "arrow/flight/sql/example/sqlite_statement.h"
#include "arrow/flight/sql/example/sqlite_statement_batch_reader.h"
#include "arrow/flight/sql/example/sqlite_tables_schema_batch_reader.h"
#include "arrow/flight/sql/example/sqlite_type_info.h"
#include "arrow/flight/sql/server.h"
#include "arrow/scalar.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/logging.h"

namespace arrow {
namespace flight {
namespace sql {
namespace example {

using arrow::internal::checked_cast;

namespace {

std::string PrepareQueryForGetTables(const GetTables& command) {
  std::stringstream table_query;

  table_query << "SELECT 'main' as catalog_name, null as schema_name, name as "
                 "table_name, type as table_type FROM sqlite_master where 1=1";

  if (command.catalog.has_value()) {
    table_query << " and catalog_name='" << command.catalog.value() << "'";
  }

  if (command.db_schema_filter_pattern.has_value()) {
    table_query << " and schema_name LIKE '" << command.db_schema_filter_pattern.value()
                << "'";
  }

  if (command.table_name_filter_pattern.has_value()) {
    table_query << " and table_name LIKE '" << command.table_name_filter_pattern.value()
                << "'";
  }

  if (!command.table_types.empty()) {
    table_query << " and table_type IN (";
    size_t size = command.table_types.size();
    for (size_t i = 0; i < size; i++) {
      table_query << "'" << command.table_types[i] << "'";
      if (size - 1 != i) {
        table_query << ",";
      }
    }

    table_query << ")";
  }

  table_query << " order by table_name";
  return table_query.str();
}

template <typename Callback>
Status SetParametersOnSQLiteStatement(SqliteStatement* statement,
                                      FlightMessageReader* reader, Callback callback) {
  sqlite3_stmt* stmt = statement->GetSqlite3Stmt();
  while (true) {
    ARROW_ASSIGN_OR_RAISE(FlightStreamChunk chunk, reader->Next());
    if (chunk.data == nullptr) break;

    const int64_t num_rows = chunk.data->num_rows();
    if (num_rows == 0) continue;

    ARROW_RETURN_NOT_OK(statement->SetParameters({std::move(chunk.data)}));
    for (int i = 0; i < num_rows; ++i) {
      if (sqlite3_clear_bindings(stmt) != SQLITE_OK) {
        return Status::Invalid("Failed to reset bindings on row ", i, ": ",
                               sqlite3_errmsg(statement->db()));
      }
      // batch_index is always 0 since we're calling SetParameters
      // with a single batch at a time
      ARROW_RETURN_NOT_OK(statement->Bind(/*batch_index=*/0, i));
      ARROW_RETURN_NOT_OK(callback());
    }
  }

  return Status::OK();
}

arrow::Result<std::unique_ptr<FlightDataStream>> DoGetSQLiteQuery(
    sqlite3* db, const std::string& query, const std::shared_ptr<Schema>& schema) {
  std::shared_ptr<SqliteStatement> statement;

  ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db, query));

  std::shared_ptr<SqliteStatementBatchReader> reader;
  ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create(statement, schema));

  return std::make_unique<RecordBatchStream>(reader);
}

arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoForCommand(
    const FlightDescriptor& descriptor, const std::shared_ptr<Schema>& schema) {
  std::vector<FlightEndpoint> endpoints{FlightEndpoint{{descriptor.cmd}, {}}};
  ARROW_ASSIGN_OR_RAISE(auto result,
                        FlightInfo::Make(*schema, descriptor, endpoints, -1, -1))

  return std::make_unique<FlightInfo>(result);
}

std::string PrepareQueryForGetImportedOrExportedKeys(const std::string& filter) {
  return R"(SELECT * FROM (SELECT NULL AS pk_catalog_name,
    NULL AS pk_schema_name,
    p."table" AS pk_table_name,
    p."to" AS pk_column_name,
    NULL AS fk_catalog_name,
    NULL AS fk_schema_name,
    m.name AS fk_table_name,
    p."from" AS fk_column_name,
    p.seq AS key_sequence,
    NULL AS pk_key_name,
    NULL AS fk_key_name,
    CASE
        WHEN p.on_update = 'CASCADE' THEN 0
        WHEN p.on_update = 'RESTRICT' THEN 1
        WHEN p.on_update = 'SET NULL' THEN 2
        WHEN p.on_update = 'NO ACTION' THEN 3
        WHEN p.on_update = 'SET DEFAULT' THEN 4
    END AS update_rule,
    CASE
        WHEN p.on_delete = 'CASCADE' THEN 0
        WHEN p.on_delete = 'RESTRICT' THEN 1
        WHEN p.on_delete = 'SET NULL' THEN 2
        WHEN p.on_delete = 'NO ACTION' THEN 3
        WHEN p.on_delete = 'SET DEFAULT' THEN 4
    END AS delete_rule
  FROM sqlite_master m
  JOIN pragma_foreign_key_list(m.name) p ON m.name != p."table"
  WHERE m.type = 'table') WHERE )" +
         filter + R"( ORDER BY
  pk_catalog_name, pk_schema_name, pk_table_name, pk_key_name, key_sequence)";
}

}  // namespace

arrow::Result<std::shared_ptr<DataType>> GetArrowType(const char* sqlite_type) {
  if (sqlite_type == nullptr || std::strlen(sqlite_type) == 0) {
    // SQLite may not know the column type yet.
    return null();
  }

  if (boost::iequals(sqlite_type, "int") || boost::iequals(sqlite_type, "integer")) {
    return int64();
  } else if (boost::iequals(sqlite_type, "REAL")) {
    return float64();
  } else if (boost::iequals(sqlite_type, "BLOB")) {
    return binary();
  } else if (boost::iequals(sqlite_type, "TEXT") || boost::iequals(sqlite_type, "DATE") ||
             boost::istarts_with(sqlite_type, "char") ||
             boost::istarts_with(sqlite_type, "varchar")) {
    return utf8();
  }
  return Status::Invalid("Invalid SQLite type: ", sqlite_type);
}

int32_t GetSqlTypeFromTypeName(const char* sqlite_type) {
  if (sqlite_type == NULLPTR) {
    // SQLite may not know the column type yet.
    return SQLITE_NULL;
  }

  if (boost::iequals(sqlite_type, "int") || boost::iequals(sqlite_type, "integer")) {
    return SQLITE_INTEGER;
  } else if (boost::iequals(sqlite_type, "REAL")) {
    return SQLITE_FLOAT;
  } else if (boost::iequals(sqlite_type, "BLOB")) {
    return SQLITE_BLOB;
  } else if (boost::iequals(sqlite_type, "TEXT") || boost::iequals(sqlite_type, "DATE") ||
             boost::istarts_with(sqlite_type, "char") ||
             boost::istarts_with(sqlite_type, "varchar")) {
    return SQLITE_TEXT;
  } else {
    return SQLITE_NULL;
  }
}

class SQLiteFlightSqlServer::Impl {
 private:
  sqlite3* db_;
  const std::string db_uri_;
  std::mutex mutex_;
  std::unordered_map<std::string, std::shared_ptr<SqliteStatement>> prepared_statements_;
  std::unordered_map<std::string, sqlite3*> open_transactions_;
  std::default_random_engine gen_;

  arrow::Result<std::shared_ptr<SqliteStatement>> GetStatementByHandle(
      const std::string& handle) {
    std::lock_guard<std::mutex> guard(mutex_);
    auto search = prepared_statements_.find(handle);
    if (search == prepared_statements_.end()) {
      return Status::KeyError("Prepared statement not found");
    }
    return search->second;
  }

  arrow::Result<sqlite3*> GetConnection(const std::string& transaction_id) {
    if (transaction_id.empty()) {
      ARROW_LOG(INFO) << "Using default connection";
      return db_;
    }

    std::lock_guard<std::mutex> guard(mutex_);
    auto it = open_transactions_.find(transaction_id);
    if (it == open_transactions_.end()) {
      return Status::KeyError("Unknown transaction ID: ", transaction_id);
    }
    ARROW_LOG(INFO) << "Using connection for transaction " << transaction_id;
    return it->second;
  }

  // Create a Ticket that combines a query and a transaction ID.
  arrow::Result<Ticket> EncodeTransactionQuery(const std::string& query,
                                               const std::string& transaction_id) {
    std::string transaction_query = transaction_id;
    transaction_query += ':';
    transaction_query += query;
    ARROW_ASSIGN_OR_RAISE(auto ticket_string,
                          CreateStatementQueryTicket(transaction_query));
    return Ticket{std::move(ticket_string)};
  }

  arrow::Result<std::pair<std::string, std::string>> DecodeTransactionQuery(
      const std::string& ticket) {
    auto divider = ticket.find(':');
    if (divider == std::string::npos) {
      return Status::Invalid("Malformed ticket");
    }
    std::string transaction_id = ticket.substr(0, divider);
    std::string query = ticket.substr(divider + 1);
    return std::make_pair(std::move(query), std::move(transaction_id));
  }

 public:
  explicit Impl(sqlite3* db, std::string uri) : db_(db), db_uri_(std::move(uri)) {}

  ~Impl() {
    sqlite3_close(db_);
    for (const auto& pair : open_transactions_) {
      sqlite3_close(pair.second);
    }
  }

  std::string GenerateRandomString() {
    uint32_t length = 16;

    // MSVC doesn't support char types here
    std::uniform_int_distribution<uint16_t> dist(static_cast<uint16_t>('0'),
                                                 static_cast<uint16_t>('Z'));
    std::string ret(length, 0);
    // Don't generate symbols to simplify parsing in DecodeTransactionQuery
    auto get_random_char = [&]() {
      char res;
      while (true) {
        res = static_cast<char>(dist(gen_));
        if (res <= '9' || res >= 'A') break;
      }
      return res;
    };
    std::generate_n(ret.begin(), length, get_random_char);
    return ret;
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoStatement(
      const ServerCallContext& context, const StatementQuery& command,
      const FlightDescriptor& descriptor) {
    const std::string& query = command.query;
    ARROW_ASSIGN_OR_RAISE(auto db, GetConnection(command.transaction_id));
    ARROW_ASSIGN_OR_RAISE(auto statement, SqliteStatement::Create(db, query));
    ARROW_ASSIGN_OR_RAISE(auto schema, statement->GetSchema());
    ARROW_ASSIGN_OR_RAISE(auto ticket,
                          EncodeTransactionQuery(query, command.transaction_id));
    std::vector<FlightEndpoint> endpoints{FlightEndpoint{std::move(ticket), {}}};
    ARROW_ASSIGN_OR_RAISE(auto result,
                          FlightInfo::Make(*schema, descriptor, endpoints, -1, -1))

    return std::make_unique<FlightInfo>(result);
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetStatement(
      const ServerCallContext& context, const StatementQueryTicket& command) {
    ARROW_ASSIGN_OR_RAISE(auto pair, DecodeTransactionQuery(command.statement_handle));
    const std::string& sql = pair.first;
    const std::string transaction_id = pair.second;
    ARROW_ASSIGN_OR_RAISE(auto db, GetConnection(transaction_id));

    std::shared_ptr<SqliteStatement> statement;
    ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db, sql));

    std::shared_ptr<SqliteStatementBatchReader> reader;
    ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create(statement));

    return std::make_unique<RecordBatchStream>(reader);
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoCatalogs(
      const ServerCallContext& context, const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetCatalogsSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetCatalogs(
      const ServerCallContext& context) {
    // https://www.sqlite.org/cli.html
    // > The ".databases" command shows a list of all databases open
    // > in the current connection. There will always be at least
    // > 2. The first one is "main", the original database opened. The
    // > second is "temp", the database used for temporary tables.
    // For our purposes, return only "main" and ignore other databases.
    const std::shared_ptr<Schema>& schema = SqlSchema::GetCatalogsSchema();
    StringBuilder catalog_name_builder;
    ARROW_RETURN_NOT_OK(catalog_name_builder.Append("main"));
    ARROW_ASSIGN_OR_RAISE(auto catalog_name, catalog_name_builder.Finish());
    std::shared_ptr<RecordBatch> batch =
        RecordBatch::Make(schema, 1, {std::move(catalog_name)});
    ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchReader::Make({batch}));
    return std::make_unique<RecordBatchStream>(reader);
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoSchemas(
      const ServerCallContext& context, const GetDbSchemas& command,
      const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetDbSchemasSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetDbSchemas(
      const ServerCallContext& context, const GetDbSchemas& command) {
    // SQLite doesn't support schemas, so pretend we have a single
    // unnamed schema.
    const std::shared_ptr<Schema>& schema = SqlSchema::GetDbSchemasSchema();
    StringBuilder catalog_name_builder;
    StringBuilder schema_name_builder;

    int64_t length = 0;
    // XXX: we don't really implement the full pattern match here
    if ((!command.catalog || command.catalog == "main") &&
        (!command.db_schema_filter_pattern || command.db_schema_filter_pattern == "%")) {
      ARROW_RETURN_NOT_OK(catalog_name_builder.Append("main"));
      ARROW_RETURN_NOT_OK(schema_name_builder.AppendNull());
      length++;
    }

    ARROW_ASSIGN_OR_RAISE(auto catalog_name, catalog_name_builder.Finish());
    ARROW_ASSIGN_OR_RAISE(auto schema_name, schema_name_builder.Finish());
    std::shared_ptr<RecordBatch> batch = RecordBatch::Make(
        schema, length, {std::move(catalog_name), std::move(schema_name)});
    ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchReader::Make({batch}));
    return std::make_unique<RecordBatchStream>(reader);
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoTables(
      const ServerCallContext& context, const GetTables& command,
      const FlightDescriptor& descriptor) {
    std::vector<FlightEndpoint> endpoints{FlightEndpoint{{descriptor.cmd}, {}}};

    bool include_schema = command.include_schema;
    ARROW_LOG(INFO) << "GetTables include_schema=" << include_schema;

    ARROW_ASSIGN_OR_RAISE(
        auto result,
        FlightInfo::Make(include_schema ? *SqlSchema::GetTablesSchemaWithIncludedSchema()
                                        : *SqlSchema::GetTablesSchema(),
                         descriptor, endpoints, -1, -1))

    return std::make_unique<FlightInfo>(std::move(result));
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetTables(
      const ServerCallContext& context, const GetTables& command) {
    std::string query = PrepareQueryForGetTables(command);
    ARROW_LOG(INFO) << "GetTables: " << query;

    std::shared_ptr<SqliteStatement> statement;
    ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db_, query));

    std::shared_ptr<SqliteStatementBatchReader> reader;
    ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create(
                                      statement, SqlSchema::GetTablesSchema()));

    if (command.include_schema) {
      std::shared_ptr<SqliteTablesWithSchemaBatchReader> table_schema_reader =
          std::make_shared<SqliteTablesWithSchemaBatchReader>(reader, query, db_);
      return std::make_unique<RecordBatchStream>(table_schema_reader);
    } else {
      return std::make_unique<RecordBatchStream>(reader);
    }
  }

  arrow::Result<int64_t> DoPutCommandStatementUpdate(const ServerCallContext& context,
                                                     const StatementUpdate& command) {
    const std::string& sql = command.query;
    ARROW_ASSIGN_OR_RAISE(auto db, GetConnection(command.transaction_id));
    ARROW_LOG(INFO) << "Executing update: " << sql;
    ARROW_ASSIGN_OR_RAISE(auto statement, SqliteStatement::Create(db, sql));
    return statement->ExecuteUpdate();
  }

  arrow::Result<ActionCreatePreparedStatementResult> CreatePreparedStatement(
      const ServerCallContext& context,
      const ActionCreatePreparedStatementRequest& request) {
    std::shared_ptr<SqliteStatement> statement;
    ARROW_ASSIGN_OR_RAISE(auto db, GetConnection(request.transaction_id));
    ARROW_LOG(INFO) << "Creating prepared statement: " << request.query;
    ARROW_ASSIGN_OR_RAISE(statement, SqliteStatement::Create(db, request.query));
    std::string handle = GenerateRandomString();

    {
      std::lock_guard<std::mutex> guard(mutex_);
      prepared_statements_[handle] = statement;
    }

    ARROW_ASSIGN_OR_RAISE(auto dataset_schema, statement->GetSchema());

    sqlite3_stmt* stmt = statement->GetSqlite3Stmt();
    const int parameter_count = sqlite3_bind_parameter_count(stmt);
    FieldVector parameter_fields;
    parameter_fields.reserve(parameter_count);

    // As SQLite doesn't know the parameter types before executing the query, the
    // example server is accepting any SQLite supported type as input by using a dense
    // union.
    const std::shared_ptr<DataType>& dense_union_type = GetUnknownColumnDataType();

    for (int i = 0; i < parameter_count; i++) {
      const char* parameter_name_chars = sqlite3_bind_parameter_name(stmt, i + 1);
      std::string parameter_name;
      if (parameter_name_chars == NULLPTR) {
        parameter_name = std::string("parameter_") + std::to_string(i + 1);
      } else {
        parameter_name = parameter_name_chars;
      }
      parameter_fields.push_back(field(parameter_name, dense_union_type));
    }

    std::shared_ptr<Schema> parameter_schema = arrow::schema(parameter_fields);
    return ActionCreatePreparedStatementResult{
        std::move(dataset_schema), std::move(parameter_schema), std::move(handle)};
  }

  Status ClosePreparedStatement(const ServerCallContext& context,
                                const ActionClosePreparedStatementRequest& request) {
    std::lock_guard<std::mutex> guard(mutex_);
    const std::string& prepared_statement_handle = request.prepared_statement_handle;

    auto search = prepared_statements_.find(prepared_statement_handle);
    if (search != prepared_statements_.end()) {
      prepared_statements_.erase(prepared_statement_handle);
    } else {
      return Status::Invalid("Prepared statement not found");
    }

    return Status::OK();
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoPreparedStatement(
      const ServerCallContext& context, const PreparedStatementQuery& command,
      const FlightDescriptor& descriptor) {
    std::lock_guard<std::mutex> guard(mutex_);
    const std::string& prepared_statement_handle = command.prepared_statement_handle;

    auto search = prepared_statements_.find(prepared_statement_handle);
    if (search == prepared_statements_.end()) {
      return Status::Invalid("Prepared statement not found");
    }

    std::shared_ptr<SqliteStatement> statement = search->second;

    ARROW_ASSIGN_OR_RAISE(auto schema, statement->GetSchema());

    return GetFlightInfoForCommand(descriptor, schema);
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetPreparedStatement(
      const ServerCallContext& context, const PreparedStatementQuery& command) {
    std::lock_guard<std::mutex> guard(mutex_);
    const std::string& prepared_statement_handle = command.prepared_statement_handle;

    auto search = prepared_statements_.find(prepared_statement_handle);
    if (search == prepared_statements_.end()) {
      return Status::Invalid("Prepared statement not found");
    }

    std::shared_ptr<SqliteStatement> statement = search->second;

    std::shared_ptr<SqliteStatementBatchReader> reader;
    ARROW_ASSIGN_OR_RAISE(reader, SqliteStatementBatchReader::Create(statement));

    return std::make_unique<RecordBatchStream>(reader);
  }

  Status DoPutPreparedStatementQuery(const ServerCallContext& context,
                                     const PreparedStatementQuery& command,
                                     FlightMessageReader* reader,
                                     FlightMetadataWriter* writer) {
    const std::string& prepared_statement_handle = command.prepared_statement_handle;
    ARROW_ASSIGN_OR_RAISE(auto statement,
                          GetStatementByHandle(prepared_statement_handle));
    // Save params here and execute later
    ARROW_ASSIGN_OR_RAISE(auto batches, reader->ToRecordBatches());
    ARROW_RETURN_NOT_OK(statement->SetParameters(std::move(batches)));
    return Status::OK();
  }

  arrow::Result<int64_t> DoPutPreparedStatementUpdate(
      const ServerCallContext& context, const PreparedStatementUpdate& command,
      FlightMessageReader* reader) {
    const std::string& prepared_statement_handle = command.prepared_statement_handle;
    ARROW_ASSIGN_OR_RAISE(std::shared_ptr<SqliteStatement> statement,
                          GetStatementByHandle(prepared_statement_handle));

    int64_t rows_affected = 0;
    if (sqlite3_bind_parameter_count(statement->GetSqlite3Stmt()) == 0) {
      ARROW_ASSIGN_OR_RAISE(rows_affected, statement->ExecuteUpdate());
    } else {
      ARROW_RETURN_NOT_OK(SetParametersOnSQLiteStatement(statement.get(), reader, [&]() {
        ARROW_ASSIGN_OR_RAISE(int64_t rows, statement->ExecuteUpdate());
        rows_affected += rows;
        return statement->Reset().status();
      }));
    }
    return rows_affected;
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoTableTypes(
      const ServerCallContext& context, const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetTableTypesSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetTableTypes(
      const ServerCallContext& context) {
    std::string query = "SELECT DISTINCT type as table_type FROM sqlite_master";

    return DoGetSQLiteQuery(db_, query, SqlSchema::GetTableTypesSchema());
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoTypeInfo(
      const ServerCallContext& context, const GetXdbcTypeInfo& command,
      const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetXdbcTypeInfoSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetTypeInfo(
      const ServerCallContext& context, const GetXdbcTypeInfo& command) {
    const std::shared_ptr<RecordBatch>& type_info_result =
        command.data_type.has_value() ? DoGetTypeInfoResult(command.data_type.value())
                                      : DoGetTypeInfoResult();

    ARROW_ASSIGN_OR_RAISE(auto reader, RecordBatchReader::Make({type_info_result}));
    return std::make_unique<RecordBatchStream>(reader);
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoPrimaryKeys(
      const ServerCallContext& context, const GetPrimaryKeys& command,
      const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetPrimaryKeysSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetPrimaryKeys(
      const ServerCallContext& context, const GetPrimaryKeys& command) {
    std::stringstream table_query;

    // The field key_name can not be recovered by the sqlite, so it is being set
    // to null following the same pattern for catalog_name and schema_name.
    table_query << "SELECT null as catalog_name, null as schema_name, table_name, "
                   "name as column_name,  pk as key_sequence, null as key_name\n"
                   "FROM pragma_table_info(table_name)\n"
                   "    JOIN (SELECT null as catalog_name, null as schema_name, name as "
                   "table_name, type as table_type\n"
                   "FROM sqlite_master) where 1=1 and pk != 0";

    const TableRef& table_ref = command.table_ref;
    if (table_ref.catalog.has_value()) {
      table_query << " and catalog_name LIKE '" << table_ref.catalog.value() << "'";
    }

    if (table_ref.db_schema.has_value()) {
      table_query << " and schema_name LIKE '" << table_ref.db_schema.value() << "'";
    }

    table_query << " and table_name LIKE '" << table_ref.table << "'";

    return DoGetSQLiteQuery(db_, table_query.str(), SqlSchema::GetPrimaryKeysSchema());
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoImportedKeys(
      const ServerCallContext& context, const GetImportedKeys& command,
      const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetImportedKeysSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetImportedKeys(
      const ServerCallContext& context, const GetImportedKeys& command) {
    const TableRef& table_ref = command.table_ref;
    std::string filter = "fk_table_name = '" + table_ref.table + "'";
    if (table_ref.catalog.has_value()) {
      filter += " AND fk_catalog_name = '" + table_ref.catalog.value() + "'";
    }
    if (table_ref.db_schema.has_value()) {
      filter += " AND fk_schema_name = '" + table_ref.db_schema.value() + "'";
    }
    std::string query = PrepareQueryForGetImportedOrExportedKeys(filter);

    return DoGetSQLiteQuery(db_, query, SqlSchema::GetImportedKeysSchema());
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoExportedKeys(
      const ServerCallContext& context, const GetExportedKeys& command,
      const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetExportedKeysSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetExportedKeys(
      const ServerCallContext& context, const GetExportedKeys& command) {
    const TableRef& table_ref = command.table_ref;
    std::string filter = "pk_table_name = '" + table_ref.table + "'";
    if (table_ref.catalog.has_value()) {
      filter += " AND pk_catalog_name = '" + table_ref.catalog.value() + "'";
    }
    if (table_ref.db_schema.has_value()) {
      filter += " AND pk_schema_name = '" + table_ref.db_schema.value() + "'";
    }
    std::string query = PrepareQueryForGetImportedOrExportedKeys(filter);

    return DoGetSQLiteQuery(db_, query, SqlSchema::GetExportedKeysSchema());
  }

  arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoCrossReference(
      const ServerCallContext& context, const GetCrossReference& command,
      const FlightDescriptor& descriptor) {
    return GetFlightInfoForCommand(descriptor, SqlSchema::GetCrossReferenceSchema());
  }

  arrow::Result<std::unique_ptr<FlightDataStream>> DoGetCrossReference(
      const ServerCallContext& context, const GetCrossReference& command) {
    const TableRef& pk_table_ref = command.pk_table_ref;
    std::string filter = "pk_table_name = '" + pk_table_ref.table + "'";
    if (pk_table_ref.catalog.has_value()) {
      filter += " AND pk_catalog_name = '" + pk_table_ref.catalog.value() + "'";
    }
    if (pk_table_ref.db_schema.has_value()) {
      filter += " AND pk_schema_name = '" + pk_table_ref.db_schema.value() + "'";
    }

    const TableRef& fk_table_ref = command.fk_table_ref;
    filter += " AND fk_table_name = '" + fk_table_ref.table + "'";
    if (fk_table_ref.catalog.has_value()) {
      filter += " AND fk_catalog_name = '" + fk_table_ref.catalog.value() + "'";
    }
    if (fk_table_ref.db_schema.has_value()) {
      filter += " AND fk_schema_name = '" + fk_table_ref.db_schema.value() + "'";
    }
    std::string query = PrepareQueryForGetImportedOrExportedKeys(filter);

    return DoGetSQLiteQuery(db_, query, SqlSchema::GetCrossReferenceSchema());
  }

  Status ExecuteSql(const std::string& sql) { return ExecuteSql(db_, sql); }

  Status ExecuteSql(sqlite3* db, const std::string& sql) {
    char* err_msg = nullptr;
    int rc = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &err_msg);
    if (rc != SQLITE_OK) {
      std::string error_msg;
      if (err_msg != nullptr) {
        error_msg = err_msg;
        sqlite3_free(err_msg);
      }
      return Status::IOError(error_msg);
    }
    if (err_msg) sqlite3_free(err_msg);
    return Status::OK();
  }

  arrow::Result<ActionBeginTransactionResult> BeginTransaction(
      const ServerCallContext& context, const ActionBeginTransactionRequest& request) {
    std::string handle = GenerateRandomString();
    sqlite3* new_db = nullptr;
    if (sqlite3_open_v2(db_uri_.c_str(), &new_db,
                        SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI,
                        /*zVfs=*/nullptr) != SQLITE_OK) {
      std::string error_message = "Can't open new connection: ";
      if (new_db) {
        error_message += sqlite3_errmsg(new_db);
        sqlite3_close(new_db);
      }
      return Status::Invalid(error_message);
    }

    ARROW_RETURN_NOT_OK(ExecuteSql(new_db, "BEGIN TRANSACTION"));

    ARROW_LOG(INFO) << "Beginning transaction on " << handle;

    std::lock_guard<std::mutex> guard(mutex_);
    open_transactions_[handle] = new_db;
    return ActionBeginTransactionResult{std::move(handle)};
  }

  Status EndTransaction(const ServerCallContext& context,
                        const ActionEndTransactionRequest& request) {
    Status status;
    sqlite3* transaction = nullptr;
    {
      std::lock_guard<std::mutex> guard(mutex_);
      auto it = open_transactions_.find(request.transaction_id);
      if (it == open_transactions_.end()) {
        return Status::KeyError("Unknown transaction ID: ", request.transaction_id);
      }

      if (request.action == ActionEndTransactionRequest::kCommit) {
        ARROW_LOG(INFO) << "Committing on " << request.transaction_id;
        status = ExecuteSql(it->second, "COMMIT");
      } else {
        ARROW_LOG(INFO) << "Rolling back on " << request.transaction_id;
        status = ExecuteSql(it->second, "ROLLBACK");
      }
      transaction = it->second;
      open_transactions_.erase(it);
    }
    sqlite3_close(transaction);
    return status;
  }
};

// Give each server instance its own in-memory DB
std::atomic<int64_t> kDbCounter(0);

SQLiteFlightSqlServer::SQLiteFlightSqlServer(std::shared_ptr<Impl> impl)
    : impl_(std::move(impl)) {}

arrow::Result<std::shared_ptr<SQLiteFlightSqlServer>> SQLiteFlightSqlServer::Create() {
  sqlite3* db = nullptr;

  // All sqlite3* instances created from this URI will share data
  std::string uri = "file:memorydb";
  uri += std::to_string(kDbCounter++);
  uri += "?mode=memory&cache=shared";
  if (sqlite3_open_v2(uri.c_str(), &db,
                      SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI,
                      /*zVfs=*/nullptr)) {
    std::string err_msg = "Can't open database: ";
    if (db != nullptr) {
      err_msg += sqlite3_errmsg(db);
      sqlite3_close(db);
    } else {
      err_msg += "Unable to start SQLite. Insufficient memory";
    }

    return Status::Invalid(err_msg);
  }

  std::shared_ptr<Impl> impl = std::make_shared<Impl>(db, std::move(uri));

  std::shared_ptr<SQLiteFlightSqlServer> result(
      new SQLiteFlightSqlServer(std::move(impl)));
  for (const auto& id_to_result : GetSqlInfoResultMap()) {
    result->RegisterSqlInfo(id_to_result.first, id_to_result.second);
  }

  ARROW_RETURN_NOT_OK(result->ExecuteSql(R"(
    CREATE TABLE foreignTable (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    foreignName varchar(100),
    value int);

    CREATE TABLE intTable (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    keyName varchar(100),
    value int,
    foreignId int references foreignTable(id));

    INSERT INTO foreignTable (foreignName, value) VALUES ('keyOne', 1);
    INSERT INTO foreignTable (foreignName, value) VALUES ('keyTwo', 0);
    INSERT INTO foreignTable (foreignName, value) VALUES ('keyThree', -1);
    INSERT INTO intTable (keyName, value, foreignId) VALUES ('one', 1, 1);
    INSERT INTO intTable (keyName, value, foreignId) VALUES ('zero', 0, 1);
    INSERT INTO intTable (keyName, value, foreignId) VALUES ('negative one', -1, 1);
    INSERT INTO intTable (keyName, value, foreignId) VALUES (NULL, NULL, NULL);
    INSERT INTO intTable (keyName, value, foreignId) VALUES ('null', NULL, NULL);
  )"));

  return result;
}

SQLiteFlightSqlServer::~SQLiteFlightSqlServer() = default;

Status SQLiteFlightSqlServer::ExecuteSql(const std::string& sql) {
  return impl_->ExecuteSql(sql);
}

arrow::Result<std::unique_ptr<FlightInfo>> SQLiteFlightSqlServer::GetFlightInfoStatement(
    const ServerCallContext& context, const StatementQuery& command,
    const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoStatement(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetStatement(
    const ServerCallContext& context, const StatementQueryTicket& command) {
  return impl_->DoGetStatement(context, command);
}

arrow::Result<std::unique_ptr<FlightInfo>> SQLiteFlightSqlServer::GetFlightInfoCatalogs(
    const ServerCallContext& context, const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoCatalogs(context, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetCatalogs(
    const ServerCallContext& context) {
  return impl_->DoGetCatalogs(context);
}

arrow::Result<std::unique_ptr<FlightInfo>> SQLiteFlightSqlServer::GetFlightInfoSchemas(
    const ServerCallContext& context, const GetDbSchemas& command,
    const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoSchemas(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetDbSchemas(
    const ServerCallContext& context, const GetDbSchemas& command) {
  return impl_->DoGetDbSchemas(context, command);
}

arrow::Result<std::unique_ptr<FlightInfo>> SQLiteFlightSqlServer::GetFlightInfoTables(
    const ServerCallContext& context, const GetTables& command,
    const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoTables(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetTables(
    const ServerCallContext& context, const GetTables& command) {
  return impl_->DoGetTables(context, command);
}

arrow::Result<int64_t> SQLiteFlightSqlServer::DoPutCommandStatementUpdate(
    const ServerCallContext& context, const StatementUpdate& command) {
  return impl_->DoPutCommandStatementUpdate(context, command);
}

arrow::Result<ActionCreatePreparedStatementResult>
SQLiteFlightSqlServer::CreatePreparedStatement(
    const ServerCallContext& context,
    const ActionCreatePreparedStatementRequest& request) {
  return impl_->CreatePreparedStatement(context, request);
}

Status SQLiteFlightSqlServer::ClosePreparedStatement(
    const ServerCallContext& context,
    const ActionClosePreparedStatementRequest& request) {
  return impl_->ClosePreparedStatement(context, request);
}

arrow::Result<std::unique_ptr<FlightInfo>>
SQLiteFlightSqlServer::GetFlightInfoPreparedStatement(
    const ServerCallContext& context, const PreparedStatementQuery& command,
    const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoPreparedStatement(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>>
SQLiteFlightSqlServer::DoGetPreparedStatement(const ServerCallContext& context,
                                              const PreparedStatementQuery& command) {
  return impl_->DoGetPreparedStatement(context, command);
}

Status SQLiteFlightSqlServer::DoPutPreparedStatementQuery(
    const ServerCallContext& context, const PreparedStatementQuery& command,
    FlightMessageReader* reader, FlightMetadataWriter* writer) {
  return impl_->DoPutPreparedStatementQuery(context, command, reader, writer);
}

arrow::Result<int64_t> SQLiteFlightSqlServer::DoPutPreparedStatementUpdate(
    const ServerCallContext& context, const PreparedStatementUpdate& command,
    FlightMessageReader* reader) {
  return impl_->DoPutPreparedStatementUpdate(context, command, reader);
}

arrow::Result<std::unique_ptr<FlightInfo>> SQLiteFlightSqlServer::GetFlightInfoTableTypes(
    const ServerCallContext& context, const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoTableTypes(context, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetTableTypes(
    const ServerCallContext& context) {
  return impl_->DoGetTableTypes(context);
}

arrow::Result<std::unique_ptr<FlightInfo>>
SQLiteFlightSqlServer::GetFlightInfoXdbcTypeInfo(
    const ServerCallContext& context, const arrow::flight::sql::GetXdbcTypeInfo& command,
    const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoTypeInfo(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetXdbcTypeInfo(
    const ServerCallContext& context,
    const arrow::flight::sql::GetXdbcTypeInfo& command) {
  return impl_->DoGetTypeInfo(context, command);
}

arrow::Result<std::unique_ptr<FlightInfo>>
SQLiteFlightSqlServer::GetFlightInfoPrimaryKeys(const ServerCallContext& context,
                                                const GetPrimaryKeys& command,
                                                const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoPrimaryKeys(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetPrimaryKeys(
    const ServerCallContext& context, const GetPrimaryKeys& command) {
  return impl_->DoGetPrimaryKeys(context, command);
}

arrow::Result<std::unique_ptr<FlightInfo>>
SQLiteFlightSqlServer::GetFlightInfoImportedKeys(const ServerCallContext& context,
                                                 const GetImportedKeys& command,
                                                 const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoImportedKeys(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetImportedKeys(
    const ServerCallContext& context, const GetImportedKeys& command) {
  return impl_->DoGetImportedKeys(context, command);
}

arrow::Result<std::unique_ptr<FlightInfo>>
SQLiteFlightSqlServer::GetFlightInfoExportedKeys(const ServerCallContext& context,
                                                 const GetExportedKeys& command,
                                                 const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoExportedKeys(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>> SQLiteFlightSqlServer::DoGetExportedKeys(
    const ServerCallContext& context, const GetExportedKeys& command) {
  return impl_->DoGetExportedKeys(context, command);
}

arrow::Result<std::unique_ptr<FlightInfo>>
SQLiteFlightSqlServer::GetFlightInfoCrossReference(const ServerCallContext& context,
                                                   const GetCrossReference& command,
                                                   const FlightDescriptor& descriptor) {
  return impl_->GetFlightInfoCrossReference(context, command, descriptor);
}

arrow::Result<std::unique_ptr<FlightDataStream>>
SQLiteFlightSqlServer::DoGetCrossReference(const ServerCallContext& context,
                                           const GetCrossReference& command) {
  return impl_->DoGetCrossReference(context, command);
}

arrow::Result<ActionBeginTransactionResult> SQLiteFlightSqlServer::BeginTransaction(
    const ServerCallContext& context, const ActionBeginTransactionRequest& request) {
  return impl_->BeginTransaction(context, request);
}
Status SQLiteFlightSqlServer::EndTransaction(const ServerCallContext& context,
                                             const ActionEndTransactionRequest& request) {
  return impl_->EndTransaction(context, request);
}

}  // namespace example
}  // namespace sql
}  // namespace flight
}  // namespace arrow


text:
package com.socialthingy.plusf.z80.operations;

import com.socialthingy.plusf.z80.ContentionModel;
import com.socialthingy.plusf.z80.Processor;

public class OpRet extends RetOperation {
    public OpRet(final Processor processor) {
        super(processor);
    }

    @Override
    public void execute(ContentionModel contentionModel, int initialPcValue, int irValue) {
        contentionModel.applyContention(initialPcValue, 4);
        final int sp = processor.register("sp").get();
        contentionModel.applyContention(sp, 3);
        contentionModel.applyContention(sp + 1, 3);
        ret();
    }

    @Override
    public String toString() {
        return "ret";
    }
}

redpajama_set_name: RedPajamaGithub


text:
package com.deleidos.rtws.commons.net.dns.security;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.security.NoSuchAlgorithmException;
import java.security.InvalidKeyException;
import java.security.SignatureException;
import org.apache.commons.codec.binary.Base64;

public class Sha1DigestSignature implements Signature {

	private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";

	public String sign(String inputData, byte[] key) throws SignatureException {
		String digest = "";
		try {
			SecretKeySpec signingKey = new SecretKeySpec(key, HMAC_SHA1_ALGORITHM);
			// get an hmac_sha1 Mac instance and initialize with the signing key
			Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM);
			mac.init(signingKey);

			// compute the hmac on input data bytes
			byte[] rawHmac = mac.doFinal(inputData.getBytes());

			// base64-encode the hmac
			digest = Base64.encodeBase64String(rawHmac);

		} catch (IllegalArgumentException iae) {
			throw new SignatureException("Illegal Argument: " + iae.getMessage(), iae);
		} catch (NoSuchAlgorithmException nse) {
			throw new SignatureException("No Such Algorithm: " + nse.getMessage(), nse);
		} catch (InvalidKeyException ike) {
			throw new SignatureException("Invalid Key: " + ike.getMessage(), ike);
		} catch (Exception ex) {
			throw new SignatureException("Signature Error: " + ex.getMessage(), ex);
		}

		return digest.trim();
	}
}


text:
using System;
using Delivered;

namespace DemoDistributor.Endpoints.Sharepoint
{
    public class SharepointEndpoint : IEndpoint
    {
        public Guid Id { get; set; }
        public string Uri { get; set; }
    }
}


text:
layout: page
title: This is me :)
permalink: /about-me
image: "images/my-pic.jpg"
---

Introverted, Dreamer yet a Rational. Science Enthusiast. Wanderer and an Amateur Writer.
MBTI type - INTJ
A student from India currently pursuing graduate studies.


text:
/*global angular, define, console, window, navigator, document, Hammer, Cropper*/

define(function () {
    'use strict';

    function ctrl(HueService, $scope, DataService, $filter, PlaceholderDataUrl) {

        $scope.$on('modal.shown', function (event, modal) {
            if (modal.id === 'sceneModalImageCrop') {

                initImageCropping();
                initPinchEvent();
            }
        });

        var initImageCropping = function () {
            var image = document.getElementById('imageToCrop');
            image.setAttribute('crossOrigin', 'anonymous');
            image.onload = function () {
                $scope.cropper = new Cropper(image, {
                    viewMode: 3,
                    dragMode: 'move',
                    aspectRatio: 1 / 1,
                    zoomable: false,
                    toggleDragModeOnDblclick: false,
                    minCropBoxWidth: 100,
                    minCropBoxHeight: 100,
                });

            };
            image.src = "data:image/jpeg;base64," + DataService.getSceneImage();
        };

        var initPinchEvent = function () {
            var myElement = document.getElementById('pinchContainer');
            var mc = new Hammer.Manager(myElement);
            var pinch = new Hammer.Pinch();
            mc.add([pinch]);
            var lastDeltaTime = 0;
            var widthHeight = 0;
            mc.on("pinchstart", function (ev) {
                ev.preventDefault();
                var data = $scope.cropper.getData();
                widthHeight = data.width;
            });
            mc.on("pinchmove", function (ev) {
                ev.preventDefault();
                var val = parseInt(widthHeight * ev.scale);
                var data2 = $scope.cropper.getData();
                data2.width = val;
                data2.height = val;
                $scope.cropper.setData(data2);
            });
        };

        $scope.closeImageCropModal = function () {
            $scope.sceneModalImageCrop.remove();
        };

        $scope.saveImage = function () {
            var jpgDataUrlSrc = $scope.cropper.getCroppedCanvas({
                width: 400,
                height: 400
            }).toDataURL('image/jpeg');

            DataService.setSceneImageCropped(jpgDataUrlSrc);
            $scope.closeImageCropModal();
        };

    }

    ctrl.$inject = ['HueService', '$scope', 'DataService', '$filter', 'PlaceholderDataUrl'];
    return ctrl;

});


text:
@interface PodsDummy_Pods_seanYang_NSDateUtillties : NSObject
@end
@implementation PodsDummy_Pods_seanYang_NSDateUtillties
@end


text:
<?xml version="1.0" encoding="utf-8"?>
<androidx.cardview.widget.CardView xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:card_view="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:layout_gravity="center"
    android:layout_marginBottom="8dp"
    android:layout_marginLeft="8dp"
    card_view:cardCornerRadius="4dp">

    <LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
        android:id="@+id/announcement_layout"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:background="?android:selectableItemBackground"
        android:orientation="vertical">

        <ImageView
            android:id="@+id/image_view"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:adjustViewBounds="true"
            android:maxHeight="175dp"/>

        <TextView
            android:id="@+id/title_view"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_marginLeft="@dimen/text_margin"
            android:layout_marginRight="@dimen/text_margin"
            android:layout_marginTop="@dimen/text_margin"
            android:textAppearance="@style/TextAppearance.AppCompat.Title" />

        <TextView
            android:id="@+id/description_view"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_margin="@dimen/text_margin"
            android:textAppearance="@style/TextAppearance.AppCompat.Small" />

        <LinearLayout
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:layout_marginBottom="@dimen/text_margin"
            android:layout_marginLeft="@dimen/text_margin"
            android:layout_marginRight="@dimen/text_margin"
            android:orientation="horizontal">

            <ImageView
                android:id="@+id/source_view"
                android:layout_width="wrap_content"
                android:layout_height="match_parent"
                android:scaleType="fitStart"/>

            <TextView
                android:id="@+id/date_view"
                android:layout_width="wrap_content"
                android:layout_height="wrap_content"
                android:textAppearance="@style/TextAppearance.AppCompat.Caption" />

            <View
                android:layout_width="0dp"
                android:layout_height="0dp"
                android:layout_weight="1" />

            <TextView
                android:layout_width="wrap_content"
                android:layout_height="wrap_content"
                android:layout_gravity="center"
                android:text="@string/frag_announcement_action_more"
                android:textAppearance="@style/TextAppearance.AppCompat.Caption" />
        </LinearLayout>

    </LinearLayout>
</androidx.cardview.widget.CardView>


text:
/*
 * This file is auto-generated by Qpid Gentools v.0.1 - do not modify.
 * Supported AMQP versions:
 *   8-0
 */
package org.wso2.andes.server.output;

import org.wso2.andes.server.output.ProtocolOutputConverter.Factory;
import org.wso2.andes.server.protocol.AMQProtocolSession;
import org.wso2.andes.framing.ProtocolVersion;

import java.util.Map;
import java.util.HashMap;

public class ProtocolOutputConverterRegistry
{

    private static final Map<ProtocolVersion, Factory> _registry =
            new HashMap<ProtocolVersion, Factory>();

    static
    {
        register(ProtocolVersion.v8_0, org.wso2.andes.server.output.amqp0_8.ProtocolOutputConverterImpl.getInstanceFactory());
        register(ProtocolVersion.v0_9, org.wso2.andes.server.output.amqp0_9.ProtocolOutputConverterImpl.getInstanceFactory());
        register(ProtocolVersion.v0_91, org.wso2.andes.server.output.amqp0_9_1.ProtocolOutputConverterImpl.getInstanceFactory());
    }

    private static void register(ProtocolVersion version, Factory converter)
    {

        _registry.put(version,converter);
    }

    public static ProtocolOutputConverter getConverter(AMQProtocolSession session)
    {
        return _registry.get(session.getProtocolVersion()).newInstance(session);
    }
}


text:
"""

Copyright 2014 Sotera Defense Solutions, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import sys
import math

import mysql.connector
from mysql.connector import errorcode
import tangelo

from datawake.conf import datawakeconfig as dbconfig

"""

Interface to datawake relational database tables (mysql)

"""

#
# Define each datawake_table
#
TABLES = {}

TABLES['datawake_org'] = """
    CREATE TABLE datawake_org (
      email VARCHAR(300),
      org VARCHAR(300)
    )
"""

TABLES['datawake_domains'] = """
    CREATE TABLE datawake_domains (
      name VARCHAR(300),
      description TEXT,
      PRIMARY KEY(name)
    )
"""

TABLES['datawake_selections'] = """
  CREATE TABLE datawake_selections (
  id INT NOT NULL AUTO_INCREMENT,
  postId INT NOT NULL,
  selection TEXT,
  PRIMARY KEY(id),
  FOREIGN KEY(postId) REFERENCES datawake_data(id) ON DELETE CASCADE
  )

"""

TABLES['datawake_data'] = """
    CREATE TABLE datawake_data (
      id INT NOT NULL AUTO_INCREMENT,
      ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
      url TEXT,
      userId TEXT,
      userName TEXT,
      trail VARCHAR(100),
      org VARCHAR(300),
      domain VARCHAR(300),
      PRIMARY KEY(id),
      INDEX(url(30))
    )

"""
TABLES['datawake_trails'] = """
    CREATE TABLE datawake_trails (
      name VARCHAR(100) NOT NULL,
      created TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
      created_by TEXT,
      description TEXT,
      org VARCHAR(300),
      domain VARCHAR(300),
      PRIMARY KEY(name,org,domain)
    )

"""

TABLES['starred_features'] = """
    CREATE TABLE starred_features (
      org VARCHAR(300),
      trail VARCHAR(100) NOT NULL,
      type VARCHAR(100),
      value VARCHAR(1024),
      INDEX(org,trail)
    )
"""

TABLES['datawake_url_rank'] = """
   CREATE TABLE datawake_url_rank (
    id INT NOT NULL AUTO_INCREMENT,
    url TEXT,
    userId TEXT,
    trailname VARCHAR(100),
    rank INT,
    org VARCHAR(300),
    domain VARCHAR(300),
    PRIMARY KEY(id),
	INDEX(url(30),userId(20),trailname)
)
"""

TABLES['datawake_domain_entities'] = """
    CREATE TABLE datawake_domain_entities (
      rowkey varchar(1024),
      INDEX(rowkey(300))
    )

"""

TABLES['general_extractor_web_index'] = """
    CREATE TABLE general_extractor_web_index (
      url varchar(1024),
      entity_type varchar(100),
      entity_value varchar(1024),
      ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
      index(url(300))
    )
"""

TABLES['domain_extractor_web_index'] = """
  CREATE TABLE domain_extractor_web_index (
      domain VARCHAR(300),
      url varchar(1024),
      entity_type varchar(100),
      entity_value varchar(1024),
      ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
      index(domain(300),url(300))
    )
"""

TABLES['domain_extractor_runtimes'] = """
  CREATE TABLE domain_extractor_runtimes (
      domain VARCHAR(300),
      url varchar(1024),
      entity_type varchar(100),
       ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
      index(domain(300),url(300))
    )
"""

#

# ##  GENERIC DB FUNCTIONS ###

#
# Returns a connection to the datawake database
#
def get_cnx():
    user = dbconfig.DATAWAKE_CORE_DB['user']
    db = dbconfig.DATAWAKE_CORE_DB['database']
    pw = dbconfig.DATAWAKE_CORE_DB['password']
    host = dbconfig.DATAWAKE_CORE_DB['host']
    port = dbconfig.DATAWAKE_CORE_DB['port']
    return mysql.connector.connect(user=user, password=pw, host=host, database=db, port=port)

#
# Executes a sql statement and commits it.
# returns the cursor last row id, which may be null
#
def dbCommitSQL(sql, params):
    cnx = get_cnx()
    cursor = cnx.cursor()
    try:
        cursor.execute(sql, params)
        cnx.commit()
        return cursor.lastrowid
    finally:
        cnx.close()
        cursor.close()

#
# Return all rows from a select query
#
def dbGetRows(sql, params):
    cnx = get_cnx()
    cursor = cnx.cursor()
    try:
        cursor.execute(sql, params)
        return cursor.fetchall()
    finally:
        cnx.close()
        cursor.close()

# ###   BROWSE PATH SCRAPE  ###

#
# Add a post to the posts table (datawake_data)
#
def addBrowsePathData(org, url, userId, userName, trail=None, domain='default'):
    org = org.upper()
    sql = " INSERT INTO datawake_data (url,userId,userName,trail,org,domain) VALUES (%s,%s,%s,%s,%s,%s) "
    params = [url, userId, userName, trail, org, domain]
    lastId = dbCommitSQL(sql, params)
    return lastId

def get_post_id(url):
    sql = "select id from datawake_data where url=%s limit 1;"
    params = [url]
    rows = dbGetRows(sql, params)
    if len(rows) == 0:
        return -1
    return rows[0][0]

#
# Get a post from the posts table (datwake_data)
#

def getBrowsePathData(org, id, domain='default'):
    org = org.upper()
    sql = "SELECT id,ts,url,trail,org FROM datawake_data where org = %s AND id = %s AND domain = %s"
    params = [org, id, domain]
    row = dbGetRows(sql, params)[0]
    return {
        'id': row[0],
        'ts': row[1],
        'url': row[2].encode(),
        'trail': row[3],
        'org': row[4]
    }

#
# Get all time stamps from the selected trail,users,org
# returns a dictionary of the form  {'min':0,'max':0,'data':[]}
#
def getTimeWindow(org, users, trail='*', domain='default'):
    sql = 'SELECT unix_timestamp(ts) as ts from datawake_data WHERE org = %s AND domain = %s '
    params = [org.upper(), domain]

    # add where clause for trail and users
    if trail != '*':
        sql = sql + ' AND trail = %s '
        params.append(trail)
    if len(users) > 0:
        param_string = ','.join(['%s' for i in range(len(users))])
        sql = sql + ' AND userName in (' + param_string + ') '
        params.extend(users)

    sql = sql + ' order by ts asc'
    rows = dbGetRows(sql, params)
    data = map(lambda x: x[0], rows)
    if len(data) == 0:
        return {'min': 0, 'max': 0, 'data': []}
    else:
        return {'min': data[0], 'max': data[-1], 'data': data}

#
# Return a list of hourly counts in the form {ts:_,count:_}
#
def getHourlyBrowsePathCounts(org, users, trail, domain='default'):
    sql = 'SELECT (unix_timestamp(ts) DIV 3600)*3600  as group_hour, count(1) from datawake_data where org = %s AND domain = %s '
    params = [org.upper(), domain]
    if trail != '*' and trail != '':
        sql = sql + ' AND trail = %s '
        params.append(trail)
    if len(users) > 0:
        param_string = ','.join(['%s' for i in range(len(users))])
        sql = sql + ' AND userId in (' + param_string + ') '
        params.extend(users)
    sql = sql + " GROUP BY group_hour"

    rows = dbGetRows(sql, params)
    result = []
    delta = 3600
    if len(rows) > 0: curr = rows[0][0]
    for row in rows:
        if row[0] is None: continue
        print 'row ', row
        dt = row[0]
        while (dt - curr > 3600):
            curr = curr + delta
            result.append({'ts': curr, 'count': 0})
        result.append({'ts': dt, 'count': row[1]})
        curr = dt

    # add one hour
    if len(result) > 0:
        curr = curr + 3600
        result.append({'ts': curr, 'count': 0})

    return result

#
# Delete all user activity for a selected user within a time frame and org
#
def deleteUserData(org, user, startdate, enddate, domain='default'):
    sql = """DELETE from datawake_data
              where org = %s AND
              domain = %s AND
              userName = %s AND
              unix_timestamp(ts) >=%s AND
              unix_timestamp(ts) <= %s
           """
    params = [org.upper(), domain, user, startdate, enddate]
    return dbCommitSQL(sql, params)

#
# Associate a text selection with a post Id
# by adding it to the selections table
#
def addSelection(postId, selection):
    sql = " INSERT INTO datawake_selections (postId,selection) VALUES (%s,%s) "
    return dbCommitSQL(sql, [postId, selection])

def getSelections(domain, trail, url, org):
    sql = """SELECT datawake_selections.selection
              FROM datawake_selections
              INNER JOIN datawake_data
              ON datawake_selections.postId=datawake_data.id
              WHERE datawake_data.url = %s
              AND datawake_data.domain = %s
              AND datawake_data.trail = %s
              AND datawake_data.org = %s"""
    params = [url, domain, trail, org]
    rows = dbGetRows(sql, params)
    return map(lambda x: x[0], rows)

# ##  ORGANIZATIONS  ###

#
# Add a link from email address to org
#
def addOrgLink(emailAddress, orgName):
    emailAddress = emailAddress.lower()
    orgName = orgName.upper()
    sql = "INSERT INTO datawake_org (email,org) VALUES (%s,%s)"
    params = [emailAddress, orgName]
    return dbCommitSQL(sql, params)

#
# Delete a link from email address to org
#
def deleteOrgLink(emailAddress, orgName):
    emailAddress = emailAddress.lower()
    orgName = orgName.upper()
    sql = "DELETE FROM datawake_org WHERE email = %s AND org = %s"
    params = [emailAddress, orgName]
    return dbCommitSQL(sql, params)

#
# Return a list or orgs linked to an email
#
def getOrgLinks(emailAddress):
    sql = "SELECT org from datawake_org where email = %s"
    params = [emailAddress.lower()]
    rows = dbGetRows(sql, params)
    return map(lambda x: x[0], rows)

#
# Return users within an org who have been active on at least one trail
# returns a list of dicts, {name:_, id:_}
#
def getActiveUsers(org):
    sql = """select distinct(userId), username from datawake_data where org = %s """
    rows = dbGetRows(sql, [org.upper()])
    result = map(lambda x: {'name': x[1].encode(), 'id': x[0]}, rows)
    return result

# ## TRAILS ###

#
# Ad a new trail to the trails table
#
def addTrail(org, name, description, userId, domain='default'):
    org = org.upper()
    sql = "INSERT INTO datawake_trails (name,description,created_by,org,domain) values (%s,%s,%s,%s,%s)"
    return dbCommitSQL(sql, [name, description, userId, org, domain])

#
# Get a list of all trails name and description
#
def listTrails(org, domain='default'):
    org = org.upper()
    sql = "select name,description,unix_timestamp(created) from datawake_trails where org = %s AND domain = %s ORDER BY created DESC"
    rows = dbGetRows(sql, [org, domain])
    return map(lambda x: {'name': x[0], 'description': x[1], 'created': x[2]}, rows)

#
# Return a list of trail names found in the datawake_data table
#
def get_active_trail_names(org, domain='default'):
    sql = "Select distinct(trail) from datawake_data where org = %s AND domain = %s "
    params = [org.upper(), domain]
    trails = dbGetRows(sql, params)
    trails = map(lambda x: x[0], trails)
    trails = filter(lambda x: x is not None and x != '', trails)
    return trails

#
# Return a mapping of org name to list of active trails
#
def get_all_active_trails(domain='default'):
    sql = "select distinct trail,org from datawake_data where domain = %s "
    rows = dbGetRows(sql, [domain])
    org_trails = {}
    for row in rows:
        (trail, org) = row
        if org not in org_trails:
            org_trails[org] = []
        org_trails[org].append(trail)
    return org_trails

#
# Return list of trails within an org  with counts of user and sze of trail
#
def getTrailsWithUserCounts(org):
    org = org.upper()
    sql = """
        select domain,trail, count(distinct(userId)) as users, count(distinct(id)) as records
        from datawake_data
        where trail is not NULL AND org = %s
        group by domain,trail
       """
    rows = dbGetRows(sql, [org])
    return map(lambda x: {'domain': x[0], 'trail': x[1], 'userCount': x[2], 'records': x[3]}, rows)

#
# Star a feature for a given trail
#
def starFeatureForTrail(org, trail, type, value):
    sql = "INSERT INTO starred_features (org,trail,type,value) VALUES(%s,%s,%s,%s)"
    params = [org, trail, type, value]
    dbCommitSQL(sql, params)

#
# un star a feature for a given trail
#
def unStarFeatureForTrail(org, trail, type, value):
    sql = "DELETE FROM starred_features WHERE org = %s and trail = %s and type = %s and value = %s"
    params = [org, trail, type, value]
    dbCommitSQL(sql, params)

#
# Return list of starred features for a trail
#
def getStarredFeaturesForTrail(org, trail):
    sql = "SELECT type,value FROM starred_features WHERE trail = %s and org = %s"
    params = [trail, org]
    rows = dbGetRows(sql, params)
    if len(rows) == 0:
        return []
    else:
        return map(lambda x: {'type': x[0], 'value': x[1]}, rows)

# ###   URL RANKS   ####

#
# Rank a url
#
def rankUrl(org, userId, trailname, url, rank, domain='default'):
    org = org.upper()
    sql = """SELECT id
            FROM datawake_url_rank
            WHERE org = %s AND
            domain = %s AND
            userId = %s AND
            trailname = %s AND
            url = %s"""
    params = [org, domain, userId, trailname, url]
    rows = dbGetRows(sql, params)

    # update existing row
    if len(rows) > 0:
        id = rows[0][0]
        sql = """UPDATE datawake_url_rank
                 SET rank = %s
                 WHERE id = %s
               """
        dbCommitSQL(sql, [rank, id])

    # insert a new row
    else:
        sql = """ INSERT INTO datawake_url_rank (userId,trailname,url,rank,org,domain)
                  VALUES (%s,%s,%s,%s,%s,%s)
              """
        params = [userId, trailname, url, rank, org, domain]
        dbCommitSQL(sql, params)

#
# Get url rank for a user and trail
# Always returns a rank, defaults to 0
#
def getUrlRank(org, userId, trailname, url, domain='default'):
    org = org.upper()
    sql = """ SELECT rank
              FROM datawake_url_rank
              WHERE org = %s AND domain = %s AND userId = %s AND trailname = %s AND url = %s
          """
    params = [org, domain, userId, trailname, url]
    rows = dbGetRows(sql, params)
    if len(rows) == 0:
        return 0
    else:
        return rows[0][0]

#
# return a dict of url->rank for a single user within a trail and org
#
def getUserUrlRanks(org, userId, trail, domain='default'):
    sql = "SELECT url,rank from datawake_url_rank where org = %s AND domain =%s AND userId = %s AND trailname= %s"
    params = [org.upper(), domain, userId, trail]
    rows = dbGetRows(sql, params)
    ranks = {}
    for row in rows:
        ranks[row[0]] = row[1]
    return ranks

# ###  URL Counts ####

#
# Get the number of times a url has been recorded
# by the data wake
#
def getUrlCount(org, url, domain='default'):
    org = org.upper()
    sql = "SELECT count(1) from datawake_data where url = %s AND org = %s AND domain = %s "
    params = [url, org, domain]
    rows = dbGetRows(sql, params)
    if len(rows) == 0:
        return 0
    else:
        return rows[0][0]

# ### Datawake Domains ####

def get_domains(domain=''):
    sql = "SELECT name,description from datawake_domains"
    params = []
    if domain != '':
        sql = sql + " where domain = %s"
        params.append(domain)
    rows = dbGetRows(sql, params)
    return rows

def add_new_domain(domain, description):
    sql = "INSERT INTO datawake_domains (name,description) values (%s,%s) ON DUPLICATE KEY UPDATE description = %s"
    params = [domain, description, description]
    dbCommitSQL(sql, params)

def remove_domain(domain):
    sql = "DELETE FROM datawake_domains WHERE name = %s"
    params = [domain]
    dbCommitSQL(sql, params)

def domain_exists(name):
    sql = "select count(*) from datawake_domains where name=%s"
    result = dbGetRows(sql, [name])
    rows = map(lambda x: x[0], result)
    return rows[0] == 1

def does_trail_entity_exist(org, domain, trail, entity):
    sql = "select count(*) from trail_based_entities where org=%s and domain=%s and trail=%s and entity=%s"
    return dbGetRows(sql, [org, domain, trail, entity])[0][0] >= 1

def does_irrelevant_entity_exist(org, domain, trail, entity):
    sql = "select count(*) from irrelevant_trail_based_entities where org=%s and domain=%s and trail=%s and entity=%s"
    return dbGetRows(sql, [org, domain, trail, entity])[0][0] >= 1

def add_trail_based_entity(org, domain, trail, entity):
    sql = "insert into trail_based_entities(org, domain, trail, entity, google_result_count) value (%s, %s,%s,%s, %s)"
    return dbCommitSQL(sql, [org, domain, trail, entity, "0"])

def add_irrelevant_trail_entity(org, domain, trail, entity):
    sql = "insert into irrelevant_trail_based_entities(org, domain, trail, entity, google_result_count) value (%s, %s,%s,%s, %s)"
    return dbCommitSQL(sql, [org, domain, trail, entity, "0"])

def get_trail_based_entities(org, domain, trail):
    sql = "select entity,google_result_count from trail_based_entities " \
          "where org=%s and " \
          "domain=%s and " \
          "trail=%s and " \
          "google_result_count != '0' and google_result_count != '1' " \
          "order by google_result_count"
    params = [org, domain, trail]
    return build_dictionary_of_entities(dbGetRows(sql, params))

def build_dictionary_of_entities(rows):
    entities = {}
    for x in rows:
        entities[x[0]] = 1.0 / math.log(float(x[1]), 2)
    return entities

def get_irrelevant_trail_based_entities(org, domain, trail):
    sql = "select entity,google_result_count from irrelevant_trail_based_entities " \
          "where org=%s and " \
          "domain=%s and " \
          "trail=%s and " \
          "google_result_count != '0' and google_result_count != '1' " \
          "order by google_result_count"
    params = [org, domain, trail]
    return build_dictionary_of_entities(dbGetRows(sql, params))

def get_trail_entity_links(org, domain, trail):
    sql = "select url, title, rank from trail_term_rank where org=%s and domain=%s and trail=%s order by rank desc"
    params = [org, domain, trail]
    return map(lambda x: dict(url=x[0].replace(' ',''), title=x[1], rank=x[2]), dbGetRows(sql, params))

def get_visited_trail_entity_links(org, domain, trail):
    sql = "select distinct lower(a.url), a.title, a.rank from trail_term_rank as a " \
          "join datawake_data as b " \
          "on lower(a.url) = lower(b.url) " \
          "where a.org=%s and a.domain=%s and a.trail=%s and a.removed=0 " \
          " and b.org=%s and b.domain=%s and b.trail=%s"
    params = [org, domain, trail, org, domain, trail]
    return map(lambda x: dict(url=x[0], title=x[1], rank=x[2]), dbGetRows(sql, params))

def get_not_visited_trail_entity_links(org, domain, trail):
    sql = "select a.url, a.title, a.rank from trail_term_rank as a " \
          "left join datawake_data as b " \
          "on lower(a.url) = lower(b.url) " \
          "where a.org=%s and a.domain=%s and a.trail=%s and b.url is NULL and a.removed=0 "
    params = [org, domain, trail]
    return map(lambda x: dict(url=x[0], title=x[1], rank=x[2]), dbGetRows(sql, params))

def get_entities_on_url(org, domain, trail, url):
    sql = "select entity, relevant from entities_on_url where org=%s and domain=%s and trail=%s and url=%s"
    params = [org, domain, trail, url]
    return map(lambda x: dict(entity=x[0], relevant=x[1] == 0), dbGetRows(sql, params))

def delete_link_from_trail(org, domain, trail, url):
    sql = "UPDATE trail_term_rank SET removed=1 WHERE org=%s and domain=%s and trail=%s and url=%s"
    return dbCommitSQL(sql, [org, domain, trail, url])

def add_extractor_feedback(org, domain, raw_text, entity_type, entity_value, url):
    sql = "insert into scraping_feedback(org, domain, raw_text, entity_type, entity_value, url) value (%s,%s,%s,%s,%s)"
    return dbCommitSQL(sql, [org, domain, raw_text, entity_type, entity_value, url])

def get_feedback_entities(org, domain, url):
    sql = "select entity_type, entity_value from scraping_feedback where url=%s and domain=%s and org=%s"
    params = [url, domain, org]
    rows = dbGetRows(sql, params)
    return map(lambda x: dict(type=x[0], value=x[1]), rows)

def get_invalid_extracted_entity_count(user_name, entity_type, entity_value, domain, org):
    sql = "select count(*) from invalid_extracted_entity where userName=%s and entity_type=%s and entity_value=%s and domain=%s and org=%s"
    params = [user_name, entity_type, entity_value, domain, org]
    return dbGetRows(sql, params)[0][0]

def mark_invalid_extracted_entity(user_name, entity_type, entity_value, domain, org):
    count = get_invalid_extracted_entity_count(user_name, entity_type, entity_value, domain, org)
    if count == 0:
        params = [user_name, entity_type, entity_value, domain, org]
        sql = "insert into invalid_extracted_entity(userName, entity_type, entity_value, domain, org) value (%s,%s,%s,%s,%s)"
        return dbCommitSQL(sql, params)
    else:
        return -1

def get_marked_entities(org, domain, user_name):
    sql = "select entity_value from invalid_extracted_entity where org=%s and domain=%s and userName=%s"
    params = [org, domain, user_name]
    rows = dbGetRows(sql, params)
    return map(lambda x: dict(value=x[0]), rows)

# ### OLD STUFF, needs cleaned and updated ####

#
# Drop and re-create all tables
#
def drop_and_create_tables():
    cnx = get_cnx()
    cursor = cnx.cursor()
    try:
        for name in [
            'datawake_domain_entities',
            'datawake_domains',
            'datawake_org',
            'datawake_url_rank',
            'datawake_trails',
            'datawake_selections',
            'datawake_data',
            'starred_features',
            'general_extractor_web_index',
            'domain_extractor_web_index',
            'domain_extractor_runtimes'
        ]:
            print 'DROP TABLE ', name
            cursor.execute('DROP TABLE IF EXISTS `' + name + '`')
        for name in [
            'datawake_domain_entities',
            'datawake_domains',
            'datawake_org', 'datawake_data',
            'datawake_url_rank',
            'datawake_trails',
            'datawake_selections',
            'starred_features',
            'general_extractor_web_index',
            'domain_extractor_web_index',
            'domain_extractor_runtimes'
        ]:
            ddl = TABLES[name]
            print("Creating table {}: ".format(name))
            cursor.execute(ddl)
    finally:
        cursor.close()
        cnx.close()

#
# Main function allows to run as a script to drop and create tables.
#
if __name__ == '__main__':
    if sys.argv[1] == 'create-db':
        drop_and_create_tables()
    else:
        print 'usage: python ' + sys.argv[0] + ' create-db (will drop and recreate db)'


text:
package com.linkedin.jpa.service;

import java.util.ArrayList;
import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import com.linkedin.jpa.entity.Company;
import com.linkedin.jpa.repositories.CompanyRepository;

@Service

public class CompanyServiceImp extends EntityServiceImp<Company> implements CompanyService {

	private CompanyRepository companyRepository;

	@Autowired
	public CompanyServiceImp(CompanyRepository companyRepository) {
		this.companyRepository = companyRepository;
	}

	@Override
	public List<Company> listAll() {
		List<Company> companys = new ArrayList<Company>();
		companyRepository.findAll().forEach(companys::add);// fun with Java 8

		return companys;
	}

	@Override
	public Company getById(Long id) {
		return companyRepository.findOne(id);
	}

	@Override
	public Company saveOrUpdate(Company company) {
		return companyRepository.save(company);
	}

	@Override
	public void delete(Long id) {
		companyRepository.delete(id);

	}
}


text:
require_relative 'helper'

describe Taco::List do

  before do
    setup_files
    @list = setup_list
  end

  it "adds the todo to the stack" do
    @list.items.size.must_equal 4
  end

  it 'assigns a default tag unless provided' do
    @list.items.first.context.must_equal '@'
  end

  it "creates a hash of attributes from the todo items" do
    Taco.storage.to_hash.must_equal({
        :"@" => ["Take the dog for a walk"],
        :@work => ["Pay lease bill"],
        :@home => ["Buy Duck Typing from RubyRags", "Buy Ruby Nerd from RubyRags"]
      })
  end

  it 'deletes a todo' do
    @list.delete(2).text.must_equal "Pay lease bill"
    @list.items.size.must_equal 3
  end

  it 'completes a todo' do
    @list.done(2).context.must_equal "@done"
    @list.done(3).context.must_equal "@done"
  end

  it 'should move the specific item to the top' do
    @list.bump(3)
    @list.items.first.text.must_equal 'Buy Duck Typing from RubyRags'
  end

end


text:
#ifndef _CW_TABLE_VIEW_H_
#define _CW_TABLE_VIEW_H_

#include <set>
#include <vector>
#include <tuple>
#include "ui/UIScrollView.h"

namespace cw {
    /**
     * Abstract class for SWTableView cell node
     */
    class TableViewCell : public cocos2d::ui::Widget {
    public:
        CREATE_FUNC(TableViewCell);

        TableViewCell() {}
        /**
         * The index used internally by SWTableView and its subclasses
         */
        ssize_t getIdx() const { return _idx; }
        void setIdx(ssize_t uIdx) { _idx = uIdx; }
        /**
         * Cleans up any resources linked to this cell and resets idx property.
         */
        void reset() { _idx = cocos2d::CC_INVALID_INDEX; }

    private:
        ssize_t _idx;
    };

    template <class ..._ARGS> class TableViewCellEx : public TableViewCell {
    public:
        static TableViewCellEx<_ARGS...> *create() {
            TableViewCellEx<_ARGS...> *ret = new (std::nothrow) TableViewCellEx<_ARGS...>();
            if (ret != nullptr && ret->init()) {
                ret->autorelease();
                return ret;
            }
            CC_SAFE_DELETE(ret);
            return nullptr;
        }

        typedef std::tuple<_ARGS...> ExtDataType;

        std::tuple<_ARGS...> &getExtData() {
            return _extData;
        }

        const std::tuple<_ARGS...> &getExtData() const {
            return _extData;
        }

    protected:
        std::tuple<_ARGS...> _extData;
    };

    class TableView;

    /**
     * Data source that governs table backend data.
     */
    class TableViewDelegate {
    public:
        /**
         * @js NA
         * @lua NA
         */
        virtual ~TableViewDelegate() { }

        /**
         * Returns number of cells in a given table view.
         *
         * @return number of cells
         */
        virtual ssize_t numberOfCellsInTableView(TableView *table) = 0;

        /**
         * cell size for a given index
         *
         * @param idx the index of a cell to get a size
         * @return size of a cell at given index, height of a cell if the table view is VERTICAL or width if HORIZONTAL

         */
        virtual float tableCellSizeForIndex(TableView *table, ssize_t idx) {
            CC_UNUSED_PARAM(table);
            CC_UNUSED_PARAM(idx);
            return 0.0f;
        }

        /**
         * a cell instance at a given index
         *
         * @param idx index to search for a cell
         * @return cell found at idx
         */
        virtual TableViewCell* tableCellAtIndex(TableView *table, ssize_t idx) = 0;

        /**
         * Delegate called when the cell is about to be recycled. Immediately
         * after this call the cell will be removed from the scene graph and
         * recycled.
         *
         * @param table table contains the given cell
         * @param cell  cell that is pressed
         * @js NA
         * @lua NA
         */
        virtual void tableCellWillRecycle(TableView *table, TableViewCell *cell) {
            CC_UNUSED_PARAM(table);
            CC_UNUSED_PARAM(cell);
        }
    };

    class TableView : public cocos2d::ui::ScrollView {
        DECLARE_CLASS_GUI_INFO

    public:
        enum class VerticalFillOrder {
            TOP_DOWN,
            BOTTOM_UP
        };

        /**
         * Default constructor
         */
        TableView();

        /**
         * Default destructor
         */
        virtual ~TableView();

        /**
         * Allocates and initializes.
         */
        static TableView *create();
        virtual bool init() override;

        /**
         * delegate
         */
        TableViewDelegate *getDelegate() { return _delegate; }
        void setDelegate(TableViewDelegate *pDelegate) { _delegate = pDelegate; }

        /**
         * determines how cell is ordered and filled in the view.
         */
        void setVerticalFillOrder(VerticalFillOrder order);
        VerticalFillOrder getVerticalFillOrder();

        /**
         * Updates the content of the cell at a given index.
         *
         * @param idx index to find a cell
         */
        void updateCellAtIndex(ssize_t idx);
        /**
         * Inserts a new cell at a given index
         *
         * @param idx location to insert
         */
        void insertCellAtIndex(ssize_t idx);
        /**
         * Removes a cell at a given index
         *
         * @param idx index to find a cell
         */
        void removeCellAtIndex(ssize_t idx);
        /**
         * reloads data from data source.  the view will be refreshed.
         */
        void reloadData();
        void reloadDataInplacement();

        /**
         * Dequeues a free cell if available. nil if not.
         *
         * @return free cell
         */
        TableViewCell *dequeueCell();

        /**
         * Returns an existing cell at a given index. Returns nil if a cell is nonexistent at the moment of query.
         *
         * @param idx index
         * @return a cell at a given index
         */
        TableViewCell *cellAtIndex(ssize_t idx);

        virtual void setDirection(cocos2d::ui::ScrollView::Direction dir) override;

        cocos2d::Vec2 maxContainerOffset();
        cocos2d::Vec2 minContainerOffset();

        /**
         * @brief Jump to specific cell
         * @param cellIndex Specifies the cell's index
         */
        void jumpToCell(ssize_t cellIndex);

        /**
         * @brief Scroll to specific cell
         * @param cellIndex Specifies the cell's index
         * @param timeInSec Scroll time
         * @param attenuated Whether scroll speed attenuate or not
         */
        void scrollToCell(ssize_t cellIndex, float timeInSec, bool attenuated);

    protected:
        virtual void onSizeChanged() override;

        virtual void moveInnerContainer(const cocos2d::Vec2 &deltaMove, bool canStartBounceBack) override;

        ssize_t __indexFromOffset(const cocos2d::Vec2 &offset, ssize_t cellsCount);
        ssize_t _indexFromOffset(cocos2d::Vec2 offset, ssize_t cellsCount);
        cocos2d::Vec2 __offsetFromIndex(ssize_t index);
        cocos2d::Vec2 _offsetFromIndex(ssize_t index);

        void _moveCellOutOfSight(TableViewCell *cell);
        void _setIndexForCell(ssize_t index, TableViewCell *cell);
        void _addCellIfNecessary(TableViewCell *cell);

        void _updateCellPositions(ssize_t cellsCount);
        void _scrollViewDidScroll(ssize_t cellsCount);

        cocos2d::Vec2 _destinationFromIndex(ssize_t index);

        /**
         * vertical direction of cell filling
         */
        VerticalFillOrder _vordering;

        /**
         * index set to query the indexes of the cells used.
         */
        std::set<ssize_t> _indices;

        /**
         * vector with all cell positions
         */
        std::vector<float> _cellsPositions;
        //NSMutableIndexSet *indices_;
        /**
         * cells that are currently in the table
         */
        cocos2d::Vector<TableViewCell *> _cellsUsed;
        /**
         * free list of cells
         */
        cocos2d::Vector<TableViewCell *> _cellsFreed;

        TableViewDelegate *_delegate;

        bool _isUsedCellsDirty;

        void _updateInnerContainerSize(ssize_t cellsCount);

    public:
        void updateInnerContainerSize();

    private:
        using cocos2d::ui::ScrollView::setInnerContainerSize;
        using cocos2d::ui::ScrollView::addChild;
        using cocos2d::ui::ScrollView::removeAllChildren;
        using cocos2d::ui::ScrollView::removeAllChildrenWithCleanup;
        using cocos2d::ui::ScrollView::removeChild;

        using cocos2d::ui::ScrollView::addEventListener;
    };
}

#endif


text:
define([],
    function() {
        return {

            /**
             * Asserts that two arrays are equal deeply
             * @param {Array} arr The first array to compare
             * @param {Array} arr2 The second array to compare
             */
            arrayEquals: function(arr, arr2) {
                assert.equal(arr.length, arr2.length, "Array lengths differ");
                arr.forEach(function (item, i) {
                    assert.equal(item, arr2[i]);
                });
            },

            /**
             * Asserts that when the given function is run, it throws an exception with a message containing the given message
             * @param {String} message The expected message
             * @param {Function} exFunc The function that should thrown an exception
             */
            assertException: function(message, exFunc) {
                try {
                    exFunc();
                    assert.fail("Exception \"" + message + "\" not thrown!");
                } catch(e) {
                    assert.equal(e.message, message);
                }
            },

            /**
             * Provides a very basic clone method
             * @param {Object} e The object to clone
             */
            clone: function(e) {
                return JSON.parse(JSON.stringify(e));
            }
        }
    });


text:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>xyy.java.note</groupId>
    <artifactId>logging</artifactId>
    <version>1.0-SNAPSHOT</version>
    <dependencies>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.7.22</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-simple -->
        <!--<dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>1.7.24</version>
        </dependency>-->

        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-jdk14 -->
        <!--<dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-jdk14</artifactId>
            <version>1.7.24</version>
        </dependency>-->

        <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-slf4j-impl -->
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-slf4j-impl</artifactId>
            <version>2.8</version>
        </dependency>

    </dependencies>

</project>


text:
package utils.secrets

import com.google.inject.AbstractModule
import com.mohiva.play.silhouette.api.Environment
import com.typesafe.config.ConfigFactory
import models.daos.UserDAO
import net.codingwell.scalaguice.ScalaModule
import org.specs2.mock.Mockito
import org.specs2.specification.Scope
import play.api.{ Application, Configuration }
import play.api.inject.guice.GuiceApplicationBuilder
import play.api.test.{ PlaySpecification, WithApplication }
import play.modules.reactivemongo.ReactiveMongoApi
import services.FakeReactiveMongoApi
import utils.auth.DefaultEnv

trait BiscuitSecretsManagerSpecTestContext extends Scope {

  class FakeModule extends AbstractModule with ScalaModule {
    def configure(): Unit = {
      bind[ReactiveMongoApi].toInstance(new FakeReactiveMongoApi)
    }
  }

  lazy val application: Application = GuiceApplicationBuilder()
    .configure(Configuration(ConfigFactory.load("application.test.conf")))
    .overrides(new FakeModule)
    .build()
}

class BiscuitSecretsManagerSpec extends PlaySpecification with Mockito {

  "The BiscuitSecretsManager" should {
    "return unencrypted secret" in new BiscuitSecretsManagerSpecTestContext {
      new WithApplication(application) {
        val secretsManager: SecretsManager = app.injector.instanceOf[SecretsManager]

        val secret = secretsManager.getSecretUtf8("test::secret")

        secret must be equalTo "stuff"
      }
    }
  }
}


text:
namespace De.Osthus.Ambeth.Util
{
    public interface Runnable
    {
        void Run();
    }
}


text:
..  Copyright 2016 Sebastian Semper, Christoph Wagner
        https://www.tu-ilmenau.de/it-ems/

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.

.. _strides:

Low-Overhead Array Striding Interface
=====================================

Fastmat offers a special C-level interface allowing the creation, processing
and manipulation of views into the underlying data of :py:class:`numpy.ndarray`
objects without the overhead of creating view or memoryview objects of that
array object. As the implementation is based on C structures, no interaction
with the python object interface is necessary, thus increasing the efficiency
of advanced linear operators from within cython code. By mimimizing memory
operations occuring during view creation, structure- or object allocation or
copying, this helps minimizing the already low overhead on using cython
memoryviews further.

The main container for defining and using strides is the ``STRIDE_s`` structure:

.. code-block:: cython

    ctypedef struct STRIDE_s:
        char *          base
        intsize         strideElement
        intsize         strideVector
        intsize         numElements
        intsize         numVectors
        np.uint8_t      sizeItem
        ftype           dtype

:ref:`fastmat Type Identifier<ftype>`

The striding interface supports:
  * Two-dimensional :py:class:`numpy.ndarray` objects
  * Non-contiguous (striding) access into the data
  * Modifying views (substriding

``fastmat.core.strides``
------------------------

.. automodule:: fastmat.core.strides
    :members:
    :undoc-members:
    :show-inheritance:


text:
#ifndef MICROPY_INCLUDED_EXTMOD_MACHINE_SPI_H
#define MICROPY_INCLUDED_EXTMOD_MACHINE_SPI_H

#include "py/obj.h"
#include "py/mphal.h"

// SPI protocol
typedef struct _mp_machine_spi_p_t {
    void (*transfer)(mp_obj_base_t *obj, size_t len, const uint8_t *src, uint8_t *dest);
} mp_machine_spi_p_t;

typedef struct _mp_machine_soft_spi_obj_t {
    mp_obj_base_t base;
    uint32_t delay_half; // microsecond delay for half SCK period
    uint8_t polarity;
    uint8_t phase;
    mp_hal_pin_obj_t sck;
    mp_hal_pin_obj_t mosi;
    mp_hal_pin_obj_t miso;
} mp_machine_soft_spi_obj_t;

void mp_machine_soft_spi_transfer(mp_obj_base_t *self, size_t len, const uint8_t *src, uint8_t *dest);

MP_DECLARE_CONST_FUN_OBJ(mp_machine_spi_read_obj);
MP_DECLARE_CONST_FUN_OBJ(mp_machine_spi_readinto_obj);
MP_DECLARE_CONST_FUN_OBJ(mp_machine_spi_write_obj);
MP_DECLARE_CONST_FUN_OBJ(mp_machine_spi_write_readinto_obj);

#endif // MICROPY_INCLUDED_EXTMOD_MACHINE_SPI_H


text:
include $(srctree)/drivers/misc/mediatek/Makefile.custom
ccflags-y += -I$(srctree)/drivers/misc/mediatek/cmdq/$(MTK_PLATFORM) \
             -I$(srctree)/drivers/misc/mediatek/dispsys/$(MTK_PLATFORM) \
             -I$(srctree)/drivers/misc/mediatek/smi/$(MTK_PLATFORM) \
             -I$(srctree)/drivers/misc/mediatek/mach/$(MTK_PLATFORM)/include

ifeq ($(CONFIG_ARCH_MT6735),y)
ccflags-y += -I$(srctree)/drivers/misc/mediatek/dispsys/$(MTK_PLATFORM)/mt6735/
ccflags-y += -I$(srctree)/drivers/misc/mediatek/cmdq/$(MTK_PLATFORM)/mt6735/
endif
ifeq ($(CONFIG_ARCH_MT6735M),y)
ccflags-y += -I$(srctree)/drivers/misc/mediatek/dispsys/$(MTK_PLATFORM)/mt6735m/
ccflags-y += -I$(srctree)/drivers/misc/mediatek/cmdq/$(MTK_PLATFORM)/mt6735m/
endif
ifeq ($(CONFIG_ARCH_MT6753),y)
ccflags-y += -I$(srctree)/drivers/misc/mediatek/dispsys/$(MTK_PLATFORM)/mt6753/
ccflags-y += -I$(srctree)/drivers/misc/mediatek/cmdq/$(MTK_PLATFORM)/mt6753/
endif

# Enable MET
ccflags-y += -DMET_USER_EVENT_SUPPORT

ifneq ($(TARGET_BUILD_VARIANT),eng)
ccflags-y += -D_MTK_USER_
endif

# Common Clock Framework
ifneq ($(CONFIG_MTK_LEGACY),y)
ifeq ($(CONFIG_COMMON_CLK),y)
ccflags-y += -DCMDQ_USE_CCF
endif
endif

# secure path
ifeq ($(strip $(CONFIG_MTK_SEC_VIDEO_PATH_SUPPORT)),y)
ifeq ($(strip $(CONFIG_TRUSTONIC_TEE_SUPPORT)),y)
ccflags-y += -I$(srctree)/drivers/misc/mediatek/gud/$(MTK_PLATFORM)/gud/MobiCoreKernelApi/public
ccflags-y += -I$(srctree)/drivers/misc/mediatek/gud/$(MTK_PLATFORM)/gud/MobiCoreKernelApi/include

ccflags-y += -I$(srctree)/drivers/misc/mediatek/mach/$(MTK_PLATFORM)/include/trustzone/cmdq/cmdq_sec_dr \
             -I$(srctree)/drivers/misc/mediatek/mach/$(MTK_PLATFORM)/include/trustzone/cmdq/cmdq_sec_tl
endif
endif

ifneq ($(MTK_PLATFORM),mt8163)
# driver module
obj-y += cmdq_record.o
obj-y += cmdq_core.o
obj-y += cmdq_driver.o
obj-y += cmdq_mmp.o
obj-y += cmdq_mdp_common.o
obj-y += cmdq_prof.o
obj-y += cmdq_test.o
obj-y += cmdq_sec.o
obj-y += cmdq_device_common.o
endif

obj-y += $(subst ",,$(CONFIG_MTK_PLATFORM))/


text:
module("data-steamer tests");
test("basic", function() {
    ok(true, "ok test is true");
    //DOCS: http://api.qunitjs.com/ok/
   equal(1, true, "equal test with 1");
   notEqual(0, true, "not equal test with 0");
});


text:
const FILE_USER_POOLS = "user-pools";
const NAME_CLIENT = "example-client";

export { FILE_USER_POOLS, NAME_CLIENT };