text:
#ifndef TENSORFLOW_CORE_KERNELS_DATA_DATASET_TEST_BASE_H_
#define TENSORFLOW_CORE_KERNELS_DATA_DATASET_TEST_BASE_H_

#include <stddef.h>

#include <functional>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "absl/strings/string_view.h"
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/common_runtime/graph_constructor.h"
#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/cancellation.h"
#include "tensorflow/core/framework/dataset.h"
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/framework/function_handle_cache.h"
#include "tensorflow/core/framework/function_testlib.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/data/name_utils.h"
#include "tensorflow/core/lib/core/status_test_util.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/lib/io/zlib_compression_options.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/refcount.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/platform/threadpool.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/tensor_slice_reader_cache.h"

namespace tensorflow {
namespace data {

typedef std::vector<
    std::pair<string, tensorflow::FunctionDefHelper::AttrValueWrapper>>
    AttributeVector;

constexpr int kDefaultCPUNum = 2;
constexpr int kDefaultThreadNum = 2;

// Creates a tensor with the specified dtype, shape, and value.
template <typename T>
static Tensor CreateTensor(const TensorShape& input_shape,
                           const gtl::ArraySlice<T>& input_data) {
  Tensor tensor(DataTypeToEnum<T>::value, input_shape);
  test::FillValues<T>(&tensor, input_data);
  return tensor;
}

// Creates a vector of tensors with the specified dtype, shape, and values.
template <typename T>
std::vector<Tensor> CreateTensors(
    const TensorShape& shape, const std::vector<gtl::ArraySlice<T>>& values) {
  std::vector<Tensor> result;
  result.reserve(values.size());
  for (auto& value : values) {
    result.emplace_back(CreateTensor<T>(shape, value));
  }
  return result;
}

enum class CompressionType { ZLIB = 0, GZIP = 1, RAW = 2, UNCOMPRESSED = 3 };

// Returns a string representation for the given compression type.
string ToString(CompressionType compression_type);

// Gets the specified zlib compression options according to the compression
// type. Note that `CompressionType::UNCOMPRESSED` is not supported because
// `ZlibCompressionOptions` does not have an option.
io::ZlibCompressionOptions GetZlibCompressionOptions(
    CompressionType compression_type);

// Used to specify parameters when writing data into files with compression.
// `input_buffer_size` and `output_buffer_size` specify the input and output
// buffer size when ZLIB and GZIP compression is used.
struct CompressionParams {
  CompressionType compression_type = CompressionType::UNCOMPRESSED;
  int32 input_buffer_size = 0;
  int32 output_buffer_size = 0;
};

// Writes the input data into the file without compression.
Status WriteDataToFile(const string& filename, const char* data);

// Writes the input data into the file with the specified compression.
Status WriteDataToFile(const string& filename, const char* data,
                       const CompressionParams& params);

// Writes the input data into the TFRecord file with the specified compression.
Status WriteDataToTFRecordFile(const string& filename,
                               const std::vector<absl::string_view>& records,
                               const CompressionParams& params);

// Provides the parameters for running the dataset op.
class DatasetParams {
 public:
  DatasetParams(DataTypeVector output_dtypes,
                std::vector<PartialTensorShape> output_shapes,
                string node_name);

  virtual ~DatasetParams() {}

  // Returns the inputs (except the input datasets) as a tensor vector.
  virtual std::vector<Tensor> GetInputTensors() const = 0;

  // Returns the dataset input names as a string vector.
  virtual Status GetInputNames(std::vector<string>* input_names) const = 0;

  // Returns the dataset attributes as a vector.
  virtual Status GetAttributes(AttributeVector* attributes) const = 0;

  // Checks if the tensor is a dataset variant tensor.
  static bool IsDatasetTensor(const Tensor& tensor);

  string node_name() const { return node_name_; }

  DataTypeVector output_dtypes() const { return output_dtypes_; }

  std::vector<PartialTensorShape> output_shapes() const {
    return output_shapes_;
  }

  string iterator_prefix() const { return iterator_prefix_; }

  const std::vector<std::shared_ptr<DatasetParams>>& input_dataset_params()
      const {
    return input_dataset_params_;
  }

  // Returns the functions that will be used when running the dataset op.
  virtual std::vector<FunctionDef> func_lib() const { return {}; }

  // Returns the dataset type for the op represented by these parameters. This
  // type usually needs to match the constant called `kDatasetType` defined in
  // the dataset kernel.
  virtual string dataset_type() const = 0;

  virtual int op_version() const { return op_version_; }

 protected:
  std::vector<std::shared_ptr<DatasetParams>> input_dataset_params_;
  DataTypeVector output_dtypes_;
  std::vector<PartialTensorShape> output_shapes_;
  string node_name_;
  string iterator_prefix_ = "Iterator";
  int op_version_ = 1;
};

// `RangeDatasetParams` is a common dataset parameter type that are used in
// testing.
class RangeDatasetParams : public DatasetParams {
 public:
  RangeDatasetParams(int64 start, int64 stop, int64 step,
                     DataTypeVector output_dtypes,
                     std::vector<PartialTensorShape> output_shapes,
                     string node_name);

  RangeDatasetParams(int64 start, int64 stop, int64 step);

  RangeDatasetParams(int64 start, int64 stop, int64 step,
                     DataTypeVector output_dtypes);

  std::vector<Tensor> GetInputTensors() const override;

  Status GetInputNames(std::vector<string>* input_names) const override;

  Status GetAttributes(AttributeVector* attr_vector) const override;

  string dataset_type() const override;

 private:
  int64 start_;
  int64 stop_;
  int64 step_;
};

// `BatchDatasetParams` is a common dataset parameter type that are used in
// testing.
class BatchDatasetParams : public DatasetParams {
 public:
  template <typename T>
  BatchDatasetParams(T input_dataset_params, int64 batch_size,
                     bool drop_remainder, bool parallel_copy,
                     DataTypeVector output_dtypes,
                     std::vector<PartialTensorShape> output_shapes,
                     string node_name)
      : DatasetParams(std::move(output_dtypes), std::move(output_shapes),
                      std::move(node_name)),
        batch_size_(batch_size),
        drop_remainder_(drop_remainder),
        parallel_copy_(parallel_copy) {
    input_dataset_params_.push_back(std::make_unique<T>(input_dataset_params));
    op_version_ = 2;
    iterator_prefix_ =
        name_utils::IteratorPrefix(input_dataset_params.dataset_type(),
                                   input_dataset_params.iterator_prefix());
  }

  std::vector<Tensor> GetInputTensors() const override;

  Status GetInputNames(std::vector<string>* input_names) const override;

  Status GetAttributes(AttributeVector* attr_vector) const override;

  string dataset_type() const override;

 private:
  int64 batch_size_;
  bool drop_remainder_;
  bool parallel_copy_;
};

// `MapDatasetParams` is a common dataset parameter type that are used in
// testing.
class MapDatasetParams : public DatasetParams {
 public:
  template <typename T>
  MapDatasetParams(T input_dataset_params, std::vector<Tensor> other_arguments,
                   FunctionDefHelper::AttrValueWrapper func,
                   std::vector<FunctionDef> func_lib,
                   DataTypeVector type_arguments, DataTypeVector output_dtypes,
                   std::vector<PartialTensorShape> output_shapes,
                   bool use_inter_op_parallelism, bool preserve_cardinality,
                   string node_name)
      : DatasetParams(std::move(output_dtypes), std::move(output_shapes),
                      std::move(node_name)),
        other_arguments_(std::move(other_arguments)),
        func_(std::move(func)),
        func_lib_(std::move(func_lib)),
        type_arguments_(std::move(type_arguments)),
        use_inter_op_parallelism_(use_inter_op_parallelism),
        preserve_cardinality_(preserve_cardinality) {
    input_dataset_params_.push_back(absl::make_unique<T>(input_dataset_params));
    iterator_prefix_ =
        name_utils::IteratorPrefix(input_dataset_params.dataset_type(),
                                   input_dataset_params.iterator_prefix());
  }

  std::vector<Tensor> GetInputTensors() const override;

  Status GetInputNames(std::vector<string>* input_names) const override;

  Status GetAttributes(AttributeVector* attr_vector) const override;

  string dataset_type() const override;

  std::vector<FunctionDef> func_lib() const override;

 private:
  std::vector<Tensor> other_arguments_;
  FunctionDefHelper::AttrValueWrapper func_;
  std::vector<FunctionDef> func_lib_;
  DataTypeVector type_arguments_;
  bool use_inter_op_parallelism_;
  bool preserve_cardinality_;
};

// `TensorSliceDatasetParams` is a common dataset parameter type that are used
// in testing.
class TensorSliceDatasetParams : public DatasetParams {
 public:
  TensorSliceDatasetParams(std::vector<Tensor> components, string node_name);

  std::vector<Tensor> GetInputTensors() const override;

  Status GetInputNames(std::vector<string>* input_names) const override;

  Status GetAttributes(AttributeVector* attr_vector) const override;

  string dataset_type() const override;

  int64 num_slices() const { return components_[0].dim_size(0); }

  size_t num_tensors_per_slice() const { return components_.size(); }

 private:
  DataTypeVector TensorSliceDtypes(const std::vector<Tensor>& input_components);

  std::vector<PartialTensorShape> TensorSliceShapes(
      const std::vector<Tensor>& input_components);

 public:
  std::vector<Tensor> components_;
};

// `TakeDatasetParams` is a common dataset parameter type that are used in
// testing.
class TakeDatasetParams : public DatasetParams {
 public:
  template <typename T>
  TakeDatasetParams(T input_dataset_params, int count,
                    DataTypeVector output_dtypes,
                    std::vector<PartialTensorShape> output_shapes,
                    string node_name)
      : DatasetParams(std::move(output_dtypes), std::move(output_shapes),
                      std::move(node_name)),
        count_(count) {
    input_dataset_params_.push_back(absl::make_unique<T>(input_dataset_params));
    iterator_prefix_ =
        name_utils::IteratorPrefix(input_dataset_params.dataset_type(),
                                   input_dataset_params.iterator_prefix());
  }

  std::vector<Tensor> GetInputTensors() const override;

  Status GetInputNames(std::vector<string>* input_names) const override;

  Status GetAttributes(AttributeVector* attr_vector) const override;

  string dataset_type() const override;

 private:
  int64 count_;
};

// `ConcatenateDatasetParams` is a common dataset parameter type that are used
// in testing.
class ConcatenateDatasetParams : public DatasetParams {
 public:
  template <typename T, typename P>
  ConcatenateDatasetParams(T input_dataset_params_0, P input_dataset_params_1,
                           DataTypeVector output_dtypes,
                           std::vector<PartialTensorShape> output_shapes,
                           string node_name)
      : DatasetParams(std::move(output_dtypes), std::move(output_shapes),
                      std::move(node_name)) {
    input_dataset_params_.push_back(
        absl::make_unique<T>(input_dataset_params_0));
    input_dataset_params_.push_back(
        absl::make_unique<T>(input_dataset_params_1));
    iterator_prefix_ =
        name_utils::IteratorPrefix(input_dataset_params_0.dataset_type(),
                                   input_dataset_params_0.iterator_prefix());
  }

  std::vector<Tensor> GetInputTensors() const override;

  Status GetInputNames(std::vector<string>* input_names) const override;

  Status GetAttributes(AttributeVector* attr_vector) const override;

  string dataset_type() const override;
};

template <typename T>
struct GetNextTestCase {
  GetNextTestCase(T dataset_params, std::vector<Tensor> expected_outputs,
                  bool compare_order = true)
      : dataset_params(std::move(dataset_params)),
        expected_outputs(std::move(expected_outputs)),
        compare_order(compare_order) {}

  T dataset_params;
  std::vector<Tensor> expected_outputs;
  bool compare_order;
};

template <typename T>
struct SkipTestCase {
  SkipTestCase(T dataset_params, int num_to_skip, int expected_num_skipped,
               bool get_next = false, std::vector<Tensor> expected_outputs = {},
               bool compare_order = true)
      : dataset_params(std::move(dataset_params)),
        num_to_skip(num_to_skip),
        expected_num_skipped(expected_num_skipped),
        get_next(get_next),
        expected_outputs(std::move(expected_outputs)),
        compare_order(compare_order) {}

  T dataset_params;
  int num_to_skip;
  int expected_num_skipped;
  bool get_next;
  std::vector<Tensor> expected_outputs;
  bool compare_order;
};

template <typename T>
struct DatasetNodeNameTestCase {
  T dataset_params;
  string expected_node_name;
};

template <typename T>
struct DatasetTypeStringTestCase {
  T dataset_params;
  string expected_dataset_type_string;
};

template <typename T>
struct DatasetOutputDtypesTestCase {
  T dataset_params;
  DataTypeVector expected_output_dtypes;
};

template <typename T>
struct DatasetOutputShapesTestCase {
  T dataset_params;
  std::vector<PartialTensorShape> expected_output_shapes;
};

template <typename T>
struct CardinalityTestCase {
  T dataset_params;
  int64 expected_cardinality;
};

template <typename T>
struct DatasetSaveTestCase {
  T dataset_params;
};

template <typename T>
struct IteratorOutputDtypesTestCase {
  T dataset_params;
  DataTypeVector expected_output_dtypes;
};

template <typename T>
struct IteratorOutputShapesTestCase {
  T dataset_params;
  std::vector<PartialTensorShape> expected_output_shapes;
};

template <typename T>
struct IteratorPrefixTestCase {
  T dataset_params;
  string expected_iterator_prefix;
};

template <typename T>
struct IteratorSaveAndRestoreTestCase {
  IteratorSaveAndRestoreTestCase(T dataset_params, std::vector<int> breakpoints,
                                 std::vector<Tensor> expected_outputs,
                                 bool compare_order = true)
      : dataset_params(std::move(dataset_params)),
        breakpoints(std::move(breakpoints)),
        expected_outputs(std::move(expected_outputs)),
        compare_order(compare_order) {}

  T dataset_params;
  std::vector<int> breakpoints;
  std::vector<Tensor> expected_outputs;
  bool compare_order;
};

// Class composing a dataset with its dependencies.
class TestDataset {
 public:
  // TestDataset expects that the caller has Ref'd the wrapped dataset. When
  // TestDataset is destroyed, it will Unref the dataset.
  TestDataset(std::unique_ptr<OpKernel> kernel_,
              std::unique_ptr<OpKernelContext::Params> ctx_params,
              std::unique_ptr<OpKernelContext> ctx,
              std::vector<std::unique_ptr<Tensor>> input_tensors,
              DatasetBase* dataset)
      : kernel_(std::move(kernel_)),
        ctx_params_(std::move(ctx_params)),
        ctx_(std::move(ctx)),
        input_tensors_(std::move(input_tensors)),
        dataset_(dataset),
        scoped_unref_(dataset) {}

  DatasetBase* dataset() const { return dataset_; }

  OpKernelContext* op_kernel_context() const { return ctx_.get(); }

 protected:
  std::unique_ptr<OpKernel> kernel_;
  std::unique_ptr<OpKernelContext::Params> ctx_params_;
  std::unique_ptr<OpKernelContext> ctx_;
  // The input tensors that this dataset depends on. They must outlive the
  // dataset.
  std::vector<std::unique_ptr<Tensor>> input_tensors_;
  DatasetBase* dataset_;
  core::ScopedUnref scoped_unref_;
};

// Class composing a dataset iterator with its dependencies.
class TestIterator {
 public:
  TestIterator(std::unique_ptr<IteratorContext> ctx,
               std::unique_ptr<IteratorBase> iterator)
      : iterator_(std::move(iterator)), ctx_(std::move(ctx)) {}

  IteratorBase* iterator() const { return iterator_.get(); }

  IteratorContext* ctx() const { return ctx_.get(); }

  Status GetNext(std::vector<Tensor>* out_tensors, bool* end_of_sequence) {
    return iterator_->GetNext(ctx(), out_tensors, end_of_sequence);
  }

 protected:
  std::unique_ptr<IteratorBase> iterator_;
  std::unique_ptr<IteratorContext> ctx_;
};

// Helpful functions to test Dataset op kernels.
class DatasetOpsTestBase : public ::testing::Test {
 public:
  DatasetOpsTestBase();

  // Initializes the runtime and creates a dataset and iterator.
  Status Initialize(const DatasetParams& dataset_params);

  // Initializes the parts of the runtime needed to run dataset ops.
  Status InitializeRuntime(const DatasetParams& dataset_params);

  // Creates a dataset.
  Status MakeDataset(const DatasetParams& dataset_params,
                     std::unique_ptr<TestDataset>* dataset);

  // Creates an iterator for the given dataset, using the specified split
  // provider.
  Status MakeIterator(const DatasetParams& dataset_params,
                      const TestDataset& dataset,
                      std::unique_ptr<SplitProvider> split_provider,
                      std::unique_ptr<TestIterator>* iterator);
  // Creates an iterator for the given dataset.
  Status MakeIterator(const DatasetParams& dataset_params,
                      const TestDataset& dataset,
                      std::unique_ptr<TestIterator>* iterator);

  // Runs the dataset operation according to the predefined dataset params and
  // produces outputs. Different from `MakeDataset()` which returns a Dataset
  // object, `RunDatasetOp()` executes the dataset kernel based on the input
  // DatasetParams and returns the produced outputs as a tensor vector. It can
  // be used to run some dataset operations that do not have an internal
  // customized `Dataset` class (e.g. `ReduceDatasetOp`).
  Status RunDatasetOp(const DatasetParams& dataset_params,
                      std::vector<Tensor>* outputs);

  // The method validates whether the two tensors have the same shape, dtype,
  // and value.
  static Status ExpectEqual(const Tensor& a, const Tensor& b);

  // The method validates whether the two tensor vectors have the same tensors.
  // If `compare_order` is false, the method will only evaluate whether the two
  // vectors have the same elements regardless of order.
  static Status ExpectEqual(std::vector<Tensor> produced_tensors,
                            std::vector<Tensor> expected_tensors,
                            bool compare_order);

  // Checks `IteratorBase::GetNext()`.
  Status CheckIteratorGetNext(const std::vector<Tensor>& expected_outputs,
                              bool compare_order);

  // Checks `IteratorBase::GetNext()`.
  Status CheckIteratorGetNext(TestIterator* iterator,
                              const std::vector<Tensor>& expected_outputs,
                              bool compare_order);

  // Checks `IteratorBase::GetNext()`.
  Status CheckIteratorGetNext(IteratorBase* iterator, IteratorContext* ctx,
                              const std::vector<Tensor>& expected_outputs,
                              bool compare_order);

  // Checks `IteratorBase::Skip()`
  Status CheckIteratorSkip(int num_to_skip, int expected_num_skipped,
                           bool get_next,
                           const std::vector<Tensor>& expected_outputs,
                           bool compare_order);

  // Checks that iterating through the dataset using a split provider produces
  // the expected outputs.
  Status CheckSplitProviderFullIteration(
      const DatasetParams& params, const std::vector<Tensor>& expected_outputs);

  // Checks that iterating through the dataset using a sharded split provider
  // with the given `num_shards` and `shard_index` produces the expected
  // outputs.
  Status CheckSplitProviderShardedIteration(
      const DatasetParams& params, int64 num_shards, int64 shard_index,
      const std::vector<Tensor>& expected_outputs);

  // Checks `DatasetBase::node_name()`.
  Status CheckDatasetNodeName(const string& expected_dataset_node_name);

  // Checks `DatasetBase::type_string()`.
  Status CheckDatasetTypeString(const string& expected_type_str);

  // Checks `DatasetBase::output_dtypes()`.
  Status CheckDatasetOutputDtypes(const DataTypeVector& expected_output_dtypes);

  // Checks `DatasetBase::output_shapes()`.
  Status CheckDatasetOutputShapes(
      const std::vector<PartialTensorShape>& expected_output_shapes);

  // Checks `DatasetBase::Cardinality()`.
  Status CheckDatasetCardinality(int expected_cardinality);

  // Checks `IteratorBase::output_dtypes()`.
  Status CheckIteratorOutputDtypes(
      const DataTypeVector& expected_output_dtypes);

  // Checks `IteratorBase::output_shapes()`.
  Status CheckIteratorOutputShapes(
      const std::vector<PartialTensorShape>& expected_output_shapes);

  // Checks `IteratorBase::prefix()`.
  Status CheckIteratorPrefix(const string& expected_iterator_prefix);

  Status CheckIteratorSaveAndRestore(
      DatasetBase* dataset, IteratorContext* iterator_ctx,
      const std::string& iterator_prefix,
      const std::vector<Tensor>& expected_outputs,
      const std::vector<int>& breakpoints, bool compare_order);

  Status CheckIteratorSaveAndRestore(
      const std::string& iterator_prefix,
      const std::vector<Tensor>& expected_outputs,
      const std::vector<int>& breakpoints, bool compare_order);

 protected:
  // Make destructor protected so that DatasetOpsTestBase objects cannot
  // be instantiated directly. Only subclasses can be instantiated.
  ~DatasetOpsTestBase() override;

  // Creates a thread pool for parallel tasks.
  Status InitThreadPool(int thread_num);

  // Initializes the runtime for computing the dataset operation and registers
  // the input function definitions. `InitThreadPool()' needs to be called
  // before this method if we want to run the tasks in parallel.
  Status InitFunctionLibraryRuntime(const std::vector<FunctionDef>& flib,
                                    int cpu_num);

  // Creates a new op kernel based on the node definition.
  Status CreateOpKernel(const NodeDef& node_def,
                        std::unique_ptr<OpKernel>* op_kernel);

  // Creates a new op kernel context.
  Status CreateDatasetContext(
      OpKernel* const dateset_kernel,
      gtl::InlinedVector<TensorValue, 4>* const inputs,
      std::unique_ptr<OpKernelContext::Params>* dataset_context_params,
      std::unique_ptr<OpKernelContext>* dataset_context);

  // Creates a new dataset.
  Status CreateDataset(OpKernel* kernel, OpKernelContext* context,
                       DatasetBase** const dataset);

  // Restores the state of the input iterator. It resets the iterator before
  // restoring it to make sure the input iterator does not hold any
  // resources or tasks. Otherwise, restoring an existing iterator may cause
  // the timeout issue or duplicated elements.
  Status RestoreIterator(IteratorContext* ctx, IteratorStateReader* reader,
                         const string& output_prefix,
                         const DatasetBase& dataset,
                         std::unique_ptr<IteratorBase>* iterator);

  // Fetches the dataset from the operation context.
  Status GetDatasetFromContext(OpKernelContext* context, int output_index,
                               DatasetBase** const dataset);

  // Runs an operation producing outputs.
  Status RunOpKernel(OpKernel* op_kernel, OpKernelContext* context);

  // Executes a function producing outputs.
  Status RunFunction(const FunctionDef& fdef, test::function::Attrs attrs,
                     const std::vector<Tensor>& args,
                     const GraphConstructorOptions& graph_options,
                     std::vector<Tensor*> rets);

  // Checks that the size of `inputs` matches the requirement of the op kernel.
  Status CheckOpKernelInput(const OpKernel& kernel,
                            const gtl::InlinedVector<TensorValue, 4>& inputs);

  // Creates a new context for running the dataset operation.
  Status CreateOpKernelContext(OpKernel* kernel,
                               gtl::InlinedVector<TensorValue, 4>* inputs,
                               std::unique_ptr<OpKernelContext>* context);

  // Creates a new context for running the dataset operation.
  Status CreateOpKernelContext(OpKernel* kernel,
                               gtl::InlinedVector<TensorValue, 4>* inputs,
                               std::unique_ptr<OpKernelContext::Params>* params,
                               std::unique_ptr<OpKernelContext>* context);

  // Creates a new iterator context for iterating the dataset.
  Status CreateIteratorContext(
      OpKernelContext* const op_context,
      std::unique_ptr<IteratorContext>* iterator_context);

  // Creates a new iterator context for iterating the dataset.
  // Creates a new serialization context for serializing the dataset and
  // iterator.
  Status CreateSerializationContext(
      std::unique_ptr<SerializationContext>* context);

 private:
  // Runs the dataset operation according to the predefined dataset params and
  // the produced outputs will be stored in `dataset_ctx`.
  Status RunDatasetOp(
      const DatasetParams& dataset_params,
      std::unique_ptr<OpKernel>* dataset_kernel,
      std::unique_ptr<OpKernelContext::Params>* dataset_ctx_params,
      std::vector<std::unique_ptr<Tensor>>* created_tensors,
      std::unique_ptr<OpKernelContext>* dataset_ctx);

  Status MakeDataset(
      const DatasetParams& dataset_params,
      std::unique_ptr<OpKernel>* dataset_kernel,
      std::unique_ptr<OpKernelContext::Params>* dataset_ctx_params,
      std::unique_ptr<OpKernelContext>* dataset_ctx,
      std::vector<std::unique_ptr<Tensor>>* created_tensors,
      DatasetBase** dataset);

  // Creates the dataset op kernel.
  Status MakeDatasetOpKernel(const DatasetParams& dataset_params,
                             std::unique_ptr<OpKernel>* dataset_kernel);

  // Creates a dataset tensor according to the input dataset params.
  Status MakeDatasetTensor(
      const DatasetParams& dataset_params,
      std::vector<std::unique_ptr<Tensor>>* created_tensors,
      std::unique_ptr<Tensor>* dataset);

  // Adds an empty tensor with the specified dtype and shape to the input
  // vector.
  Status AddDatasetInput(gtl::InlinedVector<TensorValue, 4>* inputs,
                         DataTypeVector input_types, DataType dtype,
                         const TensorShape& shape);

 protected:
  std::unique_ptr<Device> device_;
  DeviceType device_type_;
  int cpu_num_;
  int thread_num_;
  Allocator* allocator_;  // Owned by `AllocatorFactoryRegistry`.
  std::vector<AllocatorAttributes> allocator_attrs_;
  std::unique_ptr<ScopedStepContainer> step_container_;

  // Device manager is used by function handle cache and needs to outlive it.
  std::unique_ptr<DeviceMgr> device_mgr_;
  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
  FunctionLibraryRuntime* flr_;  // Owned by `pflr_`.
  std::unique_ptr<FunctionHandleCache> function_handle_cache_;
  std::function<void(std::function<void()>)> runner_;
  std::unique_ptr<FunctionLibraryDefinition> lib_def_;
  std::unique_ptr<ResourceMgr> resource_mgr_;
  std::unique_ptr<checkpoint::TensorSliceReaderCacheWrapper>
      slice_reader_cache_;
  std::unique_ptr<thread::ThreadPool> thread_pool_;
  std::vector<std::unique_ptr<Tensor>> tensors_;  // Owns tensors.
  mutex lock_for_refs_;  // Used as the Mutex for inputs added as refs.
  std::unique_ptr<CancellationManager> cancellation_manager_;

  // Indicates if the below fields have been initialized.
  bool initialized_ = false;
  std::unique_ptr<OpKernel> dataset_kernel_;
  std::unique_ptr<OpKernelContext::Params> params_;
  std::unique_ptr<OpKernelContext> dataset_ctx_;
  DatasetBase* dataset_ = nullptr;
  std::unique_ptr<IteratorContext> iterator_ctx_;
  std::unique_ptr<IteratorBase> iterator_;
};

#define ITERATOR_GET_NEXT_TEST_P(dataset_op_test_class, dataset_params_class, \
                                 test_cases)                                  \
  class ParameterizedGetNextTest                                              \
      : public dataset_op_test_class,                                         \
        public ::testing::WithParamInterface<                                 \
            GetNextTestCase<dataset_params_class>> {};                        \
                                                                              \
  TEST_P(ParameterizedGetNextTest, GetNext) {                                 \
    auto test_case = GetParam();                                              \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                       \
    TF_ASSERT_OK(                                                             \
        CheckIteratorGetNext(test_case.expected_outputs,                      \
                             /*compare_order=*/test_case.compare_order));     \
  }                                                                           \
                                                                              \
  INSTANTIATE_TEST_SUITE_P(                                                   \
      dataset_op_test_class, ParameterizedGetNextTest,                        \
      ::testing::ValuesIn(                                                    \
          std::vector<GetNextTestCase<dataset_params_class>>(test_cases)));

#define ITERATOR_SKIP_TEST_P(dataset_op_test_class, dataset_params_class,   \
                             test_cases)                                    \
  class ParameterizedSkipTest : public dataset_op_test_class,               \
                                public ::testing::WithParamInterface<       \
                                    SkipTestCase<dataset_params_class>> {}; \
                                                                            \
  TEST_P(ParameterizedSkipTest, GetNext) {                                  \
    auto test_case = GetParam();                                            \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                     \
    TF_ASSERT_OK(CheckIteratorSkip(                                         \
        test_case.num_to_skip, test_case.expected_num_skipped,              \
        test_case.get_next, test_case.expected_outputs,                     \
        /*compare_order=*/test_case.compare_order));                        \
  }                                                                         \
                                                                            \
  INSTANTIATE_TEST_SUITE_P(                                                 \
      dataset_op_test_class, ParameterizedSkipTest,                         \
      ::testing::ValuesIn(                                                  \
          std::vector<SkipTestCase<dataset_params_class>>(test_cases)));

#define DATASET_NODE_NAME_TEST_P(dataset_op_test_class, dataset_params_class, \
                                 test_cases)                                  \
  class ParameterizedDatasetNodeNameTest                                      \
      : public dataset_op_test_class,                                         \
        public ::testing::WithParamInterface<                                 \
            DatasetNodeNameTestCase<dataset_params_class>> {};                \
                                                                              \
  TEST_P(ParameterizedDatasetNodeNameTest, DatasetNodeName) {                 \
    auto test_case = GetParam();                                              \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                       \
    TF_ASSERT_OK(CheckDatasetNodeName(test_case.expected_node_name));         \
  }                                                                           \
                                                                              \
  INSTANTIATE_TEST_SUITE_P(                                                   \
      dataset_op_test_class, ParameterizedDatasetNodeNameTest,                \
      ::testing::ValuesIn(                                                    \
          std::vector<DatasetNodeNameTestCase<dataset_params_class>>(         \
              test_cases)));

#define DATASET_TYPE_STRING_TEST_P(dataset_op_test_class,                \
                                   dataset_params_class, test_cases)     \
  class ParameterizedDatasetTypeStringTest                               \
      : public dataset_op_test_class,                                    \
        public ::testing::WithParamInterface<                            \
            DatasetTypeStringTestCase<dataset_params_class>> {};         \
                                                                         \
  TEST_P(ParameterizedDatasetTypeStringTest, DatasetTypeString) {        \
    auto test_case = GetParam();                                         \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                  \
    TF_ASSERT_OK(                                                        \
        CheckDatasetTypeString(test_case.expected_dataset_type_string)); \
  }                                                                      \
                                                                         \
  INSTANTIATE_TEST_SUITE_P(                                              \
      dataset_op_test_class, ParameterizedDatasetTypeStringTest,         \
      ::testing::ValuesIn(                                               \
          std::vector<DatasetTypeStringTestCase<dataset_params_class>>(  \
              test_cases)));

#define DATASET_OUTPUT_DTYPES_TEST_P(dataset_op_test_class,                   \
                                     dataset_params_class, test_cases)        \
                                                                              \
  class ParameterizedDatasetOutputDtypesTest                                  \
      : public dataset_op_test_class,                                         \
        public ::testing::WithParamInterface<                                 \
            DatasetOutputDtypesTestCase<dataset_params_class>> {};            \
                                                                              \
  TEST_P(ParameterizedDatasetOutputDtypesTest, DatasetOutputDtypes) {         \
    auto test_case = GetParam();                                              \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                       \
    TF_ASSERT_OK(CheckDatasetOutputDtypes(test_case.expected_output_dtypes)); \
  }                                                                           \
                                                                              \
  INSTANTIATE_TEST_SUITE_P(                                                   \
      dataset_op_test_class, ParameterizedDatasetOutputDtypesTest,            \
      ::testing::ValuesIn(                                                    \
          std::vector<DatasetOutputDtypesTestCase<dataset_params_class>>(     \
              test_cases)));

#define DATASET_OUTPUT_SHAPES_TEST_P(dataset_op_test_class,                   \
                                     dataset_params_class, test_cases)        \
                                                                              \
  class ParameterizedDatasetOutputShapesTest                                  \
      : public dataset_op_test_class,                                         \
        public ::testing::WithParamInterface<                                 \
            DatasetOutputShapesTestCase<dataset_params_class>> {};            \
                                                                              \
  TEST_P(ParameterizedDatasetOutputShapesTest, DatasetOutputShapes) {         \
    auto test_case = GetParam();                                              \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                       \
    TF_ASSERT_OK(CheckDatasetOutputShapes(test_case.expected_output_shapes)); \
  }                                                                           \
                                                                              \
  INSTANTIATE_TEST_SUITE_P(                                                   \
      dataset_op_test_class, ParameterizedDatasetOutputShapesTest,            \
      ::testing::ValuesIn(                                                    \
          std::vector<DatasetOutputShapesTestCase<dataset_params_class>>(     \
              test_cases)));

#define DATASET_CARDINALITY_TEST_P(dataset_op_test_class,                  \
                                   dataset_params_class, test_cases)       \
                                                                           \
  class ParameterizedCardinalityTest                                       \
      : public dataset_op_test_class,                                      \
        public ::testing::WithParamInterface<                              \
            CardinalityTestCase<dataset_params_class>> {};                 \
                                                                           \
  TEST_P(ParameterizedCardinalityTest, Cardinality) {                      \
    auto test_case = GetParam();                                           \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                    \
    TF_ASSERT_OK(CheckDatasetCardinality(test_case.expected_cardinality)); \
  }                                                                        \
                                                                           \
  INSTANTIATE_TEST_SUITE_P(                                                \
      dataset_op_test_class, ParameterizedCardinalityTest,                 \
      ::testing::ValuesIn(                                                 \
          std::vector<CardinalityTestCase<dataset_params_class>>(          \
              test_cases)));

#define ITERATOR_OUTPUT_DTYPES_TEST_P(dataset_op_test_class,                  \
                                      dataset_params_class, test_cases)       \
  class ParameterizedIteratorOutputDtypesTest                                 \
      : public dataset_op_test_class,                                         \
        public ::testing::WithParamInterface<                                 \
            IteratorOutputDtypesTestCase<dataset_params_class>> {};           \
                                                                              \
  TEST_P(ParameterizedIteratorOutputDtypesTest, IteratorOutputDtypes) {       \
    auto test_case = GetParam();                                              \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                       \
    TF_ASSERT_OK(CheckDatasetOutputDtypes(test_case.expected_output_dtypes)); \
  }                                                                           \
                                                                              \
  INSTANTIATE_TEST_SUITE_P(                                                   \
      dataset_op_test_class, ParameterizedIteratorOutputDtypesTest,           \
      ::testing::ValuesIn(                                                    \
          std::vector<IteratorOutputDtypesTestCase<dataset_params_class>>(    \
              test_cases)));

#define ITERATOR_OUTPUT_SHAPES_TEST_P(dataset_op_test_class,                   \
                                      dataset_params_class, test_cases)        \
  class ParameterizedIteratorOutputShapesTest                                  \
      : public dataset_op_test_class,                                          \
        public ::testing::WithParamInterface<                                  \
            IteratorOutputShapesTestCase<dataset_params_class>> {};            \
                                                                               \
  TEST_P(ParameterizedIteratorOutputShapesTest, IteratorOutputShapes) {        \
    auto test_case = GetParam();                                               \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                        \
    TF_ASSERT_OK(CheckIteratorOutputShapes(test_case.expected_output_shapes)); \
  }                                                                            \
                                                                               \
  INSTANTIATE_TEST_SUITE_P(                                                    \
      dataset_op_test_class, ParameterizedIteratorOutputShapesTest,            \
      ::testing::ValuesIn(                                                     \
          std::vector<IteratorOutputShapesTestCase<dataset_params_class>>(     \
              test_cases)));

#define ITERATOR_PREFIX_TEST_P(dataset_op_test_class, dataset_params_class, \
                               test_cases)                                  \
  class ParameterizedIteratorPrefixTest                                     \
      : public dataset_op_test_class,                                       \
        public ::testing::WithParamInterface<                               \
            IteratorPrefixTestCase<dataset_params_class>> {};               \
                                                                            \
  TEST_P(ParameterizedIteratorPrefixTest, IteratorPrefix) {                 \
    auto test_case = GetParam();                                            \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                     \
    TF_ASSERT_OK(CheckIteratorPrefix(test_case.expected_iterator_prefix));  \
  }                                                                         \
                                                                            \
  INSTANTIATE_TEST_SUITE_P(                                                 \
      dataset_op_test_class, ParameterizedIteratorPrefixTest,               \
      ::testing::ValuesIn(                                                  \
          std::vector<IteratorPrefixTestCase<dataset_params_class>>(        \
              test_cases)));

#define ITERATOR_SAVE_AND_RESTORE_TEST_P(dataset_op_test_class,              \
                                         dataset_params_class, test_cases)   \
  class ParameterizedIteratorSaveAndRestoreTest                              \
      : public dataset_op_test_class,                                        \
        public ::testing::WithParamInterface<                                \
            IteratorSaveAndRestoreTestCase<dataset_params_class>> {};        \
  TEST_P(ParameterizedIteratorSaveAndRestoreTest, IteratorSaveAndRestore) {  \
    auto test_case = GetParam();                                             \
    TF_ASSERT_OK(Initialize(test_case.dataset_params));                      \
    TF_ASSERT_OK(CheckIteratorSaveAndRestore(                                \
        test_case.dataset_params.iterator_prefix(),                          \
        test_case.expected_outputs, test_case.breakpoints,                   \
        test_case.compare_order));                                           \
  }                                                                          \
  INSTANTIATE_TEST_SUITE_P(                                                  \
      dataset_op_test_class, ParameterizedIteratorSaveAndRestoreTest,        \
      ::testing::ValuesIn(                                                   \
          std::vector<IteratorSaveAndRestoreTestCase<dataset_params_class>>( \
              test_cases)));

}  // namespace data
}  // namespace tensorflow

#endif  // TENSORFLOW_CORE_KERNELS_DATA_DATASET_TEST_BASE_H_


text:
from django.conf.urls.defaults import *
from django.conf import settings

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    (r'^admin/', include(admin.site.urls)),
    (r'^media/(?P<path>.*)$', 'django.views.static.serve', {'document_root' : settings.MEDIA_ROOT}),
    (r'', include('invoicer.urls', namespace='invoicer', app_name='invoicer')),
)

redpajama_set_name: RedPajamaGithub


text:
Options_t g_opts;

int main(int argc, char *argv[]) {
  CLI::App rp("rp++: a fast ROP gadget finder for pe/elf/mach-o x86/x64/ARM "
              "binaries\nby Axel '0vercl0k' Souchet.\n");
  rp.add_option("-f,--file", g_opts.file, "Binary path")->required();
  rp.add_option("-i,--info", g_opts.display,
                "display information about the binary header");
  rp.add_option("-r,--rop", g_opts.rop,
                "find useful gadget for your future exploits, arg is the "
                "gadget maximum size in instructions");
  const std::unordered_map<std::string, CPU::E_CPU> raw_archi_map = {
      {"x86", CPU::CPU_x86},
      {"x64", CPU::CPU_x64},
      {"arm", CPU::CPU_ARM},
      {"arm64", CPU::CPU_ARM64}};
  rp.add_option("--raw", g_opts.raw, "find gadgets in a raw file")
      ->transform(CLI::CheckedTransformer(raw_archi_map, CLI::ignore_case));
  rp.add_flag("--unique", g_opts.unique, "display only unique gadget");
  rp.add_option("--search-hexa", g_opts.shexa, "try to find hex values");
  rp.add_option("--max-thread", g_opts.maxth,
                "set the maximum number of threads that can be used");
  rp.add_option("--bad-bytes", g_opts.badbytes,
                "the bytes you don't want to see in the gadgets' addresses");
  rp.add_option("--search-int", g_opts.sint,
                "try to find a pointer on a specific integer value");
  rp.add_flag("-v,--version", g_opts.version, "print version information");
  rp.add_flag("--colors", g_colors_desired, "enable colors");
  rp.add_flag("--thumb", g_opts.thumb,
              "enable thumb mode when looking for ARM gadgets");
  rp.add_option("--va", g_opts.va,
              "don't use the image base of the binary, but yours instead");
  rp.add_flag("--allow-branches", g_opts.allow_branches,
              "allow branches in a gadget");
  rp.add_flag("--print-bytes", g_opts.print_bytes, "print the gadget bytes");

  CLI11_PARSE(rp, argc, argv);

  try {
    if (g_opts.version) {
      fmt::print("You are currently using the version {} of rp++.\n", VERSION);
    }

    Program p(g_opts.file, g_opts.raw);
    if (g_opts.display >= VERBOSE_LEVEL_1 &&
        g_opts.display <= VERBOSE_LEVEL_3) {
      p.display_information(VerbosityLevel(g_opts.display));
    }

    // Here we set the base being 0 if we want to have absolute virtual
    // memory address displayed
    const uint64_t base = g_opts.va.size() > 0
                              ? std::strtoull(g_opts.va.c_str(), nullptr, 0)
                              : p.get_image_base_address();
    if (g_opts.rop > 0) {
      const uint32_t options = g_opts.thumb ? 1 : 0;
      fmt::print("\nWait a few seconds, rp++ is looking for gadgets ({} "
                 "threads max)..\n",
                 g_opts.maxth);

      GadgetMultiset all_gadgets =
          p.find_gadgets(g_opts.rop, options, g_opts.maxth, base);

      fmt::print("A total of {} gadgets found.\n", all_gadgets.size());
      std::vector<uint8_t> badbyte_list;
      if (g_opts.badbytes.size() > 0) {
        badbyte_list = string_to_hex(g_opts.badbytes);
      }

      uint64_t nb_gadgets_filtered = 0;
      if (g_opts.unique) {
        auto unique_gadgets = only_unique_gadgets(all_gadgets);

        fmt::print("You decided to keep only the unique ones, {} unique "
                   "gadgets found.\n",
                   unique_gadgets.size());

        // Now we walk the gadgets found and set the VA
        for (const auto &unique_gadget : unique_gadgets) {
          display_gadget_lf(unique_gadget.get_first_absolute_address(),
                            unique_gadget);
        }
      } else {
        for (const auto &gadget : all_gadgets) {
          display_gadget_lf(gadget.get_first_absolute_address(), gadget);
        }
      }

      if (g_opts.badbytes.size() > 0) {
        fmt::print(
            "\n{} gadgets have been filtered because of your bad-bytes.\n",
            nb_gadgets_filtered);
      }
    }

    if (g_opts.shexa.size() > 0) {
      const std::vector<uint8_t> &hex_values = string_to_hex(g_opts.shexa);
      p.search_and_display(hex_values.data(), hex_values.size(), base);
    }

    if (g_opts.sint.size() > 0) {
      const uint32_t val = std::strtoul(g_opts.sint.c_str(), nullptr, 16);
      p.search_and_display((const uint8_t *)&val, sizeof(val), base);
    }
  } catch (const std::exception &e) {
    enable_color(COLO_RED);
    fmt::print("{}\n", e.what());
    disable_color();
  }

  return 0;
}


text:
// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package kafka implements a client for the Kafka broker. The client supports
sending and receiving of messages through the Kafka message bus. It provides
both sync and async Producers for sending Kafka messages and a Consumer for
retrieving Kafka messages.

A Producer sends messages to Kafka. A Producer can be either synchronous
or asynchronous. Request to send a message using a synchronous producer
blocks until the message is published or an error is returned. A request
sent using asynchronous producer returns immediately and the success or
failure is communicated to the sender through a separate status channels.

A Consumer receives messages from Kafka for one or more topics. When a
consumer is initialized,it automatically balances/shares the total number
partitions for a message topic over all the active brokers for a topic.
Message offsets can optionally be committed to Kafka so that when a consumer
is restarted or a new consumer is initiated it knows where to begin reading
messages from the Kafka message log.

The package also provides a Multiplexer that allows to share consumer and
producers instances among multiple entities called Connections. Apart from
reusing the access to kafka brokers, the Multiplexer marks the offset of
consumed message as read. Offset marking allows to consume messages from the
last committed offset after the restart of the Multiplexer.

Note: Marking offset does not necessarily commit the offset to the backend
store immediately. This might result in a corner case where a message might
be delivered multiple times.

Usage of synchronous producer:
	// create minimal configuration
	config := client.NewConfig()
	config.SetBrokers("ip_addr:port", "ip_addr2:port")

	producer, err := client.NewSyncProducer(config, nil)
	if err != nil {
		os.Exit(1)
	}
	// key and value are of type []byte
	producer.SendMsgByte(topic, key, value, meta)

	// key and value are of type Encoder
	producer.SendMsgToPartition(topic, key, value, meta)

Usage of asynchronous producer:
	succCh := make(chan *client.ProducerMessage)
	errCh := make(chan *client.ProducerError)

	// init config
	config := client.NewConfig()
	config.SetSendSuccess(true)
	config.SetSuccessChan(succCh)
	config.SetSendError(true)
	config.SetErrorChan(errCh)
	config.SetBrokers("ip_addr:port", "ip_addr2:port")

	// init producer
	producer, err := client.NewAsyncProducer(config, nil)

	go func() {
	eventLoop:
		for {
			select {
			case <-producer.GetCloseChannel():
				break eventLoop
			case msg := <-succCh:
				fmt.Println("message sent successfully - ", msg)
			case err := <-errCh:
				fmt.Println("message errored - ", err)
			}
		}
	}()

	producer.SendMsgByte(topic, key, value, meta)

Usage of consumer:
        config := client.NewConfig()
	config.SetRecvNotification(true)
	config.SetRecvNotificationChan(make(chan *cluster.Notification))
	config.SetRecvError(true)
	config.SetRecvErrorChan(make(chan error))
	config.SetRecvMessageChan(make(chan *client.ConsumerMessage))
	config.SetBrokers("ip_addr:port", "ip_addr2:port2")
	config.SetTopics("topic1,topic2")
	config.SetGroup("Group1")

	consumer, err := client.NewConsumer(config, nil)
	if err != nil {
		log.Errorf("NewConsumer Error: %v", err)
		os.Exit(1)
	}

	go func() {
		for {
			select {
			case notification := <-config.RecvNotificationChan:
				handleNotifcation(consumer)
			case err := <-config.RecvErrorChan:
				fmt.Printf("Message Recv Errored: %v\n", err)
			case msg := <-config.RecvMessageChan:
				messageCallback(consumer, msg, *commit)
			case <-consumer.GetCloseChannel():
				return
			}
		}
	}()

In addition to basic sync/async producer and consumer the Multiplexer is provided. It's behaviour is depicted below:

 +---------------+              +--------------------+
 | Connection #1 |------+       |    Multiplexer     |
 +---------------+      |       |                    |
                        |       |    sync producer   |
 +---------------+      |       |   async producer   |		       /------------\
 | Connection #2 |------+-----> |    consumer        |<---------->/    Kafka     \
 +---------------+      |       |                    |            \--------------/
                        |       |                    |
 +---------------+      |       |                    |
 | Connection #3 |------+       +--------------------+
 +---------------+

To initialize multiplexer run:

   mx, err := mux.InitMultiplexer(pathToConfig, "name")

The config file specifies addresses of kafka brokers:
   addrs:
     - "ip_addr1:port"
     - "ip_addr2:port"

To create a Connection that reuses Multiplexer access to kafka run:

	cn := mx.NewBytesConnection("c1")

	or

	cn := mx.NewProtoConnection("c1")

Afterwards you can produce messages using sync API:

	partition, offset, err := cn.SendSyncString("test", "key", "value")

or you can use async API:
	succCh := make(chan *client.ProducerMessage, 10)
	errCh := make(chan *client.ProducerError, 10)
        cn.SendAsyncString("test", "key", "async message", "meta", succCh, errCh)

        // check if the async send succeeded
        go func() {
           select {
	   case success := <-succCh:
		fmt.Println("Successfully send async msg", success.Metadata)
	   case err := <-errCh:
		fmt.Println("Error while sending async msg", err.Err, err.Msg.Metadata)
	   }
	}()

subscribe to consume a topic:
	consumerChan := make(chan *client.ConsumerMessage
        err = cn.ConsumeTopic("test", consumerChan)

	if err == nil {
		fmt.Println("Consuming test partition")
		go func() {
			eventLoop:
				for {
					select {
					case msg := <-consumerChan:
						fmt.Println(string(msg.Key), string(msg.Value))
					case <-signalChan:
						break eventLoop
					}
				}
		}()
	}

Once all connection have subscribed for topic consumption. You have to run the following function
to actually initialize the consumer inside the Multiplexer.

        mx.Start()

To properly clean up the Multiplexer call:

        mx.Close()

The KAFKA plugin

Once kafka plugin is initialized
        plugin := kafka.Plugin{}
        // Init called by agent core

The plugin allows to create connections:

        conn := plugin.NewConnection("name")

or connection that support proto-modelled messages:

        protoConn := plugin.NewProtoConnection("protoConnection")
The usage of connections is described above.

*/
package kafka


text:
ExtractSurfaces::ExtractSurfaces() {

	registerInput(_stack, "stack");
	registerOutput(_surfaces, "surfaces");
}

void
ExtractSurfaces::updateOutputs() {

	// wrap the input stack into a volume adaptor
	ImageStackVolumeAdaptor volume(*_stack);

	// create a marching cubes instance
	MarchingCubes<ImageStackVolumeAdaptor> marchingCubes;

	// get all ids in the image stack
	std::set<unsigned int> ids;

	foreach (boost::shared_ptr<Image> image, *_stack)
		foreach (Image::value_type value, *image)
			if (value != 0)
				ids.insert(value);

	_surfaces = new Meshes;

	foreach (unsigned int id, ids) {

		_surfaces->add(
				id,
				marchingCubes.generateSurface(
					volume,
					MarchingCubes<ImageStackVolumeAdaptor>::AcceptExactly(id),
					10.0,
					10.0,
					10.0));
	}
}


text:
<?php

declare(strict_types=1);

namespace Nelliel\Tables;

defined('NELLIEL_VERSION') or die('NOPE.AVI');

use PDO;

class TableReports extends Table
{

    function __construct($database, $sql_compatibility)
    {
        $this->database = $database;
        $this->sql_compatibility = $sql_compatibility;
        $this->table_name = NEL_REPORTS_TABLE;
        $this->column_types = [
            'report_id' => ['php_type' => 'integer', 'pdo_type' => PDO::PARAM_INT],
            'board_id' => ['php_type' => 'string', 'pdo_type' => PDO::PARAM_STR],
            'content_id' => ['php_type' => 'string', 'pdo_type' => PDO::PARAM_STR],
            'reporter_ip' => ['php_type' => 'string', 'pdo_type' => PDO::PARAM_LOB],
            'hashed_reporter_ip' => ['php_type' => 'string', 'pdo_type' => PDO::PARAM_STR],
            'reason' => ['php_type' => 'string', 'pdo_type' => PDO::PARAM_STR],
            'moar' => ['php_type' => 'string', 'pdo_type' => PDO::PARAM_STR]];
        $this->column_checks = [
            'report_id' => ['row_check' => false, 'auto_inc' => true],
            'board_id' => ['row_check' => false, 'auto_inc' => false],
            'content_id' => ['row_check' => false, 'auto_inc' => false],
            'reporter_ip' => ['row_check' => false, 'auto_inc' => false],
            'hashed_reporter_ip' => ['row_check' => false, 'auto_inc' => false],
            'reason' => ['row_check' => false, 'auto_inc' => false],
            'moar' => ['row_check' => false, 'auto_inc' => false]];
        $this->schema_version = 1;
    }

    public function buildSchema(array $other_tables = null)
    {
        $auto_inc = $this->sql_compatibility->autoincrementColumn('INTEGER', false);
        $options = $this->sql_compatibility->tableOptions();
        $schema = '
        CREATE TABLE ' . $this->table_name . ' (
            report_id           ' . $auto_inc[0] . ' ' . $auto_inc[1] . ' NOT NULL,
            board_id            VARCHAR(50) NOT NULL,
            content_id          VARCHAR(255) NOT NULL,
            reporter_ip         ' . $this->sql_compatibility->sqlAlternatives('VARBINARY', '16') . ' DEFAULT NULL,
            hashed_reporter_ip  VARCHAR(128) NOT NULL,
            reason              TEXT NOT NULL,
            moar                TEXT DEFAULT NULL,
            CONSTRAINT pk_' . $this->table_name . ' PRIMARY KEY (report_id),
            CONSTRAINT fk_reports__domain_registry
            FOREIGN KEY (board_id) REFERENCES ' . NEL_DOMAIN_REGISTRY_TABLE . ' (domain_id)
            ON UPDATE CASCADE
            ON DELETE CASCADE
        ) ' . $options . ';';

        return $schema;
    }

    public function postCreate(array $other_tables = null)
    {
    }

    public function insertDefaults()
    {
    }
}


text:
package androidx.appcompat.app;

/**
 * An activity that handles locales configuration changes.
 */
public class LocalesConfigChangesActivityA extends LocalesUpdateActivity {}


text:
using System;
using System.Collections.ObjectModel;
using Raven.Abstractions.Replication;

namespace Brnkly.Raven
{
    public class Instance
    {
        public Uri Url { get; internal set; }
        public bool AllowReads { get; internal set; }
        public bool AllowWrites { get; internal set; }
        public Collection<ReplicationDestination> Destinations { get; private set; }

        public Instance()
        {
            this.Destinations = new Collection<ReplicationDestination>();
        }

        public override string ToString()
        {
            return string.Format(
                "{0}: AllowReads={1}, AllowWrites={2}, {3} destinations",
                Url,
                AllowReads,
                AllowWrites,
                Destinations.Count);
        }
    }
}


text:
CDefaultUser::CDefaultUser() :
    cookie(""), key(""), pass(""), login(""), salt(""), remouteAddress("127.0.01"),
    userAgent(" "), userId(0), userType(0), lastChange(0), lastAction(0)
{
    setIsValid(false);
}

CDefaultUser::CDefaultUser(const int userId) :
    cookie(""), key(""), pass(""), login(""), salt(""), remouteAddress("127.0.01"),
    userAgent(" "), userId(userId), userType(0), lastChange(0), lastAction(0)
{
    setIsValid(false);
}

CDefaultUser::~CDefaultUser()
{
	for(auto it = bans.begin(); it != bans.end(); ++it)
	{
		CBan* currBan = (*it);
		delete currBan;
	}
}

const std::string& CDefaultUser::getCookie() const
{
	return cookie;
}

int CDefaultUser::getUserId() const
{
	return userId;
}

int CDefaultUser::getUserType() const
{
	if(userType >= UT_ADMIN) return UT_ADMIN;
	if(userType == UT_MODER) return UT_MODER;
	if(userType <= UT_NONE)	 return UT_NONE;

	return UT_USER;
}

long CDefaultUser::getLastChange() const
{
	return lastChange;
}

long CDefaultUser::getLastAction() const
{
	return lastAction;
}

bool CDefaultUser::getIsUserGuest() const
{
	CConfigHelper* gs = CConfigHelper::getInstance();

	if(cookie == gs->getStringParamValue("guestCookieValue", "GUEST")) return true;
	return false;
}

bool CDefaultUser::getIsAccessClosed() const
{
	for(auto it = bans.begin(); it != bans.end(); ++it)
	{
		CBan* currBan = (*it);

		if(currBan->getIsAccessClosed()) return true;
	}
	return false;
}

long CDefaultUser::getReadOnlyBanTime() const
{
	time_t currUnixTime;
	time(&currUnixTime);

	long maxROTime = 0;
	for(auto it = bans.begin(); it != bans.end(); ++it)
	{
		CBan* currBan = (*it);

		long timeToEndRO = 	currBan->getBanCreadtionTime() + currBan->getROTime() - (long)currUnixTime;
		if(timeToEndRO > maxROTime) maxROTime = timeToEndRO;
	}
	return maxROTime;
}

CDefaultUser* CDefaultUser::loginUser(const std::string& login, const std::string& pass)
{
	int userId = authorization(login, pass);
	if(userId > 0)
	{
		CSessionManager* sessionManager =  CManagers::getInstance()->getSessionManager();
		CDefaultUser* newUser = sessionManager->userCreationFunc();

		CDBManager* dbManager = CManagers::getInstance()->getDBManager();
		std::shared_ptr<CDBRequest> dbRequest(dbManager->createDBRequest());
		dbRequest->updateRequest("Users", "`key` = '" + dbManager->getEscapeString(genUserKey()) + "'", "`id` = " + valueToString(userId));

		newUser->fillUserDataById(userId);
		return newUser;
	}

	return NULL;
}

int CDefaultUser::authorization(const std::string& login, const std::string& pass)
{
	CConfigHelper* gs = CConfigHelper::getInstance();
	bool isHasDB = gs->getStringParamValue("isHasDB", "false") == "true";
	if(!isHasDB) return 0;

	CDBManager* dbManager = CManagers::getInstance()->getDBManager();
	std::shared_ptr<CDBRequest> dbRequest(dbManager->createDBRequest());

	const CDBRequestResult* result = dbRequest->selectRequest(CDBValues("id,salt,pass"), "Users", "`login` = '"+dbManager->getEscapeString(login)+"'", "LIMIT 1" );
	if(dbRequest->getIsLastQuerySuccess() && result != NULL  && result->getRowsCnt() > 0)
	{
		int id = result->getIntValue(0,0);
		if(id <= 0) return 0;
		std::string salt = result->getStringValue(0,1);
		std::string userPass = result->getStringValue(0,2);

		std::string testedPassHash = md5(md5(pass) + salt);
		if(userPass == testedPassHash) return id;

		CLog::getInstance()->addWarning("Wrong pass");
	}
	CLog::getInstance()->addWarning("Login: " + login + " not exist");
	return 0;
}

CDefaultUser* CDefaultUser::createNewUser(const std::string& login, const std::string& pass, const std::string& cookieData, const std::string& remouteAddress, const  std::string& userAgent)
{
	CDBManager* dbManager = CManagers::getInstance()->getDBManager();
	std::shared_ptr<CDBRequest> dbRequest(dbManager->createDBRequest());
	time_t currUnixTime;
	time(&currUnixTime);

	CSessionManager* sessionManager = CManagers::getInstance()->getSessionManager();
	CDefaultUser* newUser = sessionManager->userCreationFunc();
	CLog::getInstance()->addInfo("New User");

	std::string key = cookieData;
	if(key.length() == 0) key = genUserKey();

	CConfigHelper* gs = CConfigHelper::getInstance();
	bool isHasDB = gs->getStringParamValue("isHasDB", "false") == "true";
	if(isHasDB)
	{
		const CDBRequestResult* result = dbRequest->selectRequest(CDBValues("id"), "Users", "`login` = '"+dbManager->getEscapeString(login)+"'", "LIMIT 1" );
		if(dbRequest->getIsLastQuerySuccess() && result != NULL  && result->getRowsCnt() > 0)
		{
			CLog::getInstance()->addWarning("User login: " + login + " already exist");
			return newUser;
		}

		std::string salt = genUserKey();
		std::string userPass = md5(md5(pass) + salt);

		CDBValues fields;
		fields.addValue("remouteAddress", remouteAddress);
		fields.addValue("type", 1);
		fields.addValue("key", key);
		fields.addValue("login", login);
		fields.addValue("pass", userPass);
		fields.addValue("salt", salt);
		fields.addValue("lastAction", (long)currUnixTime);
		fields.addValue("userAgent", userAgent);
		dbRequest->insertRequest(fields, "Users");
	}
	newUser->fillUserData(key);

	return newUser;
}

void CDefaultUser::fillUserDataById(const int userId)
{
	CDBManager* dbManager = CManagers::getInstance()->getDBManager();
	std::shared_ptr<CDBRequest> dbRequest(dbManager->createDBRequest());

	CConfigHelper* gs = CConfigHelper::getInstance();
	bool isHasDB = gs->getStringParamValue("isHasDB", "false") == "true";
	if(!isHasDB)
	{
		fillUserData(gs->getStringParamValue("guestCookieValue", "GUEST"));
		return;
	}

	const CDBRequestResult* result = dbRequest->selectRequest(CDBValues("key"), "Users", "`id` = "+valueToString(userId), "LIMIT 1" );
	if(dbRequest->getIsLastQuerySuccess() && result != NULL  && result->getRowsCnt() > 0)
	{
		std::string key = result->getStringValue(0,0);
		if((int)key.size() > 0) fillUserData(key);
	}
}

bool CDefaultUser::isUserExist(const std::string& userKey)
{
	CDBManager* dbManager = CManagers::getInstance()->getDBManager();
	std::shared_ptr<CDBRequest> dbRequest(dbManager->createDBRequest());

	const CDBRequestResult* result = dbRequest->selectRequest(CDBValues("id"), "Users", "`key` = '" + dbManager->getEscapeString(userKey) + "'", "LIMIT 1");
	if (result != NULL  && result->getRowsCnt() > 0) return true;

	return false;
}

void CDefaultUser::fillUserData(const std::string& cookieData)
{
	CConfigHelper* gs = CConfigHelper::getInstance();
	int userId = 0;
	int userType = 0;

	this->userId = 1;
    this->userType = UT_USER;
    this->key = cookieData;
    this->lastChange = 0;
    this->lastAction = 0;
    this->userAgent = "";
    this->pass = "";
    this->salt = "";
    this->login = "";
    this->cookie = gs->getStringParamValue("guestCookieValue", "GUEST");

	bool isHasDB = gs->getStringParamValue("isHasDB", "false") == "true";
	if(!isHasDB || cookieData == gs->getStringParamValue("guestCookieValue", "GUEST"))
	{
		this->userId = 1;
		this->userType = UT_USER;
		this->key = cookieData;
		this->lastChange = 0;
		this->lastAction = 0;
		this->userAgent = "";
		this->pass = "";
		this->salt = "";
		this->login = "";
		this->cookie = gs->getStringParamValue("guestCookieValue", "GUEST");

		setIsValid(true);
		return;
	}

	CDBManager* dbManager = CManagers::getInstance()->getDBManager();
	std::shared_ptr<CDBRequest> dbRequest(dbManager->createDBRequest());

	const CDBRequestResult* result = dbRequest->selectRequest(CDBValues("id,type,remouteAddress,lastChange,lastAction,userAgent,login,pass,salt"), "Users", "`key` = '" + dbManager->getEscapeString(cookieData)  + "'", "LIMIT 1");
	if(dbRequest->getIsLastQuerySuccess() && result != NULL  && result->getRowsCnt() > 0)
	{
		try
		{
			userId   = result->getIntValue(0,0);
			userType = result->getIntValue(0,1);
		}
		catch(...)
		{
			userId   = 0;
			userType = 0;
		}

		key = cookieData;
		remouteAddress = result->getStringValue(0,2);
		lastChange = result->getLongValue(0,3);
		lastAction = result->getLongValue(0,4);
		userAgent = result->getStringValue(0,5);
		login = result->getStringValue(0,6);
		pass = result->getStringValue(0,7);
		salt = result->getStringValue(0,8);
	}

	this->cookie = cookieData;
	this->userId = userId;
	this->userType = userType;

	setIsValid(true);
	if(cookie == "" || userId == 0 || userType < 0 || key == "") setIsValid(false);

	if(getIsValid())
	{
		const CDBRequestResult* result = dbRequest->selectRequest(CDBValues("*"), "Bans", "`userId` = "+valueToString(userId));
		if(dbRequest->getIsLastQuerySuccess() && result != NULL && result->getRowsCnt() > 0)
		{
			time_t currUnixTime;
			time(&currUnixTime);

			for(int i = 0; i < result->getRowsCnt(); i++)
			{
				int banId = 0;
				int banUserId = 0;
				long banCreationTime = 0;
				long banROTime = 0;
				bool banIsAccessClosed;

				banId = result->getIntValue(i, 0);
				banUserId = result->getIntValue(i, 1);
				banROTime = result->getLongValue(i, 2);
				banIsAccessClosed = result->getBoolValue(i, 3);
				banCreationTime = result->getLongValue(i, 4);

				if(banId != 0 && banUserId != 0 && (banIsAccessClosed || currUnixTime <= banCreationTime + banROTime ))
				{
					CBan* ban = new CBan(userId, banROTime, banCreationTime, banUserId, banIsAccessClosed);
					bans.push_back(ban);
				}
				else
				{
					// delete ban
				}
			}
		}
		afterFillUserData();
	}
}

void CDefaultUser::afterFillUserData()
{

}

const std::string& CDefaultUser::getLogin() const
{
	return login;
}

const std::string& CDefaultUser::getPass() const
{
	return pass;
}

const std::string& CDefaultUser::getSalt() const
{
	return salt;
}

const std::string& CDefaultUser::getUserKey() const
{
	return key;
}

const std::string& CDefaultUser::getRemouteAddress() const
{
	return remouteAddress;
}

const std::string& CDefaultUser::getUserAgent() const
{
	return userAgent;
}

const std::vector<CBan*>* CDefaultUser::getUserBans() const
{
	return &bans;
}

std::string CDefaultUser::genUserKey()
{
	char key[17];
	for(int i = 0; i < 16; i++)
	{
		key[i] = rand() % 57 + 65;
		if(key[i] >= 91 && key[i] <= 96) key[i] -= 6;
	}
	key[16] = '\0';

	std::string resultStr = key;
	return resultStr;
}


text:
using CapuchinSync.Core.GenerateSynchronizationDictionary;
using CapuchinSync.Core.Interfaces;
using NSubstitute;
using NUnit.Framework;
using System.Linq;

namespace CapuchinSync.Core.Tests
{
    [TestFixture]
    public class GenerateSyncHashesCommandLineArgumentParserTests
    {

        [Test]
        public void Constructor_ExistingDirectoryShouldResultInSuccessfulParse()
        {
            var directory = "aDirectory";
            var fileSystem = Substitute.For<IFileSystem>();
            fileSystem.DoesDirectoryExist("").ReturnsForAnyArgs(true);

            var parser = new GenerateSyncHashesCommandLineArgumentParser(new[] { $"{GenerateSyncHashesCommandLineArgumentParser.DirectoryPrefix}{directory}" }, fileSystem);

            Assert.AreEqual(parser.ErrorNumber, 0);
            var args = parser.Arguments;
            Assert.IsNotNull(args, "Null arguments were found");
            var extensionsToExclude = args.ExtensionsToExclude;
            Assert.IsNotNull(extensionsToExclude, "Extensions to exclude was expected to be an empty collection, but was null.");
            Assert.IsTrue(!extensionsToExclude.Any(), $"Expected an empty collection of extensions to exclude, found [{string.Join(", ", extensionsToExclude)}]");
            var directories = args.RootDirectories;
            Assert.IsNotNull(directories, "RootDirectories was unexpectedly null.");
            Assert.AreEqual(1, directories.Length, $"Expected a single directory, but found {directories.Length}: [{string.Join(", ", directories)}]");
            Assert.AreEqual(directory, directories.Single(), "Unexpected directory.");
        }

        [Test]
        public void Constructor_ShouldThrowErrorOnMissingDirectoryByDefault()
        {
            var missingDir = "missingDir";
            var fileSystem = Substitute.For<IFileSystem>();
            fileSystem.DoesDirectoryExist("").ReturnsForAnyArgs(false);

            var parser = new GenerateSyncHashesCommandLineArgumentParser(new[] {$"{GenerateSyncHashesCommandLineArgumentParser.DirectoryPrefix}{missingDir}"}, fileSystem);
            Assert.AreEqual(parser.ErrorNumber, GenerateSyncHashesCommandLineArgumentParser.ErrorCodes.DirectoryDoesNotExist);
        }

        [Test]
        public void Constructor_MissingDirectoryShouldResultInSuccessfulParseWithIgnoreMissingDirectoriesArgument()
        {
            var existingDirectory = "anExistingDirectory";
            var missingDirectory = "aMissingDirectory";
            var fileSystem = Substitute.For<IFileSystem>();
            fileSystem.DoesDirectoryExist(existingDirectory).Returns(true);
            fileSystem.DoesDirectoryExist(missingDirectory).Returns(false);

            var parser = new GenerateSyncHashesCommandLineArgumentParser(
                new[]
                {
                    $"{GenerateSyncHashesCommandLineArgumentParser.DirectoryPrefix}{missingDirectory}",
                    $"{GenerateSyncHashesCommandLineArgumentParser.DirectoryPrefix}{existingDirectory}",
                    GenerateSyncHashesCommandLineArgumentParser.IgnoreMissingDirectoriesArg
                }, fileSystem);

            Assert.AreEqual(parser.ErrorNumber, 0);
            var args = parser.Arguments;
            Assert.IsNotNull(args, "Null arguments were found");
            var extensionsToExclude = args.ExtensionsToExclude;
            Assert.IsNotNull(extensionsToExclude, "Extensions to exclude was expected to be an empty collection, but was null.");
            Assert.IsTrue(!extensionsToExclude.Any(), $"Expected an empty collection of extensions to exclude, found [{string.Join(", ", extensionsToExclude)}]");
            var directories = args.RootDirectories;
            Assert.IsNotNull(directories, "RootDirectories was unexpectedly null.");
            Assert.AreEqual(1, directories.Length, $"Expected a single directory, but found {directories.Length}: [{string.Join(", ", directories)}]");
            Assert.AreEqual(existingDirectory, directories.Single(), "Unexpected directory.");
        }
    }
}


text:
ACCEPTED

#### According to
Integrated Taxonomic Information System

#### Published in
Sida 20:1414. 2003

#### Original name
null

### Remarks
null


text:
from texttable import Texttable
from ussclicore.argumentParser import ArgumentParser, ArgumentParserError
from ussclicore.cmd import Cmd, CoreGlobal
from ussclicore.utils import generics_utils, printer
from hammr.utils import *
from hammr.utils.hammr_utils import *

class Platform(Cmd, CoreGlobal):
    """List platforms available"""

    cmd_name = "platform"

    def __init__(self):
        super(Platform, self).__init__()

    def arg_list(self):
        doParser = ArgumentParser(add_help=True, description="List all the target platform I can access.")
        return doParser

    def do_list(self, args):
        try:
            printer.out("Getting target platform :")
            targetPlatformsUser = self.api.Users(self.login).Targetplatforms.Getall()
            if targetPlatformsUser is None or len(targetPlatformsUser.targetPlatforms.targetPlatform) == 0:
                printer.out("There is no target platform")
                return 0
            else:
                targetPlatformsUser = generics_utils.order_list_object_by(
                    targetPlatformsUser.targetPlatforms.targetPlatform, "name")
                printer.out("Target platform list:")
                table = Texttable(200)
                table.set_cols_align(["c", "c", "c", "c"])
                table.header(["Id", "Name", "Type", "Access"])
                for item in targetPlatformsUser:
                    if item.access:
                        access = "X"
                    else:
                        access = ""
                    table.add_row([item.dbId, item.name, item.type, access])
                print table.draw() + "\n"
            return 0

        except ArgumentParserError as e:
            printer.out("In Arguments: " + str(e), printer.ERROR)
            self.help_list()
        except Exception as e:
            return handle_uforge_exception(e)

    def help_list(self):
        doParser = self.arg_list()
        doParser.print_help()


text:
namespace Cofoundry.Core.BackgroundTasks;

/// <summary>
/// Allows various types of background tasks to be scheduled.
/// </summary>
/// <remarks>
/// This is only meant to be a simple implementation to support the needs of class libraries, in an actual
/// application you'd most likely use a scheduler like HanFire directly to allow for more complex
/// scenarios.
/// </remarks>
public interface IBackgroundTaskScheduler
{
    /// <summary>
    /// Register a job to run every X number of days, at a specific time.
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <param name="days">The number of days to wait before executing the task again</param>
    /// <param name="atHour">The hour fo the day to execute the task (0-23)</param>
    /// <param name="atMinute">The minute of the hour to execute the task (0-60)</param>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler RegisterRecurringTask<TTask>(int days, int atHour = 0, int atMinute = 0) where TTask : IRecurringBackgroundTask;

    /// <summary>
    /// Register a job to run every X number of hours.
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <param name="hours">The number of hours to wait before executing the task again</param>
    /// <param name="minute">The minute of the hour to execute the task (0-60)</param>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler RegisterRecurringTask<TTask>(int hours = 0, int minute = 0) where TTask : IRecurringBackgroundTask;

    /// <summary>
    /// Register a job to run every X number of minutes.
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <param name="minutes">The number of minutes to wait before executing the task again</param>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler RegisterRecurringTask<TTask>(int minutes) where TTask : IRecurringBackgroundTask;

    /// <summary>
    /// Deregisters the specified job type (The class name/namespace is used as the identifier).
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler DeregisterRecurringTask<TTask>() where TTask : IRecurringBackgroundTask;

    /// <summary>
    /// Register a job to run every X number of days, at a specific time.
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <param name="days">The number of days to wait before executing the task again</param>
    /// <param name="atHour">The hour fo the day to execute the task (0-23)</param>
    /// <param name="atMinute">The minute of the hour to execute the task (0-60)</param>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler RegisterAsyncRecurringTask<TTask>(int days, int atHour = 0, int atMinute = 0) where TTask : IAsyncRecurringBackgroundTask;

    /// <summary>
    /// Register a job to run every X number of hours.
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <param name="hours">The number of hours to wait before executing the task again</param>
    /// <param name="minute">The minute of the hour to execute the task (0-60)</param>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler RegisterAsyncRecurringTask<TTask>(int hours = 0, int minute = 0) where TTask : IAsyncRecurringBackgroundTask;

    /// <summary>
    /// Register a job to run every X number of minutes.
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <param name="minutes">The number of minutes to wait before executing the task again</param>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler RegisterAsyncRecurringTask<TTask>(int minutes) where TTask : IAsyncRecurringBackgroundTask;

    /// <summary>
    /// Deregisters the specified job type (The class name/namespace is used as the identifier).
    /// </summary>
    /// <typeparam name="TTask">The type of task to execute</typeparam>
    /// <returns>Instance of IBackgroundTaskScheduler for method chaining</returns>
    IBackgroundTaskScheduler DeregisterAsyncRecurringTask<TTask>() where TTask : IAsyncRecurringBackgroundTask;
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<mule-project runtimeId="org.mule.tooling.server.3.7.1.ee" schemaVersion="3.5.0.0" xmlns="http://www.mulesoft.com/tooling/project">
    <name>web-service-consumer_1.3.2-SNAPSHOT</name>
    <description/>
</mule-project>


text:
def mangle(sentence):
    words = []
    for word in sentence.split():
        capitals_indices = [word.index(c) for c in word if c.isupper()]
        punctuation_data = [(i, c) for i, c in enumerate(word) if not c.isalnum()]
        for i, c in punctuation_data:
            word = word.replace(c, '')
        word = sorted(word.lower())
        for i, c in punctuation_data:
            word.insert(i, c)
        word = [c.upper() if i in capitals_indices else c for i, c in enumerate(word)]
        words.append("".join(word))
    sentence = " ".join(words)
    return sentence


text:
**Namespace:** [OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers](OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers.md)
**Assembly:** OfficeDevPnP.Core.dll
## Syntax
```C#
public virtual void Invoke(String message, Int32 step, Int32 total)
```
### Parameters
#### message
  Type: System.String

#### step
  Type: System.Int32

#### total
  Type: System.Int32

### Return Value
Type: void

## See also
- [ProvisioningProgressDelegate](OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers.ProvisioningProgressDelegate.md)
- [OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers](OfficeDevPnP.Core.Framework.Provisioning.ObjectHandlers.md)


text:
using System;
using System.Collections;
using DevExpress.XtraGrid;
using DevExpress.XtraGrid.Columns;
using DevExpress.XtraGrid.Views.Grid;

namespace KabMan
{
    public class RefreshHelper
    {
        private readonly string keyFieldName;
        private ArrayList saveExpList;
        private ArrayList saveMasterRowsList;
        private ArrayList saveSelList;
        private readonly GridView view;
        private int visibleRowIndex = -1;

        public RefreshHelper(GridView view, string keyFieldName)
        {
            this.view = view;
            this.keyFieldName = keyFieldName;
        }

        public ArrayList SaveExpList
        {
            get
            {
                if (saveExpList == null)
                    saveExpList = new ArrayList();
                return saveExpList;
            }
        }

        public ArrayList SaveSelList
        {
            get
            {
                if (saveSelList == null)
                    saveSelList = new ArrayList();
                return saveSelList;
            }
        }

        public ArrayList SaveMasterRowsList
        {
            get
            {
                if (saveMasterRowsList == null)
                    saveMasterRowsList = new ArrayList();
                return saveMasterRowsList;
            }
        }

        protected int FindParentRowHandle(RowInfo rowInfo, int rowHandle)
        {
            int result = view.GetParentRowHandle(rowHandle);
            while (view.GetRowLevel(result) != rowInfo.level)
                result = view.GetParentRowHandle(result);
            return result;
        }

        protected void ExpandRowByRowInfo(RowInfo rowInfo)
        {
            int dataRowHandle = view.LocateByValue(0, view.Columns[keyFieldName], rowInfo.Id);
            if (dataRowHandle != GridControl.InvalidRowHandle)
            {
                int parentRowHandle = FindParentRowHandle(rowInfo, dataRowHandle);
                view.SetRowExpanded(parentRowHandle, true, false);
            }
        }

        protected int GetRowHandleToSelect(RowInfo rowInfo)
        {
            int dataRowHandle = view.LocateByValue(0, view.Columns[keyFieldName], rowInfo.Id);
            if (dataRowHandle != GridControl.InvalidRowHandle)
                if (view.GetRowLevel(dataRowHandle) != rowInfo.level)
                    return FindParentRowHandle(rowInfo, dataRowHandle);
            return dataRowHandle;
        }

        protected void SelectRowByRowInfo(RowInfo rowInfo, bool isFocused)
        {
            if (isFocused)
                view.FocusedRowHandle = GetRowHandleToSelect(rowInfo);
            else
                view.SelectRow(GetRowHandleToSelect(rowInfo));
        }

        public void SaveSelectionViewInfo(ArrayList list)
        {
            list.Clear();
            GridColumn column = view.Columns[keyFieldName];
            RowInfo rowInfo;
            int[] selectionArray = view.GetSelectedRows();
            if (selectionArray != null) // otherwise we have a single focused but not selected row
                for (int i = 0; i < selectionArray.Length; i++)
                {
                    int dataRowHandle = selectionArray[i];
                    rowInfo.level = view.GetRowLevel(dataRowHandle);
                    if (dataRowHandle < 0) // group row
                        dataRowHandle = view.GetDataRowHandleByGroupRowHandle(dataRowHandle);
                    rowInfo.Id = view.GetRowCellValue(dataRowHandle, column);
                    list.Add(rowInfo);
                }
            rowInfo.Id = view.GetRowCellValue(view.FocusedRowHandle, column);
            rowInfo.level = view.GetRowLevel(view.FocusedRowHandle);
            list.Add(rowInfo);
        }

        public void SaveExpansionViewInfo(ArrayList list)
        {
            if (view.GroupedColumns.Count == 0) return;
            list.Clear();
            GridColumn column = view.Columns[keyFieldName];
            for (int i = -1; i > int.MinValue; i--)
            {
                if (!view.IsValidRowHandle(i)) break;
                if (view.GetRowExpanded(i))
                {
                    RowInfo rowInfo;
                    int dataRowHandle = view.GetDataRowHandleByGroupRowHandle(i);
                    rowInfo.Id = view.GetRowCellValue(dataRowHandle, column);
                    rowInfo.level = view.GetRowLevel(i);
                    list.Add(rowInfo);
                }
            }
        }

        public void SaveExpandedMasterRows(ArrayList list)
        {
            if (view.GridControl.Views.Count == 1) return;
            list.Clear();
            GridColumn column = view.Columns[keyFieldName];
            for (int i = 0; i < view.DataRowCount; i++)
                if (view.GetMasterRowExpanded(i))
                    list.Add(view.GetRowCellValue(i, column));
        }

        public void SaveVisibleIndex()
        {
            visibleRowIndex = view.GetVisibleIndex(view.FocusedRowHandle) - view.TopRowIndex;
        }

        public void LoadVisibleIndex()
        {
            view.MakeRowVisible(view.FocusedRowHandle, true);
            view.TopRowIndex = view.GetVisibleIndex(view.FocusedRowHandle) - visibleRowIndex;
        }

        public void LoadSelectionViewInfo(ArrayList list)
        {
            view.BeginSelection();
            try
            {
                view.ClearSelection();
                for (int i = 0; i < list.Count; i++)
                    SelectRowByRowInfo((RowInfo) list[i], i == list.Count - 1);
            }
            finally
            {
                view.EndSelection();
            }
        }

        public void LoadExpansionViewInfo(ArrayList list)
        {
            if (view.GroupedColumns.Count == 0) return;
            view.BeginUpdate();
            try
            {
                view.CollapseAllGroups();
                foreach (RowInfo info in list)
                    ExpandRowByRowInfo(info);
            }
            finally
            {
                view.EndUpdate();
            }
        }

        public void LoadExpandedMasterRows(ArrayList list)
        {
            view.BeginUpdate();
            try
            {
                view.CollapseAllDetails();
                GridColumn column = view.Columns[keyFieldName];
                for (int i = 0; i < list.Count; i++)
                {
                    int rowHandle = view.LocateByValue(0, column, list[i]);
                    view.SetMasterRowExpanded(rowHandle, true);
                }
            }
            finally
            {
                view.EndUpdate();
            }
        }

        public void SaveViewInfo()
        {
            SaveExpandedMasterRows(SaveMasterRowsList);
            SaveExpansionViewInfo(SaveExpList);
            SaveSelectionViewInfo(SaveSelList);
            SaveVisibleIndex();
        }

        public void LoadViewInfo()
        {
            LoadExpandedMasterRows(SaveMasterRowsList);
            LoadExpansionViewInfo(SaveExpList);
            LoadSelectionViewInfo(SaveSelList);
            LoadVisibleIndex();
        }

        #region Nested type: RowInfo

        [Serializable]
        public struct RowInfo
        {
            public object Id;
            public int level;
        } ;

        #endregion
    }
}


text:
<?php

namespace Zend\Http\Header;

/**
 * @throws Exception\InvalidArgumentException
 * @see http://www.ietf.org/rfc/rfc2617.txt
 */
class AuthenticationInfo implements HeaderInterface
{
    /** @var string */
    protected $value;

    public static function fromString($headerLine)
    {
        list($name, $value) = GenericHeader::splitHeaderLine($headerLine);

        // check to ensure proper header type for this factory
        if (strtolower($name) !== 'authentication-info') {
            throw new Exception\InvalidArgumentException('Invalid header line for Authentication-Info string: "' . $name . '"');
        }

        // @todo implementation details
        $header = new static($value);

        return $header;
    }

    public function __construct($value = null)
    {
        $this->value = $value;
    }

    public function getFieldName()
    {
        return 'Authentication-Info';
    }

    public function getFieldValue()
    {
        return $this->value;
    }

    public function toString()
    {
        return 'Authentication-Info: ' . $this->getFieldValue();
    }
}


text:
from flask import redirect, url_for, render_template, flash, request, send_from_directory, Response
from flask.ext.login import login_user, logout_user, current_user, login_required
from flask.ext.sqlalchemy import SQLAlchemy
from flask_mail import Message
from xd import app, lm, db, mail
from werkzeug import secure_filename
from oauth import OAuthSignIn
from models import User
from collections import defaultdict
import os, smtplib, json, datetime, requests, pprint
from datetime import timedelta

pp = pprint.PrettyPrinter(indent=4)

@app.route('/admin', methods=['GET', 'POST'])
@login_required
def admin():
    if not current_user.is_admin:
        return redirect(url_for('index'))
    return render_template('admin.html')

@lm.user_loader
def load_user(id):
    return User.query.get(int(id))

@app.route('/')
def index():
    # link to registration
    return render_template('index.html')

@app.route('/register', methods=['GET', 'POST'])
@login_required
def register():
    if request.method == 'GET':
        return render_template('registration.html')
    if request.method == 'POST':
        # handle user creation and link to dashboard
        # upload file handling: file = request.files['file']
        return render_template('dashboard.html')

@app.route('/dashboard', methods=['GET'])
@login_required
def dash():
    return render_template('dashboard.html', name=current_user.name)

@app.route('/logout')
@login_required
def logout():
    logout_user()
    return redirect(url_for('index'))

@app.route('/files/<file>', methods=['GET'])
@login_required
def files(file):
    if file == current_user.file:
        return send_from_directory(app.config["UPLOAD_FOLDER"], file)
    return render_template('404.html')

@app.route('/authorize/<provider>')
def oauth_authorize(provider):
    if not current_user.is_anonymous:
        return redirect(url_for('index'))
    oauth = OAuthSignIn.get_provider(provider)
    return oauth.authorize()

@app.route('/callback/<provider>')
def oauth_callback(provider):
    if not current_user.is_anonymous:
        return redirect(url_for('index'))
    oauth = OAuthSignIn.get_provider(provider)
    xd_id, name, email = oauth.callback()
    if xd_id is None:
        flash('Authentication failed.')
        return redirect(url_for('index'))
    user = User.query.filter_by(xd_id=xd_id).first()
    if not user: # Create, add and login new user. Redirect to /register
        user = User(xd_id=xd_id, name=name, email=email)
        db.session.add(user)
        db.session.commit()
        login_user(user, True)
        return redirect(url_for('dash')) # previously register
    else: # Login new user. Redirect to /
        login_user(user, True)
        return redirect(url_for('index'))

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1] in app.config['ALLOWED_EXTENSIONS']

def upload_file_handler(file):
    # Save unique file per user
    filename = str(current_user.id) + "_" + secure_filename(file.filename)
    file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
    current_user.file = filename
    # Delete old file
    path = os.path.abspath(app.config['UPLOAD_FOLDER'])
    list = os.listdir(path)
    for item in list:
        id = int(item.split('_')[0])
        if id == int(current_user.id) and filename != item:
            os.remove(os.path.join(path, item))

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

@app.errorhandler(Exception)
def unhandled_exception(e):
    return render_template('500.html'), 500


text:
using Fhwk.Core.Config;
using Fhwk.Core.Mapping;
using NHibernate.Cfg;
using NHibernate.Cfg.MappingSchema;
using NHibernate.Mapping.ByCode;
using System;

namespace Fhwk.Core
{
    /// <summary>
    /// The Firehawk entry point
    /// </summary>
    public class Firehawk
    {
        #region Members

        private FirehawkConfig rootConfig;
        private ConventionModelMapper modelMapper;

        #endregion

        #region Constructors

        /// <summary>
        /// Initializes a new instance of <see cref="Firehawk"/> class
        /// </summary>
        internal Firehawk()
        {
            this.rootConfig = new FirehawkConfig(this);
            this.modelMapper = new ConventionModelMapper();
        }

        #endregion

        #region Public Methods

        /// <summary>
        /// Starts the Firehawk configuration
        /// </summary>
        /// <returns>An instance of FirehawkConfig</returns>
        public static Firehawk Init()
        {
            return new Firehawk();
        }

        /// <summary>
        /// Starts the Firehawk configuration
        /// </summary>
        /// <returns>An instance of FirehawkConfig</returns>
        public FirehawkConfig Configure()
        {
            return rootConfig;
        }

        /// <summary>
        /// Allows to indicate that a given type is used as root entity type
        /// </summary>
        /// <typeparam name="TRootEntity">The Type of the root entity</typeparam>
        /// <param name="entityMap">The entity class mapper</param>
        /// <returns>An instance of FirehawkConfig</returns>
        public Firehawk AddCustomMapping<TRootEntity>(Action<IClassMapper<TRootEntity>> entityMap) where TRootEntity : class
        {
            this.modelMapper.Class(entityMap);

            return this;
        }

        /// <summary>
        /// Provides access to the underlying ModelMapper
        /// </summary>
        /// <param name="action">An action with the model mapper as input param</param>
        /// <returns></returns>
        public Firehawk GetModelMapper(Action<ModelMapper> action)
        {
            action(this.modelMapper);

            return this;
        }

        /// <summary>
        /// Builds the current configuration by using an instance of NH configuration
        /// </summary>
        /// <param name="config">The NH Configuration</param>
        public void BuildMappings(Configuration config)
        {
            MappingEngine engine = new MappingEngine(this.rootConfig, config, this.modelMapper);
            engine.BuildConfiguration();
        }

        /// <summary>
        /// Compiles the mappings without adding them to the configuration
        /// </summary>
        /// <param name="config">The NH Configuration</param>
        public HbmMapping CompileMappings(Configuration config)
        {
            MappingEngine engine = new MappingEngine(this.rootConfig, config, this.modelMapper);
            return engine.CompileMappings();
        }

        #endregion

    }

}