text:
#ifndef TENSORFLOW_CORE_KERNELS_LIST_KERNELS_H_
#define TENSORFLOW_CORE_KERNELS_LIST_KERNELS_H_

#define EIGEN_USE_THREADS
#if GOOGLE_CUDA
#define EIGEN_USE_GPU
#endif  // GOOGLE_CUDA

#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/variant.h"
#include "tensorflow/core/framework/variant_op_registry.h"
#include "tensorflow/core/kernels/concat_lib.h"
#include "tensorflow/core/kernels/fill_functor.h"
#include "tensorflow/core/lib/core/coding.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/util/tensor_ops_util.h"
#include "tensorflow/core/util/util.h"

namespace tensorflow {

typedef Eigen::ThreadPoolDevice CPUDevice;

// Variant compatible type for a list of tensors. This is mutable but instances
// should never be mutated after stored in a variant tensor.
struct TensorList {
 public:
  TensorList() {}
  TensorList(const TensorList& other);

  static const char kTypeName[];
  string TypeName() const { return kTypeName; }

  void Encode(VariantTensorData* data) const;

  bool Decode(const VariantTensorData& data);

  // TODO(apassos) fill this out
  string DebugString() const { return "TensorList"; }

  std::vector<Tensor> tensors;
  PartialTensorShape element_shape;
  DataType element_dtype;
  // The maximum allowed size of `tensors`. Defaults to -1 meaning that the size
  // of `tensors` is unbounded.
  int max_num_elements = -1;
};

Status TensorShapeFromTensor(const Tensor& t, PartialTensorShape* out);

Status GetElementShapeFromInput(OpKernelContext* c,
                                const TensorList& tensor_list, int index,
                                PartialTensorShape* element_shape);

template <typename Device, typename T>
class TensorListStack : public OpKernel {
 public:
  typedef std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>
      ConstMatrixVector;
  explicit TensorListStack(OpKernelConstruction* c) : OpKernel(c) {
    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
    OP_REQUIRES_OK(c, c->GetAttr("num_elements", &num_elements_));
  }

  void Compute(OpKernelContext* c) override {
    const TensorList* tensor_list =
        c->input(0).scalar<Variant>()().get<TensorList>();
    OP_REQUIRES(c, tensor_list != nullptr,
                errors::InvalidArgument(
                    "Input handle is not a list. Saw: '",
                    c->input(0).scalar<Variant>()().DebugString(), "'"));
    OP_REQUIRES(
        c, element_dtype_ == tensor_list->element_dtype,
        errors::InvalidArgument(
            "Invalid data types; op elements ", DataTypeString(element_dtype_),
            " but list elements ", DataTypeString(tensor_list->element_dtype)));
    if (num_elements_ != -1) {
      OP_REQUIRES(c, tensor_list->tensors.size() == num_elements_,
                  errors::InvalidArgument(
                      "Operation expected a list with ", num_elements_,
                      " elements but got a list with ",
                      tensor_list->tensors.size(), " elements."));
    }
    PartialTensorShape partial_element_shape;
    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 1,
                                               &partial_element_shape));
    OP_REQUIRES(
        c,
        partial_element_shape.IsFullyDefined() || !tensor_list->tensors.empty(),
        errors::InvalidArgument("Tried to stack elements of an empty ",
                                "list with non-fully-defined element_shape: ",
                                partial_element_shape.DebugString()));

    // Check that `element_shape` input tensor is compatible with the shapes of
    // element tensors.
    if (!tensor_list->element_shape.IsFullyDefined()) {
      for (int i = 0; i < tensor_list->tensors.size(); ++i) {
        const Tensor& t = tensor_list->tensors[i];
        if (t.dtype() != DT_INVALID) {
          PartialTensorShape tmp = partial_element_shape;
          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
        }
      }
    }

    // Compute the shape of the output tensor by pre-pending the leading dim to
    // the element_shape.
    TensorShape element_shape;
    OP_REQUIRES(c, partial_element_shape.AsTensorShape(&element_shape),
                errors::InvalidArgument(
                    "Tried to stack list which only contains uninitialized ",
                    "tensors and has a non-fully-defined element_shape: ",
                    partial_element_shape.DebugString()));
    TensorShape output_shape = element_shape;
    output_shape.InsertDim(0, tensor_list->tensors.size());
    Tensor* output;
    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
    if (output->NumElements() == 0) {
      return;
    }

    ConstMatrixVector inputs_flat;
    inputs_flat.reserve(tensor_list->tensors.size());
    Tensor zeros;
    for (const auto& t : tensor_list->tensors) {
      if (t.dtype() != DT_INVALID) {
        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
            t.shaped<T, 2>({1, t.NumElements()})));
      } else {
        if (!zeros.NumElements()) {
          AllocatorAttributes attr;
          if (element_dtype_ == DT_VARIANT) {
            attr.set_on_host(true);
          }
          OP_REQUIRES_OK(
              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
                                               zeros.flat<T>());
        }
        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
            const_cast<const Tensor&>(zeros).shaped<T, 2>(
                {1, zeros.NumElements()})));
      }
    }
    auto output_flat = output->shaped<T, 2>({1, output->NumElements()});

#if GOOGLE_CUDA
    if (std::is_same<Device, Eigen::GpuDevice>::value) {
      ConcatGPU<T>(c, inputs_flat, output, &output_flat);
      return;
    }
#endif  // GOOGLE_CUDA
    ConcatCPU<T>(c->device(), inputs_flat, &output_flat);
  }

 private:
  int num_elements_;
  DataType element_dtype_;
};

template <typename Device, typename T>
class TensorListGetItem : public OpKernel {
 public:
  explicit TensorListGetItem(OpKernelConstruction* c) : OpKernel(c) {
    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
  }

  void Compute(OpKernelContext* c) override {
    OP_REQUIRES(
        c, c->input(0).shape().num_elements() == 1,
        errors::InvalidArgument("List tensors are supposed to be scalars."));
    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
    OP_REQUIRES(c, l != nullptr,
                errors::InvalidArgument(
                    "Input handle is not a list. Saw: '",
                    c->input(0).scalar<Variant>()().DebugString(), "'"));
    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
                errors::InvalidArgument("Invalid data types; op elements ",
                                        DataTypeString(element_dtype_),
                                        " but list elements ",
                                        DataTypeString(l->element_dtype)));
    int32 index = c->input(1).scalar<int32>()();
    OP_REQUIRES(c, index < l->tensors.size(),
                errors::InvalidArgument("Trying to access element ", index,
                                        " in a list with ", l->tensors.size(),
                                        " elements."));
    if (l->tensors[index].dtype() != DT_INVALID) {
      c->set_output(0, l->tensors[index]);
    } else {
      PartialTensorShape partial_element_shape;
      OP_REQUIRES_OK(
          c, GetElementShapeFromInput(c, *l, 2, &partial_element_shape));
      TensorShape element_shape;
      // If l->element_shape and the element_shape input are both not fully
      // defined, try to infer the shape from other list elements. This requires
      // that all initialized list elements have the same shape.
      // NOTE(srbs): This might be a performance bottleneck since we are
      // iterating over the entire list here. This is necessary for feature
      // parity with TensorArray.read. TensorArray has a mode in which all
      // elements are required to be of the same shape, TensorList does not.
      // In that mode TensorArray sets the array's element_shape on the first
      // write call. We could do something similar here if needed.
      if (!partial_element_shape.IsFullyDefined()) {
        for (const Tensor& t : l->tensors) {
          if (t.dtype() != DT_INVALID) {
            PartialTensorShape tmp = partial_element_shape;
            OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
          }
        }
      }
      OP_REQUIRES(
          c, partial_element_shape.AsTensorShape(&element_shape),
          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
                                  "element_shape is not fully defined: ",
                                  partial_element_shape.DebugString(),
                                  " and no list element is set."));
      Tensor* result;
      AllocatorAttributes attr;
      if (element_dtype_ == DT_VARIANT) {
        attr.set_on_host(true);
      }
      OP_REQUIRES_OK(c, c->allocate_output(0, element_shape, &result, attr));
      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
                                           result->flat<T>());
    }
  }

 private:
  DataType element_dtype_;
};

template <typename Device, typename T>
class TensorListPopBack : public OpKernel {
 public:
  explicit TensorListPopBack(OpKernelConstruction* c) : OpKernel(c) {
    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
  }

  void Compute(OpKernelContext* c) override {
    const TensorList* l = c->input(0).scalar<Variant>()().get<TensorList>();
    OP_REQUIRES(c, l != nullptr,
                errors::InvalidArgument(
                    "Input handle is not a list. Saw: '",
                    c->input(0).scalar<Variant>()().DebugString(), "'"));
    OP_REQUIRES(c, element_dtype_ == l->element_dtype,
                errors::InvalidArgument("Invalid data types; op elements ",
                                        DataTypeString(element_dtype_),
                                        " but list elements ",
                                        DataTypeString(l->element_dtype)));

    OP_REQUIRES(c, !l->tensors.empty(),
                errors::InvalidArgument("Trying to pop from an empty list."));

    const Tensor& t = l->tensors.back();
    if (t.dtype() != DT_INVALID) {
      c->set_output(1, t);
    } else {
      PartialTensorShape partial_element_shape;
      OP_REQUIRES_OK(
          c, GetElementShapeFromInput(c, *l, 1, &partial_element_shape));
      TensorShape element_shape;
      OP_REQUIRES(
          c, partial_element_shape.AsTensorShape(&element_shape),
          errors::InvalidArgument("Trying to read an uninitialized tensor but ",
                                  "element_shape is not fully defined.",
                                  partial_element_shape.DebugString()));
      Tensor* result;
      AllocatorAttributes attr;
      if (element_dtype_ == DT_VARIANT) {
        attr.set_on_host(true);
      }
      OP_REQUIRES_OK(c, c->allocate_output(1, element_shape, &result, attr));
      functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
                                           result->flat<T>());
    }
    AllocatorAttributes attr;
    attr.set_on_host(true);
    std::unique_ptr<Tensor> maybe_result = c->forward_input(
        0, 0, DT_VARIANT, TensorShape{}, c->input_memory_type(0), attr);
    if (maybe_result != nullptr) {
      maybe_result->scalar<Variant>()().get<TensorList>()->tensors.pop_back();
    } else {
      TensorList output;
      output = *l;
      output.tensors.pop_back();
      Tensor* result;
      OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr));
      result->scalar<Variant>()() = std::move(output);
    }
  }

 private:
  DataType element_dtype_;
};

template <typename Device, typename T>
class TensorListConcat : public OpKernel {
 public:
  using ConstMatrixVector =
      std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>;
  explicit TensorListConcat(OpKernelConstruction* c) : OpKernel(c) {
    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
    // TODO(skyewm): the HasAttr check can be removed once the
    // element_shape_except_first_dim attr has been checked in for 2 weeks
    // (around 1/14/2019).
    if (c->HasAttr("element_shape")) {
      PartialTensorShape element_shape;
      OP_REQUIRES_OK(c, c->GetAttr("element_shape", &element_shape));
      if (!element_shape.unknown_rank()) {
        element_shape_except_first_dim_ = PartialTensorShape(
            gtl::ArraySlice<int64>(element_shape.dim_sizes()).subspan(1));
      }
    }
  }

  void Compute(OpKernelContext* c) override {
    // Check that the input Variant tensor is indeed a TensorList and has the
    // correct element type.
    const TensorList* tensor_list =
        c->input(0).scalar<Variant>()().get<TensorList>();
    OP_REQUIRES(c, tensor_list != nullptr,
                errors::InvalidArgument(
                    "Input handle is not a list. Saw: '",
                    c->input(0).scalar<Variant>()().DebugString(), "'"));
    OP_REQUIRES(
        c, element_dtype_ == tensor_list->element_dtype,
        errors::InvalidArgument(
            "Invalid data types; op elements ", DataTypeString(element_dtype_),
            " but list elements ", DataTypeString(tensor_list->element_dtype)));
    // The leading dimension of all list elements if they are all the same.
    // This is used as the leading dim of uninitialized tensors in the list
    // if leading_dims is not provided.
    int64 first_dim = -1;
    if (c->num_inputs() > 1) {
      // TensorListConcatV2
      PartialTensorShape element_shape;
      OP_REQUIRES_OK(
          c, GetElementShapeFromInput(c, *tensor_list, 1, &element_shape));
      OP_REQUIRES(c, element_shape.unknown_rank() || element_shape.dims() >= 1,
                  errors::InvalidArgument(
                      "Concat requires elements to be at least vectors, ",
                      "found scalars instead."));
      // Split `element_shape` into `first_dim` and
      // `element_shape_except_first_dim_`.
      first_dim = element_shape.dim_size(0);
      element_shape_except_first_dim_ = element_shape;
      element_shape_except_first_dim_.RemoveDim(0);
    }
    // If the TensorList is empty, element_shape_except_first_dim_ must be fully
    // defined.
    OP_REQUIRES(c,
                !tensor_list->tensors.empty() ||
                    element_shape_except_first_dim_.IsFullyDefined(),
                errors::InvalidArgument(
                    "All except the first dimension must be fully defined ",
                    "when concating an empty tensor list. element_shape: ",
                    element_shape_except_first_dim_.DebugString()));
    // 1. Check that `element_shape_except_first_dim_` input tensor is
    //    compatible with the shapes of element tensors.
    // 2. Check that the elements have the same shape except the first dim.
    // 3. If `first_dim` is known, check that it is compatible with the leading
    //    dims of all elements.
    // 4. If `first_dim` is unknown (-1), check whether all initialized
    //    elements have the same leading dim and if so set `first_dim` to that
    //    value.
    if (!tensor_list->element_shape.IsFullyDefined()) {
      bool check_dim = (first_dim == -1);
      int64 inferred_first_dim = first_dim;
      for (int i = 0; i < tensor_list->tensors.size(); ++i) {
        const Tensor& t = tensor_list->tensors[i];
        if (t.dtype() != DT_INVALID) {
          PartialTensorShape tmp = element_shape_except_first_dim_;
          OP_REQUIRES(
              c, TensorShapeUtils::IsVectorOrHigher(t.shape()),
              errors::InvalidArgument("Concat saw a scalar shape at index ", i,
                                      " but requires at least vectors."));
          TensorShape shape_except_first_dim = TensorShape(
              gtl::ArraySlice<int64>(t.shape().dim_sizes()).subspan(1));
          OP_REQUIRES_OK(c, tmp.MergeWith(shape_except_first_dim,
                                          &element_shape_except_first_dim_));
          OP_REQUIRES(c, first_dim == -1 || first_dim == t.shape().dim_size(0),
                      errors::InvalidArgument(
                          "First entry of element_shape input does not match ",
                          "the first dim of list element at index: ", i,
                          " Expected: ", first_dim,
                          " Actual: ", t.shape().dim_size(0)));
          if (check_dim) {
            if (inferred_first_dim == -1) {
              inferred_first_dim = t.shape().dim_size(0);
            } else if (inferred_first_dim != t.shape().dim_size(0)) {
              inferred_first_dim = -1;
              check_dim = false;
            }
          }
        }
      }
      first_dim = inferred_first_dim;
    }
    TensorShape output_shape;
    OP_REQUIRES(
        c, element_shape_except_first_dim_.AsTensorShape(&output_shape),
        errors::InvalidArgument(
            "Trying to concat list with only uninitialized tensors ",
            "but element_shape_except_first_dim_ is not fully defined: ",
            element_shape_except_first_dim_.DebugString()));
    // Build the lengths_tensor and leading dim of the output tensor by
    // iterating over all element tensors.
    Tensor* lengths_tensor = nullptr;
    OP_REQUIRES_OK(
        c,
        c->allocate_output(
            1, TensorShape({static_cast<int64>(tensor_list->tensors.size())}),
            &lengths_tensor));
    auto lengths_tensor_vec = lengths_tensor->vec<int64>();
    int64 leading_dim = 0;
    for (size_t i = 0; i < tensor_list->tensors.size(); i++) {
      int64 dim;
      if (tensor_list->tensors[i].dtype() != DT_INVALID) {
        dim = tensor_list->tensors[i].shape().dim_size(0);
      } else {
        // If leading_dims is not provided or does not contain an entry for
        // index i use the inferred `first_dim` if set.
        if ((c->num_inputs() <= 2 || i >= c->input(2).NumElements()) &&
            first_dim != -1) {
          dim = first_dim;
        } else {
          OP_REQUIRES(c, c->num_inputs() > 2,
                      errors::InvalidArgument(
                          "Concating lists with uninitialized tensors is not ",
                          "supported in this version of TensorListConcat. ",
                          "Consider updating your GraphDef to run the newer ",
                          "version."));
          OP_REQUIRES(c, i < c->input(2).NumElements(),
                      errors::InvalidArgument(
                          "List contains uninitialized tensor at index ", i,
                          " but leading_dims has only ",
                          c->input(2).NumElements(), " elements."));
          dim = c->input(2).vec<int64>()(i);
        }
      }
      leading_dim += dim;
      lengths_tensor_vec(i) = dim;
    }
    output_shape.InsertDim(0, leading_dim);
    Tensor* output;
    // Allocate the output tensor and fill it up with the concated element
    // tensors.
    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
    if (output->NumElements() == 0) {
      return;
    }

    ConstMatrixVector inputs_flat;
    inputs_flat.reserve(tensor_list->tensors.size());
    // Store the zeros tensors in a vector to prevent them from being GC'ed till
    // concat is complete.
    std::vector<Tensor> zeros_vec;
    for (int i = 0; i < tensor_list->tensors.size(); i++) {
      const Tensor& element_tensor = tensor_list->tensors[i];
      if (element_tensor.dtype() != DT_INVALID) {
        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
            element_tensor.shaped<T, 2>({1, element_tensor.NumElements()})));
      } else {
        AllocatorAttributes attr;
        if (element_dtype_ == DT_VARIANT) {
          attr.set_on_host(true);
        }
        TensorShape element_shape = output_shape;
        element_shape.set_dim(0, lengths_tensor_vec(i));
        zeros_vec.emplace_back();
        Tensor& zeros = zeros_vec.back();
        OP_REQUIRES_OK(
            c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
        functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
                                             zeros.flat<T>());
        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
            const_cast<const Tensor&>(zeros).shaped<T, 2>(
                {1, zeros.NumElements()})));
      }
    }
    auto output_flat = output->shaped<T, 2>({1, output->NumElements()});

#if GOOGLE_CUDA
    if (std::is_same<Device, Eigen::GpuDevice>::value) {
      ConcatGPU<T>(c, inputs_flat, output, &output_flat);
      return;
    }
#endif  // GOOGLE_CUDA
    ConcatCPU<T>(c->device(), inputs_flat, &output_flat);
  }

 private:
  DataType element_dtype_;
  PartialTensorShape element_shape_except_first_dim_;
};

template <typename Device, typename T>
class TensorListSplit : public OpKernel {
 public:
  TensorListSplit(OpKernelConstruction* c) : OpKernel(c) {}

  void Compute(OpKernelContext* c) override {
    Tensor* output_tensor;
    AllocatorAttributes attr;
    attr.set_on_host(true);
    OP_REQUIRES_OK(c, c->allocate_output(0, {}, &output_tensor, attr));
    PartialTensorShape element_shape;
    OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(1), &element_shape));
    OP_REQUIRES(c, element_shape.unknown_rank() || element_shape.dims() >= 1,
                errors::InvalidArgument(
                    "TensorListSplit requires element_shape to be at least of ",
                    "rank 1, but saw: ", element_shape.DebugString()));
    TensorList output_list;
    const Tensor& input_tensor = c->input(0);
    output_list.element_dtype = input_tensor.dtype();
    OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input_tensor.shape()),
                errors::InvalidArgument(
                    "Tensor must be at least a vector, but saw shape: ",
                    input_tensor.shape().DebugString()));
    TensorShape tensor_shape_without_first_dim(input_tensor.shape());
    tensor_shape_without_first_dim.RemoveDim(0);
    PartialTensorShape element_shape_without_first_dim;
    if (!element_shape.unknown_rank()) {
      element_shape_without_first_dim =
          PartialTensorShape(element_shape.dim_sizes());
      element_shape_without_first_dim.RemoveDim(0);
    }
    OP_REQUIRES(c,
                element_shape_without_first_dim.IsCompatibleWith(
                    tensor_shape_without_first_dim),
                errors::InvalidArgument(
                    "tensor shape ", input_tensor.shape().DebugString(),
                    " is not compatible with element_shape ",
                    element_shape.DebugString()));
    output_list.element_shape = element_shape;
    const Tensor& lengths = c->input(2);
    OP_REQUIRES(c, TensorShapeUtils::IsVector(lengths.shape()),
                errors::InvalidArgument(
                    "Expected lengths to be a vector, received shape: ",
                    lengths.shape().DebugString()));
    output_list.tensors.reserve(lengths.shape().dim_size(0));
    int64 start = 0;
    int64 end = 0;
    for (int i = 0; i < lengths.shape().dim_size(0); ++i) {
      int64 length = lengths.vec<int64>()(i);
      OP_REQUIRES(
          c, length >= 0,
          errors::InvalidArgument("Invalid value in lengths: ", length));
      end = start + length;
      OP_REQUIRES(c, end <= input_tensor.shape().dim_size(0),
                  errors::InvalidArgument("Attempting to slice [", start, ", ",
                                          end, "] from tensor with length ",
                                          input_tensor.shape().dim_size(0)));
      Tensor tmp = input_tensor.Slice(start, end);
      start = end;
      // TODO(apassos) maybe not always align; but weird compiler bugs seem to
      // prevent this.
      Tensor aligned;
      OP_REQUIRES_OK(c, c->allocate_temp(tmp.dtype(), tmp.shape(), &aligned));
      aligned.flat<T>().device(c->eigen_device<Device>()) =
          tmp.unaligned_flat<T>();
      output_list.tensors.emplace_back(aligned);
    }
    OP_REQUIRES(c, end == input_tensor.shape().dim_size(0),
                errors::InvalidArgument(
                    "Unused values in tensor. Length of tensor: ",
                    input_tensor.shape().dim_size(0), " Values used: ", end));
    output_tensor->scalar<Variant>()() = std::move(output_list);
  }
};

template <typename Device, typename T>
class TensorListGather : public OpKernel {
 public:
  typedef std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>
      ConstMatrixVector;
  explicit TensorListGather(OpKernelConstruction* c) : OpKernel(c) {
    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
  }

  void Compute(OpKernelContext* c) override {
    const TensorList* tensor_list =
        c->input(0).scalar<Variant>()().get<TensorList>();
    OP_REQUIRES(c, tensor_list != nullptr,
                errors::InvalidArgument(
                    "Input handle is not a list. Saw: '",
                    c->input(0).scalar<Variant>()().DebugString(), "'"));
    OP_REQUIRES(
        c, element_dtype_ == tensor_list->element_dtype,
        errors::InvalidArgument(
            "Invalid data types; op elements ", DataTypeString(element_dtype_),
            " but list elements ", DataTypeString(tensor_list->element_dtype)));
    const Tensor& indices = c->input(1);
    PartialTensorShape partial_element_shape;
    OP_REQUIRES_OK(c, GetElementShapeFromInput(c, *tensor_list, 2,
                                               &partial_element_shape));
    OP_REQUIRES(
        c, partial_element_shape.IsFullyDefined() || indices.NumElements() > 0,
        errors::InvalidArgument("Tried to gather 0-elements from "
                                "a list with non-fully-defined shape: ",
                                partial_element_shape.DebugString()));

    // Check that `element_shape` input tensor is compatible with the shapes of
    // element tensors.
    if (!tensor_list->element_shape.IsFullyDefined()) {
      for (int index = 0; index < indices.NumElements(); ++index) {
        const int i = indices.flat<int32>()(index);
        const Tensor& t = tensor_list->tensors[i];
        if (t.dtype() != DT_INVALID) {
          PartialTensorShape tmp = partial_element_shape;
          OP_REQUIRES_OK(c, tmp.MergeWith(t.shape(), &partial_element_shape));
        }
      }
    }

    // Compute the shape of the output tensor by pre-pending the leading dim to
    // the element_shape.
    TensorShape element_shape;
    OP_REQUIRES(
        c, partial_element_shape.AsTensorShape(&element_shape),
        errors::InvalidArgument("Tried to gather uninitialized tensors from a ",
                                "list with non-fully-defined element_shape: ",
                                partial_element_shape.DebugString()));
    TensorShape output_shape = element_shape;
    output_shape.InsertDim(0, indices.NumElements());
    Tensor* output;
    OP_REQUIRES_OK(c, c->allocate_output(0, output_shape, &output));
    if (output->NumElements() == 0) {
      return;
    }

    ConstMatrixVector inputs_flat;
    inputs_flat.reserve(indices.NumElements());
    Tensor zeros;
    for (int index = 0; index < indices.NumElements(); ++index) {
      const int i = indices.flat<int32>()(index);
      OP_REQUIRES(
          c, i < tensor_list->tensors.size(),
          errors::InvalidArgument("Index ", i, " out o range; list only has ",
                                  tensor_list->tensors.size(), " elements."));
      const Tensor& t = tensor_list->tensors[i];
      if (t.dtype() != DT_INVALID) {
        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
            t.shaped<T, 2>({1, t.NumElements()})));
      } else {
        if (!zeros.NumElements()) {
          AllocatorAttributes attr;
          if (element_dtype_ == DT_VARIANT) {
            attr.set_on_host(true);
          }
          OP_REQUIRES_OK(
              c, c->allocate_temp(element_dtype_, element_shape, &zeros, attr));
          functor::SetZeroFunctor<Device, T>()(c->eigen_device<Device>(),
                                               zeros.flat<T>());
        }
        inputs_flat.emplace_back(new typename TTypes<T, 2>::ConstMatrix(
            const_cast<const Tensor&>(zeros).shaped<T, 2>(
                {1, zeros.NumElements()})));
      }
    }
    auto output_flat = output->shaped<T, 2>({1, output->NumElements()});

#if GOOGLE_CUDA
    if (std::is_same<Device, Eigen::GpuDevice>::value) {
      ConcatGPU<T>(c, inputs_flat, output, &output_flat);
      return;
    }
#endif  // GOOGLE_CUDA
    ConcatCPU<T>(c->device(), inputs_flat, &output_flat);
  }

 private:
  DataType element_dtype_;
};

template <typename Device, typename T>
class TensorListFromTensor : public OpKernel {
 public:
  TensorListFromTensor(OpKernelConstruction* c) : OpKernel(c) {}

  void Compute(OpKernelContext* c) override {
    Tensor* output_tensor;
    AllocatorAttributes attr;
    attr.set_on_host(true);
    OP_REQUIRES_OK(c, c->allocate_output(0, {}, &output_tensor, attr));
    PartialTensorShape element_shape;
    OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(1), &element_shape));
    TensorList output_list;
    const Tensor& t = c->input(0);
    output_list.element_dtype = t.dtype();
    OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(t.shape()),
                errors::InvalidArgument(
                    "Tensor must be at least a vector, but saw shape: ",
                    t.shape().DebugString()));
    TensorShape output_shape(t.shape());
    output_shape.RemoveDim(0);
    OP_REQUIRES(c, element_shape.IsCompatibleWith(output_shape),
                errors::InvalidArgument(
                    "Specified a list with shape ", element_shape.DebugString(),
                    " from a tensor with shape ", output_shape.DebugString()));
    output_list.element_shape = element_shape;
    output_list.tensors.reserve(t.shape().dim_size(0));
    for (int i = 0; i < t.shape().dim_size(0); ++i) {
      Tensor tmp = t.Slice(i, i + 1);
      TensorShape tmp_shape = tmp.shape();
      tmp_shape.RemoveDim(0);
      OP_REQUIRES(c, tmp.CopyFrom(tmp, tmp_shape),
                  errors::Unknown("Unexpected shape error."));
      // TODO(apassos) maybe not always align; but weird compiler bugs seem to
      // prevent this.
      Tensor aligned;
      OP_REQUIRES_OK(c, c->allocate_temp(tmp.dtype(), tmp.shape(), &aligned));
      aligned.flat<T>().device(c->eigen_device<Device>()) =
          tmp.unaligned_flat<T>();
      output_list.tensors.push_back(aligned);
    }
    output_tensor->scalar<Variant>()() = std::move(output_list);
  }
};

template <typename Device, typename T>
class TensorListScatter : public OpKernel {
 public:
  TensorListScatter(OpKernelConstruction* c) : OpKernel(c) {}

  void Compute(OpKernelContext* c) override {
    Tensor* output_tensor;
    AllocatorAttributes attr;
    attr.set_on_host(true);
    OP_REQUIRES_OK(c, c->allocate_output(0, {}, &output_tensor, attr));
    Tensor indices = c->input(1);
    PartialTensorShape element_shape;
    OP_REQUIRES_OK(c, TensorShapeFromTensor(c->input(2), &element_shape));
    // TensorListScatterV2 passes the num_elements input, TensorListScatter does
    // not.
    int num_elements = c->num_inputs() >= 4 ? c->input(3).scalar<int>()() : -1;
    OP_REQUIRES(c, num_elements >= -1,
                errors::InvalidArgument(
                    "TensorListScatter expects num_elements >= -1, found: ",
                    num_elements));
    TensorList output_list;
    const Tensor& input_tensor = c->input(0);
    output_list.element_dtype = input_tensor.dtype();
    OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input_tensor.shape()),
                errors::InvalidArgument(
                    "Tensor must be at least a vector, but saw shape: ",
                    input_tensor.shape().DebugString()));
    TensorShape output_shape(input_tensor.shape());
    output_shape.RemoveDim(0);
    OP_REQUIRES(c, element_shape.IsCompatibleWith(output_shape),
                errors::InvalidArgument(
                    "Specified a list with shape ", element_shape.DebugString(),
                    " from a tensor with shape ", output_shape.DebugString()));
    output_list.element_shape = element_shape;

    OP_REQUIRES(c, indices.NumElements() == input_tensor.shape().dim_size(0),
                errors::InvalidArgument(
                    "Invalid number of rows in input tensor. Expected: ",
                    indices.NumElements(),
                    " Actual: ", input_tensor.shape().dim_size(0)));

    // Validate indices and resize output_list.tensors to fit the highest index.
    {
      int highest_index = -1;
      for (int index = 0; index < indices.NumElements(); ++index) {
        const int i = indices.flat<int32>()(index);
        OP_REQUIRES(
            c, i >= 0,
            errors::InvalidArgument(
                "Indices in TensorListScatter must all be non-negative."));
        OP_REQUIRES(c, num_elements == -1 || i < num_elements,
                    errors::InvalidArgument(
                        "TensorListScatter: Trying to scatter at index ", i,
                        " in list with size ", num_elements));
        if (i > highest_index) {
          highest_index = i;
        }
      }
      output_list.tensors.resize(std::max(highest_index + 1, num_elements),
                                 Tensor(DT_INVALID));
    }

    for (int index = 0; index < indices.NumElements(); ++index) {
      const int i = indices.flat<int32>()(index);
      Tensor tmp = input_tensor.Slice(index, index + 1);
      TensorShape tmp_shape = tmp.shape();
      tmp_shape.RemoveDim(0);
      OP_REQUIRES(c, tmp.CopyFrom(tmp, tmp_shape),
                  errors::Unknown("Unexpected shape error."));
      // TODO(apassos) maybe not always align; but weird compiler bugs seem to
      // prevent this.
      Tensor aligned;
      OP_REQUIRES_OK(c, c->allocate_temp(tmp.dtype(), tmp.shape(), &aligned));
      // TODO(apassos) do all slices in a single kernel invocation instead of
      // many small ondes.
      aligned.flat<T>().device(c->eigen_device<Device>()) =
          tmp.unaligned_flat<T>();
      std::swap(output_list.tensors[i], aligned);
    }
    output_tensor->scalar<Variant>()() = std::move(output_list);
  }
};

template <typename Device>
Status TensorListBinaryAdd(OpKernelContext* c, const TensorList& a,
                           const TensorList& b, TensorList* out) {
  if (a.element_dtype != b.element_dtype) {
    return errors::InvalidArgument(
        "Trying to add two lists of tensors of different dtypes. One is ",
        DataTypeString(a.element_dtype), " and the other is ",
        DataTypeString(b.element_dtype));
  }
  out->element_dtype = a.element_dtype;
  if (!a.element_shape.IsCompatibleWith(b.element_shape)) {
    return errors::InvalidArgument(
        "Trying to add two lists of tensors with incompatible element shapes. "
        "One is ",
        a.element_shape.DebugString(), " and the other is ",
        b.element_shape.DebugString());
  }

  TF_RETURN_IF_ERROR(
      a.element_shape.MergeWith(b.element_shape, &out->element_shape));
  if (a.tensors.size() != b.tensors.size()) {
    return errors::InvalidArgument(
        "Trying to add two lists of tensors with different lengths. One is ",
        a.tensors.size(), " and the other is ", b.tensors.size());
  }
  out->tensors.reserve(a.tensors.size());
  for (int i = 0; i < a.tensors.size(); ++i) {
    const Tensor& a_tensor = a.tensors[i];
    const Tensor& b_tensor = b.tensors[i];
    Tensor out_tensor;
    TF_RETURN_IF_ERROR(
        BinaryAddTensors<Device>(c, a_tensor, b_tensor, &out_tensor));
    out->tensors.push_back(out_tensor);
  }
  return Status::OK();
}

template <typename Device>
Status TensorListZerosLike(OpKernelContext* c, const TensorList& x,
                           TensorList* y) {
  y->element_dtype = x.element_dtype;
  y->element_shape = x.element_shape;
  y->tensors.reserve(x.tensors.size());
  for (const Tensor& t : x.tensors) {
    Tensor out_tensor;
    TF_RETURN_IF_ERROR(ZerosLikeTensor<Device>(c, t, &out_tensor));
    y->tensors.emplace_back(out_tensor);
  }
  return Status::OK();
}

template <typename Device, typename T>
class TensorListPushBackBatch : public OpKernel {
 public:
  explicit TensorListPushBackBatch(OpKernelConstruction* c) : OpKernel(c) {
    OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_));
  }

  void Compute(OpKernelContext* c) override {
    const Tensor& input = c->input(1);
    OP_REQUIRES(c, element_dtype_ == input.dtype(),
                errors::InvalidArgument("Invalid data types; list elements ",
                                        DataTypeString(element_dtype_),
                                        " but tried to append ",
                                        DataTypeString(input.dtype())));
    OP_REQUIRES(c, TensorShapeUtils::IsVectorOrHigher(input.shape()),
                errors::InvalidArgument(
                    "Expected tensor to be at least a vector, but saw shape: ",
                    input.shape().DebugString()));

    const TensorShape& tls_shape = c->input(0).shape();

    // For purposes of input forwarding, we want the least restrictive
    // AllocatorAttributes possible.  If we need to allocate later,
    // we'll request the DT_VARIANT be allocated on host.
    AllocatorAttributes attr;

    std::unique_ptr<Tensor> tls_alias = c->forward_input(
        0 /*input_index*/, 0 /*output_index*/, DT_VARIANT, tls_shape,
        DEVICE_MEMORY /* input is always on DEVICE_MEMORY */, attr);

    const Tensor& tls = tls_alias ? *tls_alias : c->input(0);

    OP_REQUIRES(c, tls.dtype() == DT_VARIANT,
                errors::InvalidArgument(
                    "Expected input_handles dtype to be Variant, but saw: ",
                    DataTypeString(tls.dtype())));
    OP_REQUIRES(c, TensorShapeUtils::IsVector(tls_shape),
                errors::InvalidArgument(
                    "Expected input_handles to be a vector, but saw shape: ",
                    tls_shape.DebugString()));
    const int64 batch_size = tls.NumElements();
    OP_REQUIRES(c, input.dim_size(0) == batch_size,
                errors::InvalidArgument(
                    "Expected tensor.shape[0] == input_handles.size, but saw ",
                    input.dim_size(0), " vs. ", batch_size));
    auto tls_t = tls.vec<Variant>();

    TensorShape input_element_shape = input.shape();
    input_element_shape.RemoveDim(0);
    std::vector<const TensorList*> tl_batch;
    for (int64 b = 0; b < batch_size; ++b) {
      const TensorList* l = tls_t(b).get<TensorList>();
      OP_REQUIRES(c, l != nullptr,
                  errors::InvalidArgument("Input handle at index ", b,
                                          " is not a list. Saw: '",
                                          tls_t(b).DebugString(), "'"));
      OP_REQUIRES(
          c, l->element_shape.IsCompatibleWith(input_element_shape),
          errors::InvalidArgument(
              "Tried to append a tensor with incompatible shape to a "
              "list at index ",
              b, ". Op element shape: ", input_element_shape.DebugString(),
              " list shape: ", l->element_shape.DebugString()));
      OP_REQUIRES(c, element_dtype_ == l->element_dtype,
                  errors::InvalidArgument(
                      "Invalid data type at index ", b, "; op elements ",
                      DataTypeString(element_dtype_), " but list elements ",
                      DataTypeString(l->element_dtype)));
      tl_batch.push_back(l);
    }

    Tensor* result;

    if (tls_alias) {
      result = tls_alias.get();
      c->set_output(0, *result);
    } else {
      // DT_VARIANT tensors always allocated on host.
      AllocatorAttributes attr;
      attr.set_on_host(true);
      OP_REQUIRES_OK(
          c, c->allocate_output(0, TensorShape{batch_size}, &result, attr));
    }

    if (batch_size == 0) {
      return;
    }

    auto input_t = input.flat_outer_dims<T, 2>();
    auto result_t = result->vec<Variant>();

    for (int64 b = 0; b < batch_size; ++b) {
      if (!tls_alias) {
        result_t(b) = *tl_batch[b];
      }
      TensorList* output = result_t(b).get<TensorList>();
      DCHECK(output != nullptr);
      Tensor* frame;
      PersistentTensor tmp;
      OP_REQUIRES_OK(c, c->allocate_persistent(
                            element_dtype_, input_element_shape, &tmp, &frame));
      if (input_element_shape.num_elements() > 0) {
        auto frame_t = frame->flat<T>();
        frame_t.device(c->eigen_device<Device>()) = input_t.template chip<0>(b);
      }
      output->tensors.push_back(std::move(*frame));
    }
  }

 private:
  DataType element_dtype_;
};

}  // namespace tensorflow

#endif  // TENSORFLOW_CORE_KERNELS_LIST_KERNELS_H_


text:
namespace fe
{

class GameConstants
{
private:
	static const std::string GameBackgroundTextureFile;
	static const std::string Player1TextureFile;
	static const std::string Player2TextureFile;
	static const std::string SquareTextureFile;
	static const std::string HudFontFile;

	static bool st_Initialized;

public:
	GameConstants();
	GameConstants(const GameConstants& toCopy);
	GameConstants& operator=(const GameConstants& toCopy);
	~GameConstants();

	static sf::Texture GameBackgroundTexture;
	static sf::Texture Player1Texture;
	static sf::Texture Player2Texture;
	static sf::Texture SquareTexture;

	static const sf::Vector2u WindowSize;
	static const sf::Vector2u BoardSize;
	static const sf::Vector2u SquareSize;

	static const sf::Color HudBackgroundColor;
	static const sf::Color HudPlayerNameColor;
	static const sf::Color HudScoreboardColor;
	static const sf::Color HudGameMessageColor;

	static const UI32 HudPlayerNameTextSize;
	static const UI32 HudScoreboardTextSize;
	static const UI32 HudGameMessageTextSize;

	static const sf::Vector2f HudGameMessagePosition;

	static sf::Font HudFont;

	static bool Init();
};

}

#endif

redpajama_set_name: RedPajamaGithub


text:
/* TEMPLATE GENERATED TESTCASE FILE
Filename: CWE23_Relative_Path_Traversal__wchar_t_environment_ifstream_17.cpp
Label Definition File: CWE23_Relative_Path_Traversal.label.xml
Template File: sources-sink-17.tmpl.cpp
*/
/*
 * @description
 * CWE: 23 Relative Path Traversal
 * BadSource: environment Read input from an environment variable
 * GoodSource: Use a fixed file name
 * Sink: ifstream
 *    BadSink : Open the file named in data using ifstream::open()
 * Flow Variant: 17 Control flow: for loops
 *
 * */

#include "std_testcase.h"

#ifdef _WIN32
#define BASEPATH L"c:\\temp\\"
#else
#include <wchar.h>
#define BASEPATH L"/tmp/"
#endif

#define ENV_VARIABLE L"ADD"

#ifdef _WIN32
#define GETENV _wgetenv
#else
#define GETENV getenv
#endif

#include <fstream>
using namespace std;

namespace CWE23_Relative_Path_Traversal__wchar_t_environment_ifstream_17
{

#ifndef OMITBAD

void bad()
{
    int i;
    wchar_t * data;
    wchar_t dataBuffer[FILENAME_MAX] = BASEPATH;
    data = dataBuffer;
    for(i = 0; i < 1; i++)
    {
        {
            /* Append input from an environment variable to data */
            size_t dataLen = wcslen(data);
            wchar_t * environment = GETENV(ENV_VARIABLE);
            /* If there is data in the environment variable */
            if (environment != NULL)
            {
                /* POTENTIAL FLAW: Read data from an environment variable */
                wcsncat(data+dataLen, environment, FILENAME_MAX-dataLen-1);
            }
        }
    }
    {
        ifstream inputFile;
        /* POTENTIAL FLAW: Possibly opening a file without validating the file name or path */
        inputFile.open((char *)data);
        inputFile.close();
    }
}

#endif /* OMITBAD */

#ifndef OMITGOOD

/* goodG2B() - use goodsource in the for statement */
static void goodG2B()
{
    int h;
    wchar_t * data;
    wchar_t dataBuffer[FILENAME_MAX] = BASEPATH;
    data = dataBuffer;
    for(h = 0; h < 1; h++)
    {
        /* FIX: Use a fixed file name */
        wcscat(data, L"file.txt");
    }
    {
        ifstream inputFile;
        /* POTENTIAL FLAW: Possibly opening a file without validating the file name or path */
        inputFile.open((char *)data);
        inputFile.close();
    }
}

void good()
{
    goodG2B();
}

#endif /* OMITGOOD */

} /* close namespace */

/* Below is the main(). It is only used when building this testcase on
   its own for testing or for building a binary to use in testing binary
   analysis tools. It is not used when compiling all the testcases as one
   application, which is how source code analysis tools are tested. */

#ifdef INCLUDEMAIN

using namespace CWE23_Relative_Path_Traversal__wchar_t_environment_ifstream_17; /* so that we can use good and bad easily */

int main(int argc, char * argv[])
{
    /* seed randomness */
    srand( (unsigned)time(NULL) );
#ifndef OMITGOOD
    printLine("Calling good()...");
    good();
    printLine("Finished good()");
#endif /* OMITGOOD */
#ifndef OMITBAD
    printLine("Calling bad()...");
    bad();
    printLine("Finished bad()");
#endif /* OMITBAD */
    return 0;
}

#endif


text:
{% extends "base.html" %}

{% block pagetitle %}Astro-Coffee@Example{% end %}

{% block pageheader %}

{% include "topbar.html" %}

Welcome to Astro-Coffee@Example!

{% end %}

{% block pagecontent %}

Every weekday, the {{ coffee_department }} at {{ coffee_institution }}
      gathers in {{ coffee_building }}'s {{ coffee_room }} to
      discuss the latest arXiv
      astro-ph posting at 10:30 AM over coffee. Visitors to
      the department are always welcome to join these discussions.

This astro-coffee server is meant to
      help organize the discussion. It automatically finds and highlights papers
      from local authors, and allows the community to vote for articles they
      want to talk about. New paper listings are fetched from the arXiv between
      00:25 and 00:35 UTC Sunday through Thursday during the week.

Currently, voting on papers is open between
      8:45 PM the night before and 10:28 AM
      the morning of Astro-coffee. Everyone gets five votes (one per paper) to
      use as they please.

{% end %}

{% block pagejs %}

{% end %}


text:
package me.kingka.utils;

import com.google.protobuf.ByteString;
import io.netty.channel.Channel;
import java.util.Arrays;
import java.util.List;

import me.kingka.locale.I18n;
import me.kingka.message.Message;
import me.kingka.network.data.Value;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.commons.codec.digest.DigestUtils;

/**
 * 辅助工具类
 *
 * @author swift.apple
 */
public class Toolkit {

    /**
     * 创建显示数据对象
     *
     * @param file
     * @return
     */
    public static Value makeValue(java.io.File file) {
        Value value = new Value();
        value.setName(file.getName());
        value.setLength(file.length());
        value.setPath(file.getAbsolutePath());
        value.setId(DigestUtils.md5Hex(StringUtils.getBytesUtf8(file.getAbsolutePath())));
        return value;
    }

    /**
     * 密码是否一致
     *
     * @param password
     * @param comparedPassword
     * @return
     */
    public static boolean isValidPassword(String password, String comparedPassword) {
        return comparedPassword.equalsIgnoreCase(DigestUtils.md5Hex(password));
    }

    /**
     * 发送响应
     *
     * @param channel
     * @param kind
     * @param message
     * @param status
     */
    public static void send(Channel channel, String kind, ByteString message, int status) {
        Message.Response.Builder builder = Message.Response.newBuilder();
        builder.setStatus(status);
        builder.setKind(kind);
        if (message == null) {
            builder.setData(ByteString.EMPTY);
        } else {
            builder.setData(message);
        }
        channel.write(builder.build());
        channel.flush();
    }

    /**
     * 是否是端口格式
     *
     * @param host
     * @return
     */
    public static boolean isPort(String host) {
        String regex = "^(0|[1-9]\\d{0,4})$";
        if (host != null) {
            if (host.matches(regex)) {
                int port = Integer.parseInt(host);
                if (port >= 0 && port <= 65535) {
                    return true;
                }
            }
        }
        return false;
    }

}


text:
package com.gs.obevo.util.vfs;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import org.eclipse.collections.api.list.ListIterable;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.impl.factory.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public enum FileRetrievalMode implements FileResolverStrategy {
    FILE_SYSTEM,
    CLASSPATH,;

    private static final Logger LOG = LoggerFactory.getLogger(FileRetrievalMode.class);

    @Override
    public ListIterable<FileObject> resolveFileObjects(String path) {
        switch (this) {
        case FILE_SYSTEM:
            FileObject fileObject = this.getFileObjectFromFileSystem(path);
            return isValidFileObject(fileObject) ? Lists.mutable.of(fileObject)
                    : Lists.mutable.<FileObject>empty();
        case CLASSPATH:
            return this.getFileObjectsFromClasspath(path);
        default:
            throw new IllegalArgumentException("Not expecting this enum: " + path);
        }
    }

    public FileObject resolveSingleFileObject(String path) {
        ListIterable<FileObject> fileObjects = resolveFileObjects(path);
        if (fileObjects.size() > 1) {
            LOG.warn(String.format("Path [%s] should resolve to only one source directory but found more " +
                    "than one, please investigate...", path));
        }
        return fileObjects.isEmpty() ? null : fileObjects.getFirst();
    }

    private boolean isValidFileObject(FileObject fileObject) {
        return fileObject != null && fileObject.exists();
    }

    private FileObject getFileObjectFromFileSystem(String path) {
        return VFS.getManager().resolveFile("file:" + new File(path).getAbsolutePath());
    }

    private ListIterable<FileObject> getFileObjectsFromClasspath(String path) {
        MutableList<FileObject> fileObjects = Lists.mutable.empty();

        Collection<URL> resources = getResourcesFromClasspath(path);
        for (URL resource : resources) {
            FileObject fileObject = getFileObjectUsingURL(resource);
            if (isValidFileObject(fileObject)) {
                fileObjects.add(fileObject);
            }
        }

        return fileObjects;
    }

    private FileObject getFileObjectUsingURL(URL resource) {
        String filePath = resource.getPath();
        FileObject fileObject = null;
        if (filePath.contains(".jar!")) {
            filePath = cleanFilePath(filePath);
            fileObject = VFS.getManager().resolveFile("jar:" + new File(filePath).getAbsolutePath());
        } else if (filePath.contains(".zip!")) {
            filePath = cleanFilePath(filePath);
            fileObject = VFS.getManager().resolveFile("zip:" + new File(filePath).getAbsolutePath());
        } else {
            fileObject = VFS.getManager().resolveFile("file:" + new File(filePath).getAbsolutePath());
        }
        return fileObject;
    }

    private String cleanFilePath(String filePath) {
        return filePath.substring(filePath.lastIndexOf("file:") + 5);
    }

    private static List<URL> getResourcesFromClasspath(String path) {
        try {
            return Collections.list(classLoader(path).getResources(path));
        } catch (IOException e) {
            throw new VFSFileSystemException(String.format("Error while resolving file using %s", path), e);
        }
    }

    private static ClassLoader classLoader(String resourcePath) {
        if (FileRetrievalMode.class.getClassLoader().getResource(resourcePath) != null) {
            return FileRetrievalMode.class.getClassLoader();
        }

        return Thread.currentThread().getContextClassLoader();
    }

}


text:
<?php

final class PhabricatorPeopleManageProfilePanel
  extends PhabricatorProfilePanel {

  const PANELKEY = 'people.manage';

  public function getPanelTypeName() {
    return pht('Mangage User');
  }

  private function getDefaultName() {
    return pht('Manage');
  }

  public function canHidePanel(
    PhabricatorProfilePanelConfiguration $config) {
    return false;
  }

  public function getDisplayName(
    PhabricatorProfilePanelConfiguration $config) {
    $name = $config->getPanelProperty('name');

    if (strlen($name)) {
      return $name;
    }

    return $this->getDefaultName();
  }

  public function buildEditEngineFields(
    PhabricatorProfilePanelConfiguration $config) {
    return array(
      id(new PhabricatorTextEditField())
        ->setKey('name')
        ->setLabel(pht('Name'))
        ->setPlaceholder($this->getDefaultName())
        ->setValue($config->getPanelProperty('name')),
    );
  }

  protected function newNavigationMenuItems(
    PhabricatorProfilePanelConfiguration $config) {

    $user = $config->getProfileObject();
    $id = $user->getID();

    $item = $this->newItem()
      ->setHref("/people/manage/{$id}/")
      ->setName($this->getDisplayName($config))
      ->setIcon('fa-gears');

    return array(
      $item,
    );
  }

}


text:
module HealthSeven::V2_3
class Mo < ::HealthSeven::DataType
  # quantity
  attribute :quantity, Nm, position: "MO.1"
  # denomination
  attribute :denomination, Id, position: "MO.2"
end
end


text:
**[<= Back](../05-git-and-github/tools-for-development.md)**		*	*	*	**[Next =>](../../01-syntax/00-intro/intro.md)**

## Theory

1. [Github flow model](https://guides.github.com/introduction/flow/)
1. [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/)
1. [Closing issues via commit messages](https://help.github.com/articles/closing-issues-via-commit-messages/)

## Idea

Setup such model of collaboration which would allow constant evolution and improvements of
learning content, tools and processes.

## Technical Details

Follow  open-source like model of collaboration, try to benefit from github flow,
but with some exceptions

### Trainee to Mentor

1. Trainees are doing exercises in their forks and create pull requests to mentor for review
    * [task is completed](http://c2.com/cgi/wiki?TaskCompleteDefinition) only if mentor merged it to his repository and closed corresponding issue
1. Trainee merges his completed tasks from mentor's repo after each milestone
1. Trainees may contribute to learning program by providing pull request to mentor's master branch

### Mentor to Trainee

1. Mentor provides updates in course content
1. Mentor provides additional exercises ontent and send pull requests to trainees repos

## Collaboration Diagram

![Collaboration Diagram](./sf-eng-init-collaboration-figure.png)

## Practice

### Workshop: Contribute to Open Source

> Don't forget to provide links to your projects

[How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github)

### Exercise: Send your code

* Create branch `exercises` from `master`
* Create `hello-world` branch from `exercises` branch
* Write a program which would print `Hello, World!` in a console, file for code [here](./helloWorld.js)
* Send pull request from  `<your-fork>/hello-world` to `<mentor-repo>/<your-githubname>`
* Make sure that issue is wired with your pull request
* Make sure that in our board you have issue+pull-request as a one item

### Exercise: Contribute to Course content

> You shouldn't create separate issue for that exercise, creating pull request with good name would be enough

* Try to find typos in content (there are a lot of them), broken links or provide additional content for course
* Create pull request to `master` branch

### Exercise: Merge mentor's pull request

Just merge mentor's pull request to your `exercise` branch

**[⬆ back to top](#our-model-of-collaboration)**

**[<= Back](../05-git-and-github/tools-for-development.md)**		*	*	*	**[Next =>](../../01-syntax/00-intro/intro.md)**


text:
package io.redlink.smarti.services;

import com.google.common.base.Preconditions;
import io.redlink.smarti.api.event.StoreServiceEvent;
import io.redlink.smarti.exception.BadArgumentException;
import io.redlink.smarti.exception.ConflictException;
import io.redlink.smarti.exception.NotFoundException;
import io.redlink.smarti.model.Client;
import io.redlink.smarti.model.Conversation;
import io.redlink.smarti.model.ConversationMeta;
import io.redlink.smarti.model.ConversationMeta.Status;
import io.redlink.smarti.model.Message;
import io.redlink.smarti.model.Message.Origin;
import io.redlink.smarti.repositories.AnalysisRepository;
import io.redlink.smarti.repositories.ConversationRepository;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.stereotype.Service;

import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.*;

/**
 * Conversation-related services
 */
@Service
public class ConversationService {

    private final Logger log = LoggerFactory.getLogger(ConversationService.class);

    private final ApplicationEventPublisher eventPublisher;
    private final ConversationRepository conversationRepository;
    private final AnalysisRepository analysisRepository;

    public ConversationService(ConversationRepository conversationRepository, Optional<AnalysisRepository> analysisRepository, ApplicationEventPublisher eventPublisher) {
        this.conversationRepository = conversationRepository;
        this.analysisRepository = analysisRepository.orElse(null);
        this.eventPublisher = eventPublisher;
    }

    /**
     * Updates/Stores the parsed conversation
     * @param client the client
     * @param conversation the conversation (MUST BE owned by the parsed client)
     * @return the updated conversation as stored after the update and likely before processing has completed. Use the
     * onCompleteCallback to get the updated conversation including processing results
     */
    public Conversation update(Client client, Conversation conversation) {
        Preconditions.checkNotNull(conversation);
        Preconditions.checkNotNull(client);
        if(conversation.getOwner() == null){
            conversation.setOwner(client.getId());
        } else if(!Objects.equals(client.getId(),conversation.getOwner())){
            throw new IllegalStateException("The parsed Client MUST BE the owner of the conversation!");
        }
        return store(conversation);
    }

    /**
     * Appends a message to the end of the conversation
     * @param onCompleteCallback called after the operation completes (including the optional asynchronous processing)
     * @param conversation the conversation (MUST BE owned by the parsed client)
     * @return the updated conversation as stored after the update and likely before processing has completed. Use the
     * onCompleteCallback to get the updated conversation including processing results
     */
    public Conversation appendMessage(Conversation conversation, Message message) {
        Preconditions.checkNotNull(conversation);
        Preconditions.checkNotNull(message);
        return publishSaveEvent(conversationRepository.appendMessage(conversation, message));
    }

    public Conversation completeConversation(Conversation conversation) {
        Preconditions.checkNotNull(conversation);
        Preconditions.checkNotNull(conversation.getId());
        return updateStatus(conversation.getId(), Status.Complete);
    }

    public Conversation rateMessage(Conversation conversation, String messageId, int delta) {
        Preconditions.checkNotNull(conversation);
        Preconditions.checkNotNull(conversation.getId());
        Preconditions.checkNotNull(messageId);
        if(delta == 0){
            return conversation;
        }
        return publishSaveEvent(conversationRepository.adjustMessageVotes(conversation.getId(), messageId, delta));
    }

    public Conversation getConversation(ObjectId convId){
        return getConversation(null,convId);
    }

    public Conversation getConversation(Client client, ObjectId convId){
        Preconditions.checkNotNull(convId);
        if(client != null){
            return conversationRepository.findByOwnerAndIdAndDeletedIsNull(client.getId(), convId);
        } else {
            return conversationRepository.findOneByIdAndDeletedIsNull(convId);
        }
    }

//    public Conversation getCurrentConversationByChannelId(Client client, String channelId) {
//        return getCurrentConversationByChannelId(client, channelId, Conversation::new);
//    }

//    public Conversation getCurrentConversationByChannelId(Client client, String channelId, Supplier<Conversation> supplier) {
//        Preconditions.checkNotNull(client);
//        Preconditions.checkArgument(StringUtils.isNoneBlank(channelId));
//        final ObjectId conversationId = conversationRepository.findCurrentConversationIDByChannelID(channelId);
//        if (conversationId != null) {
//            Conversation conversation = getConversation(conversationId);
//            //FIXME (Jakob) Should the AuthenticationService care about access. If so this
//            //      shoulc just return the conversation regardless of the owner
//            if(Objects.equals(conversation.getOwner(), client.getId())){
//                return getConversation(client, conversationId);
//            } else {
//                //this should never happen unless we have two clients with the same channelId
//                throw new IllegalStateException("Conversation for channel '" + channelId + "' has a different owner "
//                        + "as the current client (owner: " + conversation.getOwner() + ", client: " + client + ")!");
//            }
//        } else {
//            final Conversation c = supplier.get();
//            if(c != null){
//                c.setId(null);
//                c.setOwner(client.getId());
//                c.setChannelId(channelId);
//                return store(c);
//            } else {
//                return null;
//            }
//        }
//    }

    public Page<Conversation> listConversations(Set<ObjectId> clientIDs, int page, int pageSize) {
        final PageRequest paging = new PageRequest(page, pageSize);
        if (CollectionUtils.isNotEmpty(clientIDs)) {
            return conversationRepository.findByOwnerInAndDeletedIsNull(clientIDs, paging);
        } else {
            return conversationRepository.findByDeletedIsNull(paging);
        }
    }

    public Page<Conversation> listConversations(ObjectId clientId, int page, int pageSize) {
        if (clientId == null) {
            return listConversations(Collections.emptySet(), page, pageSize);
        } else {
            return listConversations(Collections.singleton(clientId), page, pageSize);
        }
    }

    public List<Conversation> getConversations(ObjectId owner) {
        return conversationRepository.findByOwnerAndDeletedIsNull(owner);
    }

    public void importConversations(ObjectId owner, List<Conversation> conversations) {
        importConversations(owner, conversations, false);
    }

    public void importConversations(ObjectId owner, List<Conversation> conversations, boolean replace) {
        // TODO(westei): implement this method
        throw new UnsupportedOperationException("Not yet implemented");
    }

    public boolean exists(ObjectId conversationId) {
        return conversationRepository.existsByDeletedIsNull(conversationId);
    }

    public Conversation updateStatus(ObjectId conversationId, ConversationMeta.Status newStatus) {
        return publishSaveEvent(conversationRepository.updateConversationStatus(conversationId, newStatus));
    }

    public boolean deleteMessage(ObjectId conversationId, String messageId) {
        final boolean success = conversationRepository.deleteMessage(conversationId, messageId);
        if (success) {
            final Conversation one = conversationRepository.findOne(conversationId);
            publishSaveEvent(one);
        }
        return success;
    }

    public Conversation updateMessage(ObjectId conversationId, Message updatedMessage) {
        return publishSaveEvent(conversationRepository.updateMessage(conversationId, updatedMessage));
    }

    private Conversation publishSaveEvent(Conversation conversation) {
        eventPublisher.publishEvent(StoreServiceEvent.save(conversation.getId(), conversation.getMeta().getStatus(), this));
        return conversation;
    }

    public boolean deleteConversation(ObjectId conversationId) {
        Preconditions.checkNotNull(conversationId);
        boolean deleted = conversationRepository.markAsDeleted(conversationId);
        if(deleted){
            eventPublisher.publishEvent(StoreServiceEvent.delete(conversationId, this));
            if(analysisRepository != null){
                try {
                    analysisRepository.deleteByConversation(conversationId);
                } catch (RuntimeException e) {
                    log.debug("Unable to delete storead analysis for deleted conversation[id: {}]", conversationId, e);
                }
            }
        }
        return deleted;
    }
    /**
     * Updates a field of the conversation

     * If the parsed field does not contain '.' the 'meta.' prefix
     * is assumed.

     * supported fields include (format: '{mongo-field} ({json-field-1}, {json-field-2} ...)'

     *
'context.contextType' (context.contextType)
     *
'context.domain' (context.domain)
     *
'context.environment.*' (context.environment.*, environment.*)
     *
'meta.status' (meta.status, status)
     *
'meta.properties.*' (meta.*, *)
     *

     * @param conversationId the id of the conversation (MUST NOT be NULL)
     * @param field the name of the field. See description for supported values (MUST NOT be NULL)
     * @param data the value for the vield (MUST NOT be NULL)
     * @return
     */
    public Conversation updateConversationField(ObjectId conversationId, final String field, Object data) {
        if(conversationId == null){
            throw new NullPointerException();
        }
        if(data == null){
            throw new BadArgumentException("data", null, "the parsed field data MUST NOT be NULL!");
        }
        final String mongoField = toMongoField(field);

        //handle special cases
        if("meta.status".equals(mongoField)){
            //meta status is an enumeration so only some values are allowed
            try {
                return updateStatus(conversationId, data == null ? null : Status.valueOf(data.toString()));
            } catch (IllegalArgumentException | NullPointerException e) {
                throw new BadArgumentException(field, data, "supported values are NULL or " + Status.values());
            }
        } else { //deal the default case
            log.debug("set conversation field {}: {} (parsed field: {})", mongoField, data, field);
            //we need to map fields to conversation paths
            return publishSaveEvent(conversationRepository.updateConversationField(conversationId, mongoField, data));
        }
    }
    public Conversation deleteConversationField(ObjectId conversationId, String field) {
        if(conversationId == null){
            throw new NullPointerException();
        }

        final String mongoField = toMongoField(field);

        //handle special cases
        if("meta.status".equals(mongoField)){
            //one can not delete the status!
            throw new BadArgumentException(field, null, "this field can not be deleted!");
        } else { //deal the default case
            log.debug("delete conversation field {}: {} (parsed field: {})", mongoField,  field);
            //we need to map fields to conversation paths
            return publishSaveEvent(conversationRepository.deleteConversationField(conversationId, mongoField));
        }
    }

    /**
     * Internally used to map a parsed field name to the field path as used in Mongo.

     * If the field does not contain '.' the 'meta.' prefix
     * is assumed.

     * supported fields include (format: '{mongo-field} ({json-field-1}, {json-field-2} ...)'

     *
'context.contextType' (context.contextType)
     *
'context.domain' (context.domain)
     *
'context.environment.*' (context.environment.*, environment.*)
     *
'meta.status' (meta.status, status)
     *
'meta.properties.*' (meta.*, *)
     *

     * @param field the parsed field
     * @return
     */
    private String toMongoField(final String field) {
        if(StringUtils.isBlank(field)){
            throw new BadArgumentException("field", "The parsed field MUST NOT be blank");
        }
        final String jsonField;
        final String mongoField;
        if(field.indexOf('.') < 0){ //no prefix parsed ... use 'meta' as default
            jsonField = "meta." + field;
        } else {
            jsonField = field;
        }
        int sepIdx = jsonField.indexOf('.') + 1; //we do want to cut the '.'
        if(jsonField.startsWith("context.")){
            String fieldName = jsonField.substring(sepIdx);
            if(!("contextType".equals(fieldName) ||
                    "domain".equals(fieldName) ||
                    fieldName.startsWith("environment."))){
                throw new BadArgumentException("field", "Unsupported context field '" + field
                        + "' (unknown context field: '" + fieldName + "' known: contextType, domain, environment.*)");
            }
            mongoField = "context." + fieldName;
        } else if(jsonField.startsWith("meta.")){
            String fieldName = jsonField.substring(sepIdx);
            if("status".equals(fieldName)){
                mongoField = "meta." + fieldName;
            } else {
                mongoField = "meta.properties." + fieldName;
            }
        } else if(jsonField.startsWith("environment.")){
            mongoField = "context." + jsonField;
        } else {
            throw new BadArgumentException("field", "Unsupported field name '" + field
                    + "' (unknown prefix: '" + jsonField.substring(sepIdx) + "' known: context, meta)");
        }
        return mongoField;
    }

    public Message getMessage(ObjectId conversationId, String messageId) {
        return conversationRepository.findMessage(conversationId, messageId);
    }

    public boolean exists(ObjectId conversationId, String messageId) {
        return conversationRepository.exists(conversationId, messageId);
    }
    /**
     * Updates a field of a message within the conversation
     * @param conversationId
     * @param messageId
     * @param field
     * @param data
     * @return The conversation on an update or null of no update was performed
     */
    public Conversation updateMessageField(ObjectId conversationId, String messageId, String field, Object data) {
        if(conversationId == null){
            throw new NullPointerException();
        }
        if(messageId == null){
            throw new NullPointerException();
        }
        if(data == null){
            throw new BadArgumentException("data", null, "the parsed field data MUST NOT be NULL!");
        }
        //TODO validate message field
        if("orign".equals(field)){
            try {
                Origin.valueOf(data.toString());
            } catch (IllegalArgumentException e) {
                throw new BadArgumentException(field, data, "unsupported value (supported: " + Arrays.toString(Origin.values())+ ")");
            }
        } else if("private".equals(field)){
            if(!(data instanceof Boolean)){
                data = Boolean.parseBoolean(data.toString());
            }
            field = "_private";
        } else if("votes".equals(field)){
            if(!(data instanceof Integer)){
                try {
                    data = Integer.parseInt(data.toString());
                } catch (NumberFormatException e) {
                    throw new BadArgumentException(field, data, "not a valid integer");
                }
            }
        } else if("content".equals(field)){
            if(!(data instanceof String)){
                throw new BadArgumentException(field, data, "MUST be a String");
            }
        } else if("time".equals(field)){
            if(data instanceof Long){
                data = new Date((Long)data);
            } else if(data instanceof String){
                try {
                    data = Date.from(Instant.parse(data.toString()));
                } catch (DateTimeParseException e) {
                    try {
                        data = DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT.parse(data.toString());
                    } catch (java.text.ParseException e1) {
                        throw new BadArgumentException(field, data, "not a valid date (supported: long time, ISO date/time format)");
                    }
                }
            } else if(!(data instanceof Date)){
                throw new BadArgumentException(field, data, "not a valid date (supported: long time, ISO date/time format)");
            }
        } else if(field.startsWith("metadata.")){
            throw new BadArgumentException(field, data, "Unknown field (supported: time, origin, content, private, votes, metadata.*)");
        }
        //time, origin, content, private, votes, metadata.*
        final Conversation con = conversationRepository.updateMessageField(conversationId, messageId, field, data);
        if (con != null) {
            publishSaveEvent(con);
        }
        return con;
    }

    protected final Conversation store(Conversation conversation) {
        Preconditions.checkNotNull(conversation);
        Preconditions.checkArgument(conversation.getDeleted() == null, "Unable to store %s as it is marked as deleted", conversation);
        conversation.setLastModified(new Date());
        if(conversation.getId() != null){ //if we update an existing we need to validate the clientId value
            Conversation persisted = getConversation(conversation.getId());
            if(persisted == null){
                throw new NotFoundException(Conversation.class, conversation.getId());
            } else {
                if(conversation.getOwner() == null){
                    conversation.setOwner(persisted.getOwner());
                } else if(!Objects.equals(conversation.getOwner(), persisted.getOwner())){
                    throw new ConflictException(Conversation.class, "clientId", "The clientId MUST NOT be changed for an existing conversation!");
                }
            }
        } else { //create a new conversation
            //TODO: Maybe we should check if the Owner exists
            if(conversation.getOwner() == null){
                throw new ConflictException(Conversation.class, "owner", "The owner MUST NOT be NULL nor empty for a new conversation!");
            }

        }
        return publishSaveEvent(conversationRepository.save(conversation));
    }

    public final Collection<ObjectId> listConversationIDsByUser(String userId) {
        return conversationRepository.findConversationIDsByUser(userId);
    }

    public Conversation adjustMessageVotes(ObjectId conversationId, String messageId, int delta) {
        return publishSaveEvent(conversationRepository.adjustMessageVotes(conversationId, messageId, delta));
    }

    public Iterable<ObjectId> listConversationIDs(){
        return conversationRepository.findConversationIDs();
    }

    public Conversation findLegacyConversation(Client owner, String contextType, String channelId) {
        return conversationRepository.findLegacyConversation(owner.getId(), contextType, channelId);
    }
}


text:
"""
Various utility functions used by gitology.*
"""
# imports # {{{
from django.conf.urls.defaults import patterns
from django.utils import simplejson

import sys, os, textwrap

try:
    import path
except ImportError:
    from gitology import path

import fnmatch, re, path, string
import gzip, bz2
import docutils.writers.html4css1, docutils.core
from odict import OrderedDict as odict
# }}}

class ImproperlyConfigured(Exception): pass

# path2obj # {{{
def path2obj(path):
    from django.core.urlresolvers import get_mod_func
    mod_name, obj_name = get_mod_func(path)
    return getattr(__import__(mod_name, {}, {}, ['']), obj_name)
# }}}

def text_to_html(text_input): return "<pre>%s</pre>" % text_input

# rest_to_html # {{{
def rest_to_html(rest_input, css_path=None):
    """Render ReStructuredText."""
    writer = docutils.writers.html4css1.Writer()
    from gitology.config import settings
    import os.path
    if css_path is None and "DEFAULT_REST_CSS" in settings.DEFAULTS:
        css_path = os.path.expanduser(settings.DEFAULTS.DEFAULT_REST_CSS)
    return docutils.core.publish_parts(
        rest_input, writer_name="html", settings_overrides={
            'stylesheet': css_path,
            'stylesheet_path': None,
            'embed_stylesheet': True
        }
    )['html_body']
    return writer.output
# }}}

# attrdict # {{{
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/361668
class attrdict(dict):
    """A dict whose items can also be accessed as member variables.

    >>> d = attrdict(a=1, b=2)
    >>> d['c'] = 3
    >>> print d.a, d.b, d.c
    1 2 3
    >>> d.b = 10
    >>> print d['b']
    10

    # but be careful, it's easy to hide methods
    >>> print d.get('c')
    3
    >>> d['get'] = 4
    >>> print d.get('a')
    Traceback (most recent call last):
    TypeError: 'int' object is not callable
    """
    def __init__(self, *args, **kwargs):
        dict.__init__(self, *args, **kwargs)
        self.__dict__ = self
# }}}

# NamedObject # {{{
class NamedObject(object):
    def __init__(self, name):
        self._name = name
    def _get_name(self): return self._name
    name = property(
        _get_name, doc="""
            self.name is readonly.

            >>> d = DocumentMeta("somename")
            >>> d.name
            'somename'
            >>> d.name = "new name"
            Traceback (most recent call last):
                ...
            AttributeError: can't set attribute
            >>>
        """
    )
    def __unicode__(self):
        return u"%s(%s)" % ( self.__class__.__name__, self.name )
    __str__ = __unicode__
    __repr__ = __unicode__
# }}}

# get_blog_data # {{{
class list_with_clone(list):
    def _clone(self): return self

global_blog_dict = {}
def get_blog_data(p):
    from gitology.document import Document
    blog = {}
    blog["name"] = p.basename()
    blog["document"] = Document("blogs@%s" % blog["name"])
    if p.basename() == "main": blog["prefix"] = "blog/"
    else: blog["prefix"] = "%s/" % p.basename()
    # posts
    blog["posts"] = odict()
    years = p.dirs()
    years.sort()
    for y in years:
        if y.namebase == "labels": continue
        months = y.dirs()
        months.sort()
        for m in months:
            dates = m.glob("*.lst")
            dates.sort()
            for d in dates:
                for l in d.open().readlines():
                    # format: url document_name timestamp
                    url, document_name, timestamp = l.split(" ", 2)
                    blog["posts"][url] = {
                        'date': timestamp, 'document': Document(document_name),
                    }
                    global_blog_dict[url] = blog
    blog["posts"].reverse()
    # labels
    blog["labels"] = {}
    for l in p.joinpath("labels").glob("*.lst"):
        d = {}
        d["name"] = l.namebase
        d["posts"] = list_with_clone()
        d["document"] = Document("blogs@%s@label@%s" % (blog["name"], l.namebase))
        for line in l.open().readlines():
            # format: url, data is in respective archive file
            d["posts"].append(blog["posts"][line.strip()])
            blog["posts"][line.strip()].setdefault("labels", []).append(d)
        blog["labels"][l.namebase] = d
    return blog
# }}}

# get_blog {{{
def get_blog(p):
    urls = []
    b = get_blog_data(p)
    urls.append(
        ("^%s$" % b["prefix"], "gitology.d.views.show_blog", { 'blog_data': b,})
    )
    urls.append(
        (
            "^%slabelled/(?P<label_name>[^/]+)/$" % b["prefix"],
            "gitology.d.views.show_category", { 'blog_data': b },
        )
    )
    from django.contrib.syndication.feeds import Feed
    class LatestEntries(Feed):
        title = b["document"].meta.title
        link = b["document"].meta.title
        description = b["document"].meta.subtitle

        def items(self):
            return b["posts"].values()[:10]

        def item_link(self, item):
            return item["document"].meta.url

    feeds = { 'latest': LatestEntries }

    urls.append(
        (
            '^%sfeeds/(?P<url>.*)/$' % b["prefix"],
            'django.contrib.syndication.views.feed',
            {'feed_dict': feeds},
        )
    )
    return urls
# }}}

# get_blogs # {{{
def get_blogs():
    from gitology.config import settings as gsettings
    urls = []
    blogs_folder = gsettings.LOCAL_REPO_PATH.joinpath("blogs")
    for d in blogs_folder.dirs():
        urls += get_blog(d)
    return urls
# }}}

# get_wiki # {{{
global_wiki_dict = {}
def get_wiki():
    from gitology.config import settings as gsettings
    from gitology.document import Document
    urls = []
    wiki_folder = gsettings.LOCAL_REPO_PATH.joinpath("wiki")
    for i in wiki_folder.walk():
        if not i.isfile(): continue
        wiki_document = Document(i.open().read().strip())
        wiki_url = i[len(wiki_folder):-4] + "/"
        global_wiki_dict[wiki_url] = wiki_document
    return urls
# }}}

# get redirects # {{{
global_redirect_dict = {}
def get_redirects():
    from gitology.config import settings as gsettings
    redirects_file = gsettings.LOCAL_REPO_PATH.joinpath("redirects.lst")
    if not redirects_file.exists(): return
    for line in redirects_file.open().readlines():
        try:
            before, after = line.split()
        except ValueError:
            print "Bad redirect line: ", line
            continue
        global_redirect_dict[before] = after
# }}}

# refresh_urlconf_cache # {{{
def refresh_urlconf_cache():
    print "refresh_urlconf_cache"
    from gitology.config import settings
    """ creates a urlconf that is stored """
    global global_wiki_dict, global_blog_dict, global_redirect_dict
    global_blog_dict = {}
    global_wiki_dict = {}
    global_redirect_dict = {}
    urls = ['']
    # for blog:
    # list of blogs is in $reporoot/blogs/
    # urls: /blog_name/
    # blog named "main" goes under /blog/, rest of them go to /folder_name/
    urls += get_blogs()
    urls += get_wiki()
    get_redirects()
    # for each blog, list of labels in $reporoot/blogs/blog_name/labels/
    # urls: /blog_name/label/label_name/
    # for each blog, date based heirarchy is kept in
    # $reporoot/blogs/blog_name/year/month/date.lst
    # /blog_name/year/month/date/document_name/

    # for wiki:
    # list of wiki document names are in $reporoot/wiki/document_alias.txt
    # urls: /document_alias/
    # further heirarchy is maintained:
    # $reporoot/wiki/document_alias/child_alias.txt
    # /document_alias/child_alias/

    # for notes:
    # notebooks are stored in $reporoot/notebooks/
    # urls: /notebook/ this is a dedicated app

    # for albums:
    # list of albums are in $reporoot/albums/ album_name.meta, album_name.lst
    # urls: /album|gallery/album_name/ this is document. it can contain select
    # few photos etc.
    # each photo is basically a document, its list is in the album_name.lst
    # each album photo ll have a thumbnail and caption meta data to be shown
    # on album page/
    # /album/album_name/photos/ will list all photos, each photo may be in one
    # or more albums. each photo can be a blog post too in one or more blogs

    # optimization: this info will be loaded from a file, and some other tool
    # is to update this file everytime something interesting happens.

    # this function returns a dict containing url to view mapping.
    file(settings.LOCAL_REPO_PATH.joinpath("urlconf.cache"), "w+").write(
        #simplejson.dumps(urls)
        "updated"
    )
    #print urls
    return urls
# }}}

# sort_nicely # {{{
# http://nedbatchelder.com/blog/200712/human_sorting.html
import re
def tryint(s):
    try: return int(s)
    except: return s

def alphanum_key(s):
    """ Turn a string into a list of string and number chunks.
        "z23a" -> ["z", 23, "a"]
    """
    return [ tryint(c) for c in re.split('([0-9]+)', s) ]

def sort_nicely(l):
    """ Sort the given list in the way that humans expect.
    """
    l.sort(key=alphanum_key)
# }}}

# smart_wrap # {{{
def smart_wrap(s, width=None):
    if width is None:
        width = int(os.environ.get("COLUMNS", 70))
    lines = []
    for line in s.split("\n"):
        if not line.split(): lines.append("")
        else: lines += textwrap.wrap(line, width=width)
    return "\n".join(lines)
# }}}

# getDirSize {{{
# stolen from: http://mail.python.org/pipermail/python-list/2000-June/037460.html
def calcDirSize(arg, dir, files):
    for file in files:
        stats = os.stat(os.path.join(dir, file))
        size = stats[6]
        arg.append(size)

def getDirSize(dir):
    sizes = []
    os.path.walk(dir, calcDirSize, sizes)
    total = 0
    for size in sizes:
        total = total + size
    if total > 1073741824:
        return (round(total/1073741824.0, 2), 'GB')
    if total > 1048576:
        return (round(total/1048576.0, 2), 'MB')
    if total > 1024:
        return (round(total/1024.0, 2), 'KB')
    return (total, 'bytes')
# }}}

# generators from http://www.dabeaz.com/generators/ # {{{
# these generators are one time use only

# use http://www.fiber-space.de/generator_tools/doc/generator_tools.html
# for copying generators.

def gen_find(filepat,top=""):
    if not top: top = path.path(".").abspath()
    top = os.path.expanduser(top)
    for path_, dirlist, filelist in os.walk(top):
        for name in fnmatch.filter(filelist, filepat):
            yield os.path.join(path_, name)

def gen_exclude(pattern, lines):
    patter = re.compile(pattern)
    for line in lines:
        if not re.search(pattern, line):
            yield line

def gen_grep(pattern, lines):
    patter = re.compile(pattern)
    for line in lines:
        if re.search(pattern, line):
            yield line

def gen_open(filenames):
    for name in filenames:
        if name.endswith(".gz"):
              yield gzip.open(name)
        elif name.endswith(".bz2"):
              yield bz2.BZ2File(name)
        else:
              yield open(name)

def gen_cat(sources):
    for s in sources:
        for item in s:
            yield item

def counter(gen):
    c = 0
    for item in gen: c += 1
    return c
# }}}

# is_valid_url # {{{
allowed_chars_in_urls = string.ascii_letters + string.digits + ".-_/"
def is_valid_url(url):
    if not url.startswith("/"): return False
    for c in url:
        if c not in allowed_chars_in_urls:
            return False
    # consecutive // not allowed
    for p in url.split("/")[1:-1]:
        if not p: return False
    return True
# }}}

# parse_date # {{{
def parse_date(s):
    from django.db.models.fields import DateTimeField
    dtf = DateTimeField()
    return dtf.to_python(s)
# }}}


text:
package org.dozer.vo.bidirectional;

public class A {
  private B field1;

  public B getField1() {
    return field1;
  }

  public void setField1(B field1) {
    this.field1 = field1;
  }

}


text:
/*globals EmberDev */

import run from "ember-metal/run_loop";
import Application from "ember-application/system/application";
import View from "ember-views/views/view";
import Controller from "ember-runtime/controllers/controller";
import Route from "ember-routing/system/route";
import RSVP from "ember-runtime/ext/rsvp";
import keys from "ember-metal/keys";
import compile from "ember-template-compiler/system/compile";

import "ember-routing";

var App, logs, originalLogger;

QUnit.module("Ember.Application – logging of generated classes", {
  setup() {
    logs = {};

    originalLogger = Ember.Logger.info;

    Ember.Logger.info = function() {
      var fullName = arguments[1].fullName;

      logs[fullName] = logs[fullName] || 0;
      logs[fullName]++;
    };

    run(function() {
      App = Application.create({
        LOG_ACTIVE_GENERATION: true
      });

      App.Router.reopen({
        location: 'none'
      });

      App.Router.map(function() {
        this.resource("posts");
      });

      App.deferReadiness();
    });
  },

  teardown() {
    Ember.Logger.info = originalLogger;

    run(App, 'destroy');

    logs = App = null;
  }
});

function visit(path) {
  QUnit.stop();

  var promise = run(function() {
    return new RSVP.Promise(function(resolve, reject) {
      var router = App.__container__.lookup('router:main');

      resolve(router.handleURL(path).then(function(value) {
        QUnit.start();
        ok(true, 'visited: `' + path + '`');
        return value;
      }, function(reason) {
        QUnit.start();
        ok(false, 'failed to visit:`' + path + '` reason: `' + QUnit.jsDump.parse(reason));
        throw reason;
      }));
    });
  });

  return {
    then(resolve, reject) {
      run(promise, 'then', resolve, reject);
    }
  };
}

QUnit.test("log class generation if logging enabled", function() {
  if (EmberDev && EmberDev.runningProdBuild) {
    ok(true, 'Logging does not occur in production builds');
    return;
  }

  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    equal(Ember.keys(logs).length, 6, 'expected logs');
  });
});

QUnit.test("do NOT log class generation if logging disabled", function() {
  App.reopen({
    LOG_ACTIVE_GENERATION: false
  });

  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    equal(keys(logs).length, 0, 'expected no logs');
  });
});

QUnit.test("actively generated classes get logged", function() {
  if (EmberDev && EmberDev.runningProdBuild) {
    ok(true, 'Logging does not occur in production builds');
    return;
  }

  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    equal(logs['controller:application'], 1, 'expected: ApplicationController was generated');
    equal(logs['controller:posts'], 1, 'expected: PostsController was generated');

    equal(logs['route:application'], 1, 'expected: ApplicationRoute was generated');
    equal(logs['route:posts'], 1, 'expected: PostsRoute was generated');
  });
});

QUnit.test("predefined classes do not get logged", function() {
  App.ApplicationController = Controller.extend();
  App.PostsController = Controller.extend();

  App.ApplicationRoute = Route.extend();
  App.PostsRoute = Route.extend();

  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    ok(!logs['controller:application'], 'did not expect: ApplicationController was generated');
    ok(!logs['controller:posts'], 'did not expect: PostsController was generated');

    ok(!logs['route:application'], 'did not expect: ApplicationRoute was generated');
    ok(!logs['route:posts'], 'did not expect: PostsRoute was generated');
  });
});

QUnit.module("Ember.Application – logging of view lookups", {
  setup() {
    logs = {};

    originalLogger = Ember.Logger.info;

    Ember.Logger.info = function() {
      var fullName = arguments[1].fullName;

      logs[fullName] = logs[fullName] || 0;
      logs[fullName]++;
    };

    run(function() {
      App = Application.create({
        LOG_VIEW_LOOKUPS: true
      });

      App.Router.reopen({
        location: 'none'
      });

      App.Router.map(function() {
        this.resource("posts");
      });

      App.deferReadiness();
    });
  },

  teardown() {
    Ember.Logger.info = originalLogger;

    run(App, 'destroy');

    logs = App = null;
  }
});

QUnit.test("log when template and view are missing when flag is active", function() {
  if (EmberDev && EmberDev.runningProdBuild) {
    ok(true, 'Logging does not occur in production builds');
    return;
  }

  App.register('template:application', compile("{{outlet}}"));
  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    equal(logs['template:application'], undefined, 'expected: Should not log template:application since it exists.');
    equal(logs['template:index'], 1, 'expected: Could not find "index" template or view.');
    equal(logs['template:posts'], 1, 'expected: Could not find "posts" template or view.');
  });
});

QUnit.test("do not log when template and view are missing when flag is not true", function() {
  App.reopen({
    LOG_VIEW_LOOKUPS: false
  });

  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    equal(keys(logs).length, 0, 'expected no logs');
  });
});

QUnit.test("log which view is used with a template", function() {
  if (EmberDev && EmberDev.runningProdBuild) {
    ok(true, 'Logging does not occur in production builds');
    return;
  }

  App.register('template:application', compile('{{outlet}}'));
  App.register('template:foo', compile('Template with custom view'));
  App.register('view:posts', View.extend({ templateName: 'foo' }));
  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    equal(logs['view:application'], 1, 'toplevel view always get an element');
    equal(logs['view:index'], undefined, 'expected: Should not log when index is not present.');
    equal(logs['view:posts'], 1, 'expected: Rendering posts with PostsView.');
  });
});

QUnit.test("do not log which views are used with templates when flag is not true", function() {
  App.reopen({
    LOG_VIEW_LOOKUPS: false
  });

  run(App, 'advanceReadiness');

  visit('/posts').then(function() {
    equal(keys(logs).length, 0, 'expected no logs');
  });
});