text:
#include <assert.h>
#include <iostream>
#include <utility>
#include <vector>
#include <initializer_list>

#include "../../chunkedseq/include/iterator.hpp"
#include "../../chunkedseq/include/chunkedseqextras.hpp"

#ifndef _PASL_DATA_CHUNKEDSEQBASE_H_
#define _PASL_DATA_CHUNKEDSEQBASE_H_

namespace pasl {
namespace data {
namespace chunkedseq {

/***********************************************************************/

/*!
 * \class chunkedseqbase
 * \tparam Configuration type of the class from which to access configuration
 * parameters for the container
 * \tparam Iterator type of the class to represent the iterator
 *

 later: explain the following preconditions
  Assume "Chunk" to implement fixed-capacity circular buffers.
  Assume "Topchunk_capacity >= 2" and "Recchunk_capacity >= 2".
  Assume TopItem to have a trivial destructor.

 later: study in detail the memory usage behavior of chunkedseq
  - in particular, study the alternative chunk-compaction policies
    we implemented previously in fftree; these policies still
    need to be implemented here
  - implement the bag mode
  - analyze memory usage via malloc_count

 */
template <
  class Configuration,
  template <
    class Chunkedseq,
    class Configuration1
  >
  class Iterator=iterator::random_access
>
class chunkedseqbase {
protected:

  using chunk_type = typename Configuration::chunk_type;
  using chunk_search_type = typename Configuration::chunk_search_type;
  using chunk_cache_type = typename Configuration::chunk_cache_type;
  using chunk_measured_type = typename chunk_cache_type::measured_type;
  using chunk_algebra_type = typename chunk_cache_type::algebra_type;
  using chunk_measure_type = typename chunk_cache_type::measure_type;
  using chunk_pointer = chunk_type*;

  using middle_type = typename Configuration::middle_type;
  using middle_cache_type = typename Configuration::middle_cache_type;
  using middle_measured_type = typename middle_cache_type::measured_type;
  using middle_algebra_type = typename middle_cache_type::algebra_type;
  using middle_measure_type = typename middle_cache_type::measure_type;
  using size_access = typename Configuration::size_access;

  using const_self_pointer_type = const chunkedseqbase<Configuration>*;

  static constexpr int chunk_capacity = Configuration::chunk_capacity;

public:

  /*---------------------------------------------------------------------*/
  /** @name Container-configuration types
   */
  ///@{
  using config_type = Configuration;
  using self_type = chunkedseqbase<config_type>;
  using size_type = typename config_type::size_type;
  using difference_type = typename config_type::difference_type;
  using allocator_type = typename config_type::item_allocator_type;
  ///@}

  /*---------------------------------------------------------------------*/
  /** @name Item-specific types
   */
  ///@{
  using value_type = typename config_type::value_type;
  using reference = value_type&;
  using const_reference = const value_type&;
  using pointer = value_type*;
  using const_pointer = const value_type*;
  using segment_type = typename config_type::segment_type;
  ///@}

  /*---------------------------------------------------------------------*/
  /** @name Cached-measure-specific types
   */
  ///@{
  using cache_type = chunk_cache_type;
  using measured_type = typename cache_type::measured_type;
  using algebra_type = typename cache_type::algebra_type;
  using measure_type = typename cache_type::measure_type;
  ///@}

  using iterator = Iterator<self_type, config_type>;
  friend iterator;

private:

  // representation of the structure: four chunks plus a middle sequence of chunks;
  // for efficient implementation of emptiness test, additional invariants w.r.t. the paper:
  //  - if front_outer is empty, then front_inner is empty
  //  - if back_outer is empty, then back_inner is empty
  //  - if front_outer and back_outer are empty, then middle is empty
  chunk_type front_inner, back_inner, front_outer, back_outer;
  std::unique_ptr<middle_type> middle;

  chunk_measure_type chunk_meas;
  middle_measure_type middle_meas;

  /*---------------------------------------------------------------------*/

  static inline chunk_pointer chunk_alloc() {
    return new chunk_type();
  }

  // only to free empty chunks
  static inline void chunk_free(chunk_pointer c) {
    assert(c->empty());
    delete c;
  }

  template <class Pred>
  middle_measured_type chunk_split(const Pred& p, middle_measured_type prefix,
                                   chunk_type& src, reference x, chunk_type& dst) {
    chunk_search_type chunk_search;
    return src.split(chunk_meas, p, chunk_search, middle_meas, prefix, x, dst);
  }

  /*---------------------------------------------------------------------*/

  bool is_buffer(const chunk_type* c) const {
    return c == &front_outer || c == &front_inner
        || c == &back_inner  || c == &back_outer;
  }

  // ensures that if front_outer is empty, then front_inner and middle (and back_inner)
  // are all empty
  void restore_front_outer_empty_other_empty() {
    if (front_outer.empty()) {
      if (! front_inner.empty()) {
        front_inner.swap(front_outer);
      } else if (! middle->empty()) {
        chunk_pointer c = middle->pop_front(middle_meas);
        front_outer.swap(*c);
        chunk_free(c);
      } else if (! back_inner.empty()) {
        back_inner.swap(front_outer); // optional operation
      }
    }
    assert(! front_outer.empty()
       || (front_inner.empty() && middle->empty() && back_inner.empty()) );
  }

  void restore_back_outer_empty_other_empty() {
    if (back_outer.empty()) {
      if (! back_inner.empty()) {
        back_inner.swap(back_outer);
      } else if (! middle->empty()) {
        chunk_pointer c = middle->pop_back(middle_meas);
        back_outer.swap(*c);
        chunk_free(c);
      } else if (! front_inner.empty()) {
        front_inner.swap(back_outer); // optional operation
      }
    }
    assert(! back_outer.empty()
       || (back_inner.empty() && middle->empty() && front_inner.empty()) );
  }

  // ensures that if the container is nonempty, then so is the front-outer buffer
  void ensure_front_outer_nonempty() {
    restore_front_outer_empty_other_empty();
    if (front_outer.empty() && ! back_outer.empty())
      front_outer.swap(back_outer);
    assert(empty() || ! front_outer.empty());
  }

  void ensure_back_outer_nonempty() {
    restore_back_outer_empty_other_empty();
    if (back_outer.empty() && ! front_outer.empty())
      back_outer.swap(front_outer);
    assert(empty() || ! back_outer.empty());
  }

  // assumption: invariant "both outer empty implies middle empty" may be broken;
  // calling this function restores it.
  void restore_both_outer_empty_middle_empty() {
    if (front_outer.empty() && back_outer.empty() && ! middle->empty()) {
      // pop to the front (to the back would also work)
      chunk_pointer c = middle->pop_front(middle_meas);
      front_outer.swap(*c);
      chunk_free(c);
    }
  }

  // ensures that inner buffers are empty, by pushing them in the middle if full
  void ensure_empty_inner() {
    if (! front_inner.empty())
      push_buffer_front_force(front_inner);
    if (! back_inner.empty())
      push_buffer_back_force(back_inner);
  }

  using const_chunk_pointer = const chunk_type*;

  const_chunk_pointer get_chunk_containing_last_item() const {
    if (! back_outer.empty())
      return &back_outer;
    if (! back_inner.empty())
      return &back_inner;
    if (! middle->empty())
#ifdef DEBUG_MIDDLE_SEQUENCE
      return middle->back();
#else
      return middle->cback();
#endif
    if (! front_inner.empty())
      return &front_inner;
    return &front_outer;
  }

  using position_type = enum {
    found_front_outer,
    found_front_inner,
    found_middle,
    found_back_inner,
    found_back_outer,
    found_nowhere
  };

  template <class Pred>
  middle_measured_type search(const Pred& p, middle_measured_type prefix,
                              position_type& pos) const {
    middle_measured_type cur = prefix; // prefix including current chunk
    if (! front_outer.empty()) {
      prefix = cur;
      cur = middle_algebra_type::combine(cur, middle_meas(&front_outer));
      if (p(cur)) {
        pos = found_front_outer;
        return prefix;
      }
    }
    if (! front_inner.empty()) {
      prefix = cur;
      cur = middle_algebra_type::combine(cur, middle_meas(&front_inner));
      if (p(cur)) {
        pos = found_front_inner;
        return prefix;
      }
    }
    if (! middle->empty()) {
      prefix = cur;
      cur = middle_algebra_type::combine(cur, middle->get_cached());
      if (p(cur)) {
        pos = found_middle;
        return prefix;
      }
    }
    if (! back_inner.empty()) {
      prefix = cur;
      cur = middle_algebra_type::combine(cur, middle_meas(&back_inner));
      if (p(cur)) {
        pos = found_back_inner;
        return prefix;
      }
    }
    if (! back_outer.empty()) {
      prefix = cur;
      cur = middle_algebra_type::combine(cur, middle_meas(&back_outer));
      if (p(cur)) {
        pos = found_back_outer;
        return prefix;
      }
    }
    prefix = cur;
    pos = found_nowhere;
    return prefix;
  }

  // set "found" to true or false depending on success of the search;
  // and set "cur" to the chunk that contains the item searched,
  // or to the chunk containing the last item of the sequence if the item was not found.
  // if finger search is enabled, then the routine checks whether `cur` contains a
  // null pointer or a pointer to a chunk; in the latter case, search starts from
  // the chunk
  // precondition: `cur` contains either a `nullptr` or a pointer to a "top chunk";
  // i.e., a leaf node of the middle sequence
  template <class Pred>
  middle_measured_type search_for_chunk(const Pred& p, middle_measured_type prefix,
                                        bool& found, const_chunk_pointer& cur) const {
    position_type pos;
    prefix = search(p, prefix, pos);
    found = true; // default
    switch (pos) {
      case found_front_outer: {
        cur = &front_outer;
        break;
      }
      case found_front_inner: {
        cur = &front_inner;
        break;
      }
      case found_middle: {
#ifdef DEBUG_MIDDLE_SEQUENCE
        assert(false);
#else
        prefix = middle->search_for_chunk(p, prefix, cur);
#endif
        break;
      }
      case found_back_inner: {
        cur = &back_inner;
        break;
      }
      case found_back_outer: {
        cur = &back_outer;
        break;
      }
      case found_nowhere: {
        cur = &back_outer;
        found = false;
        break;
      }
    } // end switch
    return prefix;
  }

  // precondition: other is empty
  template <class Pred>
  middle_measured_type split_aux(const Pred& p, middle_measured_type prefix, reference x, self_type& other) {
    assert(other.empty());
    copy_measure_to(other);
    ensure_empty_inner();
    position_type pos;
    prefix = search(p, prefix, pos);
    switch (pos) {
      case found_front_outer: {
        prefix = chunk_split(p, prefix, front_outer, x, other.front_outer);
        middle.swap(other.middle);
        back_outer.swap(other.back_outer);
        break;
      }
      case found_front_inner: {
        assert(false); // thanks to the call to ensure_empty_inner()
        break;
      }
      case found_middle: {
        back_outer.swap(other.back_outer);
        chunk_pointer c;
        prefix = middle->split(middle_meas, p, prefix, c, *other.middle);
        back_outer.swap(*c);
        chunk_free(c);
        prefix = chunk_split(p, prefix, back_outer, x, other.front_outer);
        break;
      }
      case found_back_inner: {
        assert(false); // thanks to the call to ensure_empty_inner()
        break;
      }
      case found_back_outer: {
        prefix = chunk_split(p, prefix, back_outer, x, other.back_outer);
        break;
      }
      case found_nowhere: {
        // don't split (item not found)
        break;
      }
    } // end switch
    restore_both_outer_empty_middle_empty();
    other.restore_both_outer_empty_middle_empty();
    return prefix;
  }

  // precondition: other is empty
  template <class Pred>
  middle_measured_type split_aux(const Pred& p, middle_measured_type prefix, self_type& other) {
    size_type sz_orig = size();
    value_type x;
    prefix = split_aux(p, prefix, x, other);
    if (size_access::csize(prefix) < sz_orig)
      other.push_front(x);
    return prefix;
  }

  /*---------------------------------------------------------------------*/

  // take a chunk "c" and push its content into the back of the middle sequence
  // as a new chunk; leaving "c" empty.
  void push_buffer_back_force(chunk_type& c) {
    chunk_pointer d = chunk_alloc();
    c.swap(*d);
    middle->push_back(middle_meas, d);
  }

  // symmetric to push_buffer_back
  void push_buffer_front_force(chunk_type& c) {
    chunk_pointer d = chunk_alloc();
    c.swap(*d);
    middle->push_front(middle_meas, d);
  }

  // take a chunk "c" and concatenate its content into the back of the middle sequence
  // leaving "c" empty.
  void push_buffer_back(chunk_type& c) {
    size_t csize = c.size();
    if (csize == 0) {
      // do nothing
    } else if (middle->empty()) {
      push_buffer_back_force(c);
    } else {
      chunk_pointer b = middle->back();
      size_t bsize = b->size();
      if (bsize + csize > chunk_capacity) {
        push_buffer_back_force(c);
      } else {
        middle->pop_back(middle_meas);
        c.transfer_from_front_to_back(chunk_meas, *b, csize);
        middle->push_back(middle_meas, b);
      }
    }
  }

  // symmetric to push_buffer_back
  void push_buffer_front(chunk_type& c) {
    size_t csize = c.size();
    if (csize == 0) {
      // do nothing
    } else if (middle->empty()) {
      push_buffer_front_force(c);
    } else {
      chunk_pointer b = middle->front();
      size_t bsize = b->size();
      if (bsize + csize > chunk_capacity) {
        push_buffer_front_force(c);
      } else {
        middle->pop_front(middle_meas);
        c.transfer_from_back_to_front(chunk_meas, *b, csize);
        middle->push_front(middle_meas, b);
      }
    }
  }

  void init() {
    middle.reset(new middle_type());
  }

public:

  /*---------------------------------------------------------------------*/
  /** @name Constructors
   */
  ///@{

  /*!
   * \brief Empty container constructor
   *
   * Constructs an empty container, with no items.
   *
   * #### Complexity ####
   * Constant
   *
   */
  chunkedseqbase() {
    init();
  }

  /*!
   * \brief Fill constructor
   *
   * Constructs an empty container, with `nb` items.
   * Each item is a copy of `val`.
   *
   * \param nb Number of items to put in the container
   * \param val Value to replicate
   *
   * #### Complexity ####
   * Linear in `nb`.
   *
   */
  chunkedseqbase(size_type n, const value_type& val) {
    init();
    std::vector<value_type> vals(chunk_capacity, val);
    const value_type* p = vals.data();
    auto prod = [&] (size_type, size_type nb) {
      return std::make_pair(p, p + nb);
    };
    stream_pushn_back(prod, n);
  }

  /*!
   * \brief Copy constructor
   *
   * Constructs a container with a copy of each of the
   * elements in `other`, in the same order.
   *
   * \param other Container from which to copy.
   *
   * #### Complexity ####
   * Linear in the resulting container size.
   *
   */
  chunkedseqbase(const self_type& other)
  : front_outer(other.front_outer),
    front_inner(other.front_inner),
    back_inner(other.back_inner),
    back_outer(other.back_outer),
    chunk_meas(other.chunk_meas),
    middle_meas(other.middle_meas) {
    middle.reset(new middle_type(*other.middle));
    check();
  }
  ///@}

  chunkedseqbase(std::initializer_list<value_type> l) {
    init();
    for (auto it = l.begin(); it != l.end(); it++)
      push_back(*it);
  }

  /*---------------------------------------------------------------------*/
  /** @name Capacity
   */
  ///@{

  /*!
   * \brief Test whether the container is empty
   *
   * Returns whether the container is empty (
   * i.e. whether its size is 0).
   *
   * #### Complexity ####
   * Constant time.
   *
   */
  bool empty() const {
    return front_outer.empty() && back_outer.empty();
  }

  /*!
   * \brief Returns size
   *
   * Returns the number of items in the container.
   *
   *
   * #### Complexity ####
   * Constant time.
   *
   */
  size_type size() const {
    size_type sz = 0;
    sz += front_outer.size();
    sz += front_inner.size();
    sz += size_access::csize(middle->get_cached());
    sz += back_inner.size();
    sz += back_outer.size();
    return sz;
  }
  ///@}

  /*---------------------------------------------------------------------*/
  /** @name Item access
   */
  ///@{

  ///@{
  /*!
   * \brief Accesses last item
   *
   * Returns a reference to the last item in the container.
   *
   * Calling this method on an `empty` container causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Amortized constant time (worst case logarithmic time).
   *
   *
   * \return A reference to the last item in the
   * container.
   *
   */
  value_type front() const {
    assert(! front_outer.empty() || front_inner.empty());
    value_type v;
    if (! front_outer.empty()) {
      return front_outer.front();
    } else if (! back_inner.empty()) {
      return back_inner.front();
    } else {
      assert(! back_outer.empty());
      return back_outer.front();
    }
  }

  /*!
   * \brief Accesses first item
   *
   * Returns a reference to the first item in the container.
   *
   * Calling this method on an `empty` container causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Amortized constant time (worst case logarithmic time).
   *
   * \return A reference to the first item in the
   * container.
   *
   */
  value_type back() const {
    assert(! back_outer.empty() || back_inner.empty());
    if (! back_outer.empty()) {
      return back_outer.back();
    } else if (! front_inner.empty()) {
      return front_inner.back();
    } else {
      assert(! front_outer.empty());
      return front_outer.back();
    }
  }

  /*!
   * \brief Access last items
   *
   * Copies the last `nb` items from the container to the
   * cells in the array `dst`.
   *
   * Calling this method when `size() < nb` causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Linear in the number of items being copied (i.e., `nb`).
   *
   */
  void backn(value_type* dst, size_type nb) const {
    extras::backn(*this, dst, nb);
  }

  /*!
   * \brief Access first items
   *
   * Copies the first `nb` items from the container to the
   * cells in the array `dst`.
   *
   * Calling this method when `size() < nb` causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Linear in the number of items being copied (i.e., `nb`).
   *
   */
  void frontn(value_type* dst, size_type nb) const {
    extras::frontn(*this, dst, nb);
  }

  /*!
   * \brief Consume last items
   *
   * Streams the first `nb` items from the container to the
   * client-supplied function `cons`.
   *
   * Let our container be `c = [c_0, c_1, ..., c_n]`. Then,
   * the call `stream_backn(cons, nb)` performs:
   *
   *       for (size_type i = nb-1; i < n+1; i++)
   *          cons(c_i);
   *
   * Calling this method when `size() < nb` causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Linear in the number of items being copied (i.e., `nb`).
   *
   */
  template <class Consumer>
  void stream_backn(const Consumer& cons, size_type nb) const {
    extras::stream_backn(*this, cons, nb);
  }

  /*!
   * \brief Consume first items
   *
   * Streams the first `nb` items from the container to the
   * client-supplied function `cons`.
   *
   * Let our container be `c = [c_0, c_1, ..., c_n]`. Then,
   * the call `stream_backn(cons, nb)` performs:
   *
   *       for (size_type i = 0; i < nb; i++)
   *          cons(c_i);
   *
   * Calling this method when `size() < nb` causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Linear in the number of items being copied (i.e., `nb`).
   *
   */
  template <class Consumer>
  void stream_frontn(const Consumer& cons, size_type nb) const {
    extras::stream_frontn(*this, cons, nb);
  }

  /*!
   * \brief Access item
   *
   * Returns a copy of the element at position `n` in
   * the vector container.
   *
   * \return The item at the specified position in the container.
   *
   * \param Position of an item in the container.
   * Notice that the first item has a position of 0 (not 1).
   * Member type size_type is an unsigned integral type.
   *
   * #### Complexity ####
   * Logarithmic time.
   *
   *
   */
  value_type operator[](size_type n) const {
    assert(n >= 0);
    assert(n < size());
    auto it = begin() + n;
    assert(it.size() == n + 1);
    return *it;
  }

  /*!
   * \brief Access item
   *
   * Returns a reference to the element at position `n` in
   * the vector container.
   *
   * \return The item at the specified position in the container.
   *
   * \param Position of an item in the container.
   * Notice that the first item has a position of 0 (not 1).
   * Member type size_type is an unsigned integral type.
   *
   * #### Complexity ####
   * Logarithmic time.
   *
   *
   */
  reference operator[](size_type n) {
    assert(n >= 0);
    assert(n < size());
    auto it = begin() + n;
    assert(it.size() == n + 1);
    return *it;
  }
  ///@}

  /*---------------------------------------------------------------------*/
  /** @name Modifiers
   */
  ///@{

  /*!
   * \brief Adds item at the beginning
   *
   * Adds a new item to the front of the container,
   * before its current first item. The content of `x` is
   * copied (or moved) to the new item.
   *
   * \param x Value to be copied (or moved) to the new item.
   *
   * #### Complexity ####
   * Amortized constant (worst case logarithmic).
   *
   */
  void push_front(const value_type& x) {
    if (front_outer.full()) {
      if (front_inner.full())
        push_buffer_front_force(front_inner);
      front_outer.swap(front_inner);
      assert(front_outer.empty());
    }
    front_outer.push_front(chunk_meas, x);
  }

  /*!
   * \brief Adds item at the end
   *
   * Adds a new item to the back of the container,
   * after its current last item. The content of `x` is
   * copied (or moved) to the new item.
   *
   * \param x Value to be copied (or moved) to the new item.
   *
   * #### Complexity ####
   * Amortized constant (worst case logarithmic).
   *
   */
  void push_back(const value_type& x) {
    if (back_outer.full()) {
      if (back_inner.full())
        push_buffer_back_force(back_inner);
      back_outer.swap(back_inner);
      assert(back_outer.empty());
    }
    back_outer.push_back(chunk_meas, x);
  }

  /*!
   * \brief Deletes first item
   *
   * Removes the first item in the container, effectively
   * reducing the container size by one.
   *
   * Calling this method destroys the removed item.
   *
   * Calling this method on an `empty` container causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Amortized constant (worst case logarithmic).
   *
   * \return A copy of the first item in the
   * container.
   *
   */
  value_type pop_front() {
    if (front_outer.empty()) {
      assert(front_inner.empty());
      if (! middle->empty()) {
        chunk_pointer c = middle->pop_front(middle_meas);
        front_outer.swap(*c);
        chunk_free(c);
      } else if (! back_inner.empty()) {
        back_inner.swap(front_outer);
      } else if (! back_outer.empty()) {
        back_outer.swap(front_outer);
      }
    }
    assert(! front_outer.empty());
    value_type x = front_outer.pop_front(chunk_meas);
    restore_front_outer_empty_other_empty();
    return x;
  }

  /*!
   * \brief Deletes last item
   *
   * Removes the last item in the container, effectively
   * reducing the container size by one.
   *
   * Calling this method destroys the removed item.
   *
   * Calling this method on an `empty` container causes undefined
   * behavior.
   *
   * #### Complexity ####
   * Amortized constant (worst case logarithmic).
   *
   * \return A copy of the last item in the container.
   *
   */
  value_type pop_back() {
    if (back_outer.empty()) {
      assert(back_inner.empty());
      if (! middle->empty()) {
        chunk_pointer c = middle->pop_back(middle_meas);
        back_outer.swap(*c);
        chunk_free(c);
      } else if (! front_inner.empty()) {
        front_inner.swap(back_outer);
      } else if (! front_outer.empty()) {
        front_outer.swap(back_outer);
      }
    }
    assert(! back_outer.empty());
    value_type x = back_outer.pop_back(chunk_meas);
    restore_back_outer_empty_other_empty();
    return x;
  }

  /*!
   * \brief Adds items at the end
   *
   * Adds new items from a given array to the back of the container,
   * after its current last item. The content of `items` is
   * copied (or moved) to the new positions in the container.
   *
   * \param items Values to be copied (or moved) to the container.
   * \param nb Number of items to be inserted.
   *
   * #### Complexity ####
   * Linear in number of inserted items.
   *
   */
  void pushn_back(const_pointer src, size_type nb) {
    extras::pushn_back(*this, src, nb);
  }

  /*!
   * \brief Adds items at the beginning
   *
   * Adds new items to the front of the container,
   * before its current first item. The content of `items` is
   * copied (or moved) to the new positions in the container.
   *
   * \param items Values to be copied (or moved) to the container.
   * \param nb Number of items to be inserted.
   *
   * #### Complexity ####
   * Linear in number of inserted items.
   *
   */
  void pushn_front(const_pointer src, size_type nb) {
    extras::pushn_front(*this, src, nb);
  }

  /*!
   * \brief Deletes first items
   *
   * Removes the first items in the container, effectively
   * reducing the container size by `nb`.
   *
   * Calling this method destroys the removed items.
   *
   * The behavior is undefined if `nb > size()`.
   *
   * #### Complexity ####
   * Linear in number of items to be removed.
   *
   * \param nb Number of items to remove.
   *
   */
  void popn_front(size_type nb) {
    auto c = [] (const_pointer, const_pointer) { };
    stream_popn_front<typeof(c), false>(c, nb);
  }

  /*!
   * \brief Deletes last items
   *
   * Removes the last items in the container, effectively
   * reducing the container size by one.
   *
   * Calling this method destroys the removed items.
   *
   * The behavior is undefined if `nb > size()`.
   *
   * #### Complexity ####
   * Linear in number of items to be removed.
   *
   * \param destination Array to which the removed values
   * are to be copied (or moved).
   * \param nb Number of items to remove.
   *
   */
  void popn_back(size_type nb) {
    auto c = [] (const_pointer, const_pointer) { };
    stream_popn_back<typeof(c), false>(c, nb);
  }

  /*!
   * \brief Deletes first items
   *
   * Removes the first items in the container, effectively
   * reducing the container size by `nb`.
   *
   * Copies the removed items to the array `destination`.
   *
   * Calling this method destroys the removed items.
   *
   * The behavior is undefined if `nb > size()`.
   *
   * #### Complexity ####
   * Linear in number of items to be removed.
   *
   * \param destination Array to which the removed values
   * are to be copied (or moved).
   * \param nb Number of items to remove.
   *
   */
  void popn_back(value_type* dst, size_type nb) {
    extras::popn_back(*this, dst, nb);
  }

  /*!
   * \brief Deletes last items
   *
   * Removes the last items in the container, effectively
   * reducing the container size by one.
   *
   * Calling this method destroys the removed items.
   *
   * The behavior is undefined if `nb > size()`.
   *
   * #### Complexity ####
   * Linear in number of items to be removed.
   *
   * \param destination Array to which the removed values
   * are to be copied (or moved).
   * \param nb Number of items to remove.
   *
   */
  void popn_front(value_type* dst, size_type nb) {
    extras::popn_front(*this, dst, nb);
  }

  /*!
   * \brief Adds items at the end
   *
   * Adds new items from a given array to the back of the container,
   * after its current last item. The content is generated by applications
   * of the client-supplied function `body`. The applications initialize
   * the cells in the container in place.
   *
   * For a container `a`, new content is generated by applying
   *   body(0, a[0]); body(1, a[1]); ... body(n-1, a[n-1]);
   * where `n` is the size of the container.
   *
   * \tparam Type of the loop body class. This class must implement the
   * interface defined by \ref foreach_loop_body.
   * \param body function to generate the contents
   * \param nb Number of items to be inserted.
   *
   * #### Complexity ####
   * Linear in number of inserted items.
   *
   */
  template <class Producer>
  void stream_pushn_back(const Producer& prod, size_type nb) {
    if (nb == 0)
      return;
    size_type sz_orig = size();
    ensure_empty_inner();
    chunk_type c;
    c.swap(back_outer);
    size_type i = 0;
    while (i < nb) {
      size_type cap = (size_type)chunk_capacity;
      size_type m = std::min(cap, nb - i);
      m = std::min(m, cap - c.size());
      std::pair<const_pointer,const_pointer> rng = prod(i, m);
      const_pointer lo = rng.first;
      const_pointer hi = rng.second;
      size_type nb = hi - lo;
      c.pushn_back(chunk_meas, lo, nb);
      push_buffer_back(c);
      i += m;
    }
    restore_back_outer_empty_other_empty();
    assert(sz_orig + nb == size());
  }

  /*!
   * \brief Adds items at the beginning
   *
   * Adds new items to the front of the container,
   * before its current first item. The content is generated by applications
   * of the client-supplied function `body`. The applications initialize
   * the cells in the container in place.
   *
   * For a container `a`, new content is generated by applying
   *   body(0, a[0]); body(1, a[1]); ... body(n-1, a[n-1]);
   * where `n` is the size of the container.
   *
   * \tparam Type of the loop body class. This class must implement the
   * interface defined by \ref foreach_loop_body.
   * \param body function to generate the contents
   * \param nb Number of items to be inserted.
   *
   * #### Complexity ####
   * Linear in number of inserted items.
   *
   */
  template <class Producer>
  void stream_pushn_front(const Producer& prod, size_type nb) {
    if (nb == 0)
      return;
    size_type sz_orig = size();
    ensure_empty_inner();
    chunk_type c;
    c.swap(front_outer);
    size_type n = nb;
    while (n > 0) {
      size_type cap = (size_type)chunk_capacity;
      size_type m = std::min(cap, n);
      m = std::min(m, cap - c.size());
      n -= m;
      std::pair<const_pointer,const_pointer> rng = prod(n, m);
      const_pointer lo = rng.first;
      const_pointer hi = rng.second;
      size_type nb = hi - lo;
      c.pushn_front(chunk_meas, lo, nb);
      push_buffer_front(c);
    }
    restore_front_outer_empty_other_empty();
    assert(sz_orig + nb == size());
  }

  /*!
   * \brief Deletes last items
   *
   * Removes the last items in the container, effectively
   * reducing the container size by `nb`.
   *
   * Let our container be `c = [c_0, c_1, ..., c_n]`. Then,
   * the call `stream_backn(cons, nb)` performs:
   *
   *       for (size_type i = nb-1; i < n+1; i++)
   *          cons(c_i);
   *
   * Calling this method destroys the removed items.
   *
   * The behavior is undefined if `nb > size()`.
   *
   * #### Complexity ####
   * Linear in number of items to be removed.
   *
   * \param cons Function to receive items being removed from the
   * container.
   * \param nb Number of items to remove.
   *
   */
  template <class Consumer, bool should_consume=true>
  void stream_popn_back(const Consumer& cons, size_type nb) {
    size_type sz_orig = size();
    assert(sz_orig >= nb);
    size_type i = 0;
    while (i < nb) {
      ensure_back_outer_nonempty();
      size_type m = std::min(back_outer.size(), nb - i);
      back_outer.template popn_back<Consumer,should_consume>(chunk_meas, cons, m);
      i += m;
    }
    ensure_back_outer_nonempty();  // to restore invariants
    assert(sz_orig == size() + nb);
  }

  /*!
   * \brief Deletes first items
   *
   * Removes the first items in the container, effectively
   * reducing the container size by `nb`.
   *
   * Let our container be `c = [c_0, c_1, ..., c_n]`. Then,
   * the call `stream_backn(cons, nb)` performs:
   *
   *       for (size_type i = 0; i < nb; i++)
   *          cons(c_i);
   *
   * Calling this method destroys the removed items.
   *
   * The behavior is undefined if `nb > size()`.
   *
   * #### Complexity ####
   * Linear in number of items to be removed.
   *
   * \param cons Function to receive items being removed from the
   * container.
   * \param nb Number of items to remove.
   *
   */
  template <class Consumer, bool should_consume=true>
  void stream_popn_front(const Consumer& cons, size_type nb) {
    size_type sz_orig = size();
    assert(sz_orig >= nb);
    size_type i = 0;
    while (i < nb) {
      ensure_front_outer_nonempty();
      size_type m = std::min(front_outer.size(), nb - i);
      front_outer.template popn_front<Consumer,should_consume>(chunk_meas, cons, m);
      i += m;
    }
    ensure_front_outer_nonempty(); // to restore invariants
    assert(sz_orig == size() + nb);
  }

  /*!
   * \brief Merges with content of another container
   *
   * Removes all items from `other`, effectively reducing its
   * size to zero.
   *
   * Adds items removed from `other` to the back of this container,
   * after its current last item.
   *
   * \param other Container from which to take items.
   *
   * #### Complexity ####
   * Logarithmic in the size of the smallest of the two containers.
   *
   */
  void concat(self_type& other) {
    if (other.size() == 0)
      return;
    if (size() == 0)
      swap(other);
    // push buffers into the middle sequences
    push_buffer_back(back_inner);
    push_buffer_back(back_outer);
    other.push_buffer_front(other.front_inner);
    other.push_buffer_front(other.front_outer);
    // fuse front and back, if needed
    if (! middle->empty() && ! other.middle->empty()) {
      chunk_pointer c1 = middle->back();
      chunk_pointer c2 = other.middle->front();
      size_type nb1 = c1->size();
      size_type nb2 = c2->size();
      if (nb1 + nb2 <= chunk_capacity) {
        middle->pop_back(middle_meas);
        other.middle->pop_front(middle_meas);
        c2->transfer_from_front_to_back(chunk_meas, *c1, nb2);
        chunk_free(c2);
        middle->push_back(middle_meas, c1);
        // note: push might be factorized with earlier operations
      }
    }
    // migrate back chunks of the other and update the weight
    back_inner.swap(other.back_inner);
    back_outer.swap(other.back_outer);
    // concatenate the middle sequences
    middle->concat(middle_meas, *other.middle);
    // restore invariants
    restore_both_outer_empty_middle_empty();
    assert(other.empty());
  }

  //! \todo document
  template <class Pred>
  bool split(const Pred& p, reference middle_item, self_type& other) {
    size_type sz_orig = size();
    auto q = [&] (middle_measured_type m) {
      return p(size_access::cclient(m));
    };
    middle_measured_type prefix = split_aux(q, middle_algebra_type::identity(), middle_item, other);
    bool found = (size_access::csize(prefix) < sz_orig);
    return found;
  }

  //! \todo document
  template <class Pred>
  void split(const Pred& p, self_type& other) {
    value_type middle_item;
    bool found = split(p, middle_item, other);
    if (found)
      other.push_front(middle_item);
  }

  /*!
   * \brief Split by index
   *
   * The container is erased after and including the item at (zero-based) index `i`.
   *
   * The erased items are moved to the `other` container.
   *
   * \pre The `other` container is empty.
   * \pre `i >= 0`
   * \pre `i < size()`
   */
  void split(size_type i, self_type& other) {
    extras::split_by_index(*this, i, other);
  }

  /*!
   * \brief Split by iterator
   *
   * The container is erased after and including the item at the
   * specified `position`.
   *
   * The erased items are moved to the `other` container.
   *
   * \pre The `other` container is empty.
   */
  void split(iterator position, self_type& other) {
    extras::split_by_iterator(*this, position, other);
  }

  void split_approximate(self_type& other) {
    extras::split_approximate(*this, other);
  }

  /*!
   * \brief Inserts items
   *
   * The container is extended by inserting new items before
   * the item at the specified position.
   *
   * This effectively increates the container size by the amount
   * of items inserted.
   *
   * Insertions at the front or back are slightly more efficient
   * than on other positions.
   *
   * The parameters determine how many elements are inserted and
   * to which values they are initialized.
   *
   * \param position Position in the container where the new
   * items are inserted
   *
   * \param val Value to be copied (or moved) to the inserted
   * items
   *
   * \return An iterator that points to the first of the newly
   * inserted items.
   *
   * #### Complexity ####
   * Logarithmic time.
   *
   */
  iterator insert(iterator position, const value_type& val) {
    return extras::insert(*this, position, val);
  }

  /*!
   * \brief Erases items
   *
   * Removes from the container either a single item
   * (`position`) or a  range of items (`[first,last)`).
   *
   * This effectively reduces the container size by the
   * number of elements removed, which are destroyed.
   *
   * #### Complexity ####
   * Linear in the number of items erased (destructions)
   * and logarithmic in the number of items in the size
   * of the sequence.
   *
   * \param first Pointer to the first item to erase.
   * \param last Pointer to the position one past the last
   * item to erase.
   *
   */
  iterator erase(iterator first, iterator last) {
    return extras::erase(*this, first, last);
  }

  /*!
   * \brief Clears items
   *
   * Removes all items from the container (which are
   * destroyed), leaving the container with a size of 0.
   *
   * #### Complexity ####
   * Linear time (destructions).
   *
   */
  void clear() {
    popn_back(size());
  }

  /*!
   * \brief Swaps content
   *
   * Exchanges the content of the container by the
   * content of `x`, which is another container
   * of the same type. Sizes may differ.
   *
   * After the call to this member function, the items
   * in this container are those which were in `x` before
   * the call, and the elements of `x` are those which were
   * in this.
   *
   * #### Complexity ####
   * Constant time.
   *
   * \param x Another container container of the same type
   * (i.e., instantiated with the same template parameters)
   * whose content is swapped with that of this container.
   *
   */
  void swap(self_type& other) {
    std::swap(chunk_meas, other.chunk_meas);
    std::swap(middle_meas, other.middle_meas);
    front_outer.swap(other.front_outer);
    front_inner.swap(other.front_inner);
    middle.swap(other.middle);
    back_inner.swap(other.back_inner);
    back_outer.swap(other.back_outer);
  }

  ///@}

  friend void extras::split_by_index<self_type,size_type>(self_type& c, size_type i, self_type& other);

  /*---------------------------------------------------------------------*/
  /** @name Iterators
   */
  ///@{

  /*!
   * \brief Returns iterator to beginning
   *
   * Returns an iterator pointing to the first element in the
   * container.
   *
   * Notice that, unlike member front(), which returns
   * a reference to the first element, this function returns a
   * random access iterator pointing to it.
   *
   * If the container is empty, the returned iterator value
   * shall not be dereferenced.
   *
   * \return An iterator to the beginning of the container.
   *
   * #### Complexity ####
   * Logarithmic time.
   *
   */
  iterator begin() const {
    return iterator(this, middle_meas, chunkedseq::iterator::begin);
  }

  /*!
   * \brief Returns iterator to end
   *
   * Returns an iterator referring to the past-the-end element
   * in the container.
   *
   * The past-the-end element is the theoretical element that
   * would follow the last element in the vector. It does not
   * point to any element, and thus shall not be dereferenced.
   *
   * Because the ranges used by functions of the standard
   * library do not include the element pointed by their closing
   * iterator, this function is often used in combination with
   * begin() to specify a range including all the elements
   * in the container.
   *
   * If the container is empty, this function returns the same as
   * begin().
   *
   * \return An iterator to the element past the end of the container.
   *
   * #### Complexity ####
   * Logarithmic time.
   *
   */
  iterator end() const {
    return iterator(this, middle_meas, chunkedseq::iterator::end);
  }

  /*!
   * \brief Visits every item in the container
   *
   * Applies a given function `f` to each item in the sequence
   * in the left-to-right order of the sequence.
   *
   * \param f Function to apply to each item. The type of this
   * function must be `std::function<void(value_type)>`.
   *
   * #### Complexity ####
   * Linear in the size of the container.
   *
   */
  template <class Body>
  void for_each(const Body& f) const {
    front_outer.for_each(f);
    front_inner.for_each(f);
    middle->for_each([&] (chunk_pointer p) {
      p->for_each(f);
    });
    back_inner.for_each(f);
    back_outer.for_each(f);
  }

  /*!
   * \brief Visits every item in a specified range
   *
   * Applies a given function `f` to each item in the range
   * specified by `[beg,end)`.
   *
   * The function is applied in left-to-right order.
   *
   * \param f Function to apply to each item. The type of this
   * function must be `std::function<void(value_type)>`.
   * \param beg Pointer to the first item to visit
   * \param end Pointer to the item one past the last item to visit
   *
   * #### Complexity ####
   * Linear in `end-beg`.
   *
   */
  template <class Body>
  void for_each(iterator beg, iterator end, const Body& f) const {
    extras::for_each(beg, end, f);
  }

  /*!
   * \brief Visits every segment of items in the container
   *
   * Applies a given function `f` to each segment in the container.
   *
   * See \ref segments for a description of the segment type.
   *
   * \param f Function to apply to each segment. The type of this
   * function must be `std::function<void(segment_type)>`.
   *
   * #### Complexity ####
   * Linear in the number of segments in the container (assuming that
   * `f` visits each segment in unit time).
   *
   */
  template <class Body>
  void for_each_segment(const Body& f) const {
    front_outer.for_each_segment(f);
    front_inner.for_each_segment(f);
    middle->for_each([&] (chunk_pointer p) {
      p->for_each_segment(f);
    });
    back_inner.for_each_segment(f);
    back_outer.for_each_segment(f);
  }

  /*!
   * \brief Visits every segment of items in a specified range
   *
   * Applies a given function `f` to each segment in the range
   * defined by `[beg,end)`.
   *
   * See \ref segments for a description of the segment type.
   *
   * \param f Function to apply to the segments. The type of this
   * function must be `std::function<void(segment_type)>`.
   * \param beg Pointer to the first item to visit
   * \param end Pointer to the item one past the last item to visit
   *
   * #### Complexity ####
   * Linear in the number of segments in the given range (assuming that
   * `f` visits each segment in unit time).
   *
   */
  template <class Body>
  static void for_each_segment(iterator begin, iterator end, const Body& f) {
    extras::for_each_segment(begin, end, f);
  }

  ///@}

  /*---------------------------------------------------------------------*/
  /** @name Cached measurement
   */
  ///@{

  /*!
   * \brief Returns cached measurement
   *
   * #### Complexity ####
   * Constant time.
   *
   */
  measured_type get_cached() const {
    measured_type m = algebra_type::identity();
    m = algebra_type::combine(m, front_outer.get_cached());
    m = algebra_type::combine(m, front_inner.get_cached());
    measured_type n = size_access::cclient(middle->get_cached());
    m = algebra_type::combine(m, n);
    m = algebra_type::combine(m, back_inner.get_cached());
    m = algebra_type::combine(m, back_outer.get_cached());
    return m;
  }

  /*!
   * \brief Returns measurement operator
   *
   * #### Complexity ####
   * Constant time.
   *
   */
  measure_type get_measure() const {
    return chunk_meas;
  }

  /*!
   * \brief Sets measurement operator
   *
   * #### Complexity ####
   * Constant time.
   *
   */
  void set_measure(measure_type meas) {
    chunk_meas = meas;
    middle_meas.set_client_measure(meas);
  }

  /*!
   * \brief Copies the measurement operator
   *
   * #### Complexity ####
   * Constant time.
   *
   */
  void copy_measure_to(self_type& other) {
    other.set_measure(get_measure());
  }

  ///@}
  /*---------------------------------------------------------------------*/
  /** @name Debugging routines
   */
  ///@{
  // TODO: instead of taking this ItemPrinter argument, we could assume the ItemPrinter to be known from Item type
  template <class ItemPrinter>
  void print_chunk(const chunk_type& c) const  {
    printf("(");
    c.for_each([&] (value_type& x) {
      ItemPrinter::print(x);
      printf(" ");
    });
    printf(")");
  }

  // TODO: see whether we want to print cache
  class DummyCachePrinter { };

  template <class ItemPrinter, class CachePrinter = DummyCachePrinter>
  void print() const {
    auto show = [&] (const chunk_type& c) {
      print_chunk<ItemPrinter>(c); };
    show(front_outer);
    printf(" ");
    show(front_inner);
    printf(" [");
    middle->for_each([&] (chunk_pointer& c) {
      show(*c);
      printf(" ");
    });
    printf("] ");
    show(back_inner);
    printf(" ");
    show(back_outer);
  }

  void check_size() const {
#ifndef NDEBUG
    size_type sz = 0;
    middle->for_each([&] (chunk_pointer& c) {
      sz += c->size();
    });
    size_type msz = size_access::csize(middle->get_cached());
    assert(msz == sz);
    size_type sz2 = 0;
    for_each([&] (value_type&) {
      sz2++;
    });
    size_type sz3 = size();
    assert(sz2 == sz3);
#endif
  }

  void check() const {
#ifndef NDEBUG
    if (front_outer.empty())
      assert(front_inner.empty());
    if (back_outer.empty())
      assert(back_inner.empty());
    if (front_outer.empty() && back_outer.empty())
      assert(middle->empty());
    size_t sprev = -1;
    middle->for_each([&sprev] (chunk_pointer& c) {
      size_t scur = c->size();
      assert(scur > 0);
      if (sprev != -1)
        assert(sprev + scur > chunk_capacity);
      sprev = scur;
    });
    check_size();
#endif
  }

  template <class Add_edge, class Process_chunk>
  void reveal_internal_structure(const Add_edge& add_edge,
                                 const Process_chunk& process_chunk) const {
    long rootptr = (long)this;
    rootptr -= 8;
    add_edge((void*)rootptr, (void*)&front_outer);
    add_edge((void*)rootptr, (void*)&front_inner);
    add_edge((void*)rootptr, (void*)middle.get());
    add_edge((void*)rootptr, (void*)&back_inner);
    add_edge((void*)rootptr, (void*)&back_outer);
    process_chunk(&front_outer);
    process_chunk(&front_inner);
    middle->reveal_internal_structure(add_edge, process_chunk);
    process_chunk(&back_inner);
    process_chunk(&back_outer);
  }

  ///@}

};

/***********************************************************************/

} // end namespace chunkedseq
} // end namespace
} // end namespace

#endif /*! _PASL_DATA_CHUNKEDSEQBASE_H_ */


text:
farming.register_hoe(":farming:hoe_wood", {
	description = "Wooden Hoe",
	inventory_image = "farming_tool_woodhoe.png",
	max_uses = 30,
	material = "group:wood",
	groups = {flammable = 2},
})

farming.register_hoe(":farming:hoe_stone", {
	description = "Stone Hoe",
	inventory_image = "farming_tool_stonehoe.png",
	max_uses = 90,
	material = "group:stone"
})

farming.register_hoe(":farming:hoe_iron", {
	description = "Iron Hoe",
	inventory_image = "farming_tool_ironhoe.png",
	max_uses = 200,
	material = "default:iron_ingot"
})

farming.register_hoe(":farming:hoe_bronze", {
	description = "Bronze Hoe",
	inventory_image = "farming_tool_bronzehoe.png",
	max_uses = 220,
	material = "default:bronze_ingot"
})

farming.register_hoe(":farming:hoe_mese", {
	description = "Mese Hoe",
	inventory_image = "farming_tool_mesehoe.png",
	max_uses = 350,
	material = "default:mese_crystal"
})

farming.register_hoe(":farming:hoe_diamond", {
	description = "Diamond Hoe",
	inventory_image = "farming_tool_diamondhoe.png",
	max_uses = 500,
	material = "default:diamond"
})

redpajama_set_name: RedPajamaGithub


text:
<!DOCTYPE html>
<html>

  {% include head.html %}
  <body>
    {{ content }}

    {% include javascript.html %}
  </body>
</html>


text:
import datetime
import operator

from flask import request
from flask_peewee.forms import BaseModelConverter
from flask_peewee._compat import reduce
from peewee import *
from wtforms import fields
from wtforms import form
from wtforms import validators
from wtforms import widgets

class QueryFilter(object):
    """
    Basic class representing a named field (with or without a list of options)
    and an operation against a given value
    """
    def __init__(self, field, name, options=None):
        self.field = field
        self.name = name
        self.options = options

    def query(self, value):
        raise NotImplementedError

    def operation(self):
        raise NotImplementedError

    def get_options(self):
        return self.options

class EqualQueryFilter(QueryFilter):
    def query(self, value):
        return self.field == value

    def operation(self):
        return 'equal to'

class NotEqualQueryFilter(QueryFilter):
    def query(self, value):
        return self.field != value

    def operation(self):
        return 'not equal to'

class LessThanQueryFilter(QueryFilter):
    def query(self, value):
        return self.field < value

    def operation(self):
        return 'less than'

class LessThanEqualToQueryFilter(QueryFilter):
    def query(self, value):
        return self.field <= value

    def operation(self):
        return 'less than or equal to'

class GreaterThanQueryFilter(QueryFilter):
    def query(self, value):
        return self.field > value

    def operation(self):
        return 'greater than'

class GreaterThanEqualToQueryFilter(QueryFilter):
    def query(self, value):
        return self.field >= value

    def operation(self):
        return 'greater than or equal to'

class StartsWithQueryFilter(QueryFilter):
    def query(self, value):
        return fn.Lower(fn.Substr(self.field, 1, len(value))) == value.lower()

    def operation(self):
        return 'starts with'

class ContainsQueryFilter(QueryFilter):
    def query(self, value):
        return self.field ** ('%%%s%%' % value)

    def operation(self):
        return 'contains'

class YearFilter(QueryFilter):
    def query(self, value):
        value = int(value)
        return self.field.year == value

    def operation(self):
        return 'year equals'

class MonthFilter(QueryFilter):
    def query(self, value):
        value = int(value)
        return self.field.month == value

    def operation(self):
        return 'month equals'

class WithinDaysAgoFilter(QueryFilter):
    def query(self, value):
        value = int(value)
        return self.field >= (
            datetime.date.today() - datetime.timedelta(days=value))

    def operation(self):
        return 'within X days ago'

class OlderThanDaysAgoFilter(QueryFilter):
    def query(self, value):
        value = int(value)
        return self.field < (
            datetime.date.today() - datetime.timedelta(days=value))

    def operation(self):
        return 'older than X days ago'

class FilterMapping(object):
    """
    Map a peewee field to a list of valid query filters for that field
    """
    string = (
        EqualQueryFilter, NotEqualQueryFilter, StartsWithQueryFilter,
        ContainsQueryFilter)
    numeric = (
        EqualQueryFilter, NotEqualQueryFilter, LessThanQueryFilter,
        GreaterThanQueryFilter, LessThanEqualToQueryFilter,
        GreaterThanEqualToQueryFilter)
    datetime_date = (numeric + (
        WithinDaysAgoFilter, OlderThanDaysAgoFilter, YearFilter, MonthFilter))
    foreign_key = (EqualQueryFilter, NotEqualQueryFilter)
    boolean = (EqualQueryFilter, NotEqualQueryFilter)

    def get_field_types(self):
        return {
            CharField: 'string',
            TextField: 'string',
            DateTimeField: 'datetime_date',
            DateField: 'datetime_date',
            TimeField: 'numeric',
            IntegerField: 'numeric',
            BigIntegerField: 'numeric',
            FloatField: 'numeric',
            DoubleField: 'numeric',
            DecimalField: 'numeric',
            BooleanField: 'boolean',
            PrimaryKeyField: 'numeric',
            ForeignKeyField: 'foreign_key',
        }

    def convert(self, field):
        mapping = self.get_field_types()

        for klass in type(field).__mro__:
            if klass in mapping:
                mapping_fn = getattr(self, 'convert_%s' % mapping[klass])
                return mapping_fn(field)

        # fall back to numeric
        return self.convert_numeric(field)

    def convert_string(self, field):
        return [f(field, field.verbose_name, field.choices) for f in self.string]

    def convert_numeric(self, field):
        return [f(field, field.verbose_name, field.choices) for f in self.numeric]

    def convert_datetime_date(self, field):
        return [f(field, field.verbose_name, field.choices) for f in self.datetime_date]

    def convert_boolean(self, field):
        boolean_choices = [('True', '1', 'False', '')]
        return [f(field, field.verbose_name, boolean_choices) for f in self.boolean]

    def convert_foreign_key(self, field):
        return [f(field, field.verbose_name, field.choices) for f in self.foreign_key]

class FieldTreeNode(object):
    def __init__(self, model, fields, children=None):
        self.model = model
        self.fields = fields
        self.children = children or {}

def make_field_tree(model, fields, exclude, force_recursion=False, seen=None):
    no_explicit_fields = fields is None # assume we want all of them
    if no_explicit_fields:
        fields = model._meta.sorted_field_names
    exclude = exclude or []
    seen = seen or set()

    model_fields = []
    children = {}

    for field_obj in model._meta.sorted_fields:
        if field_obj.name in exclude or field_obj in seen:
            continue

        if field_obj.name in fields:
            model_fields.append(field_obj)

        if isinstance(field_obj, ForeignKeyField):
            seen.add(field_obj)
            if no_explicit_fields:
                rel_fields = None
            else:
                rel_fields = [
                    rf.replace('%s__' % field_obj.name, '') \
                        for rf in fields if rf.startswith('%s__' % field_obj.name)
                ]
                if not rel_fields and force_recursion:
                    rel_fields = None

            rel_exclude = [
                rx.replace('%s__' % field_obj.name, '') \
                    for rx in exclude if rx.startswith('%s__' % field_obj.name)
            ]
            children[field_obj.name] = make_field_tree(field_obj.rel_model, rel_fields, rel_exclude, force_recursion, seen)

    return FieldTreeNode(model, model_fields, children)

class SmallSelectWidget(widgets.Select):
    def __call__(self, field, **kwargs):
        kwargs['class'] = 'span2'
        return super(SmallSelectWidget, self).__call__(field, **kwargs)

class FilterForm(object):
    base_class = form.Form
    separator = '-'
    field_operation_prefix = 'fo_'
    field_value_prefix = 'fv_'
    field_relation_prefix = 'fr_'

    def __init__(self, model, model_converter, filter_mapping, fields=None, exclude=None):
        self.model = model
        self.model_converter = model_converter
        self.filter_mapping = filter_mapping

        # convert fields and exclude into a tree
        self._field_tree = make_field_tree(model, fields, exclude)

        self._query_filters = self.load_query_filters()

    def load_query_filters(self):
        query_filters = {}
        queue = [self._field_tree]

        while queue:
            curr = queue.pop(0)
            for field in curr.fields:
                query_filters[field] = self.filter_mapping.convert(field)
            queue.extend(curr.children.values())

        return query_filters

    def get_operation_field(self, field):
        choices = []
        for i, query_filter in enumerate(self._query_filters[field]):
            choices.append((str(i), query_filter.operation()))

        return fields.SelectField(choices=choices, validators=[validators.Optional()], widget=SmallSelectWidget())

    def get_field_default(self, field):
        if isinstance(field, DateTimeField):
            return datetime.datetime.now()
        elif isinstance(field, DateField):
            return datetime.date.today()
        elif isinstance(field, TimeField):
            return datetime.time(0, 0)
        return field.default

    def get_value_field(self, field):
        field_name, form_field = self.model_converter.convert(field.model_class, field, None)

        form_field.kwargs['default'] = self.get_field_default(field)
        form_field.kwargs['validators'] = [validators.Optional()]
        return form_field

    def get_field_dict(self, node=None, prefix=None):
        field_dict = {}
        node = node or self._field_tree

        for field in node.fields:
            op_field = self.get_operation_field(field)
            val_field = self.get_value_field(field)
            field_dict['%s%s' % (self.field_operation_prefix, field.name)] = op_field
            field_dict['%s%s' % (self.field_value_prefix, field.name)] = val_field

        for prefix, node in node.children.items():
            child_fd = self.get_field_dict(node, prefix)
            field_dict['%s%s' % (self.field_relation_prefix, prefix)] = fields.FormField(
                self.get_form(child_fd),
                separator=self.separator,
            )

        return field_dict

    def get_form(self, field_dict):
        return type(
            self.model.__name__ + 'FilterForm',
            (self.base_class, ),
            field_dict,
        )

    def parse_query_filters(self):
        # reconstruct the "select" and "value" fields we are searching for in the
        # arguments from the request by depth-first searching the field tree --
        # basically what we should have at the end is the field we're querying,
        # the type of query (QueryFilter), the value requested, and the path we
        # took to get there (joins)
        accum = {}

        def _dfs(node, prefix, models, join_columns):
            for field in node.fields:
                qf_select = self.field_operation_prefix.join((prefix, field.name))
                qf_value = self.field_value_prefix.join((prefix, field.name))

                if qf_select in request.args and qf_value in request.args:
                    accum.setdefault(field, [])
                    accum[field].append((
                        request.args.getlist(qf_select),
                        request.args.getlist(qf_value),
                        models,
                        join_columns,
                        qf_select,
                        qf_value,
                    ))

            for child_prefix, child in node.children.items():
                new_prefix = prefix + self.field_relation_prefix + child_prefix + self.separator
                model_copy = list(models) + [child.model]
                join_copy = list(join_columns) + [child_prefix]
                _dfs(child, new_prefix, model_copy, join_copy)

        _dfs(self._field_tree, '', [], [])

        return accum

    def process_request(self, query):
        field_dict = self.get_field_dict()
        FormClass = self.get_form(field_dict)

        form = FormClass(request.args)
        query_filters = self.parse_query_filters()
        cleaned = []

        for field, filters in query_filters.items():
            for (filter_idx_list, filter_value_list, path, join_path, qf_s, qf_v) in filters:
                query = query.switch(self.model)
                for join, model in zip(join_path, path):
                    query = query.join(model, on=join)

                q_objects = []
                for filter_idx, filter_value in zip(filter_idx_list, filter_value_list):
                    idx = int(filter_idx)
                    cleaned.append((qf_s, idx, qf_v, filter_value))
                    query_filter = self._query_filters[field][idx]
                    q_objects.append(query_filter.query(field.db_value(filter_value)))

                query = query.where(reduce(operator.or_, q_objects))

        return form, query, cleaned

class FilterModelConverter(BaseModelConverter):
    def __init__(self, *args, **kwargs):
        super(FilterModelConverter, self).__init__(*args, **kwargs)
        self.defaults = dict(self.defaults)
        self.defaults[TextField] = fields.TextField
        self.defaults[DateTimeField] = fields.DateTimeField


text:
var registerBpmnJSPlugin = require('camunda-modeler-plugin-helpers').registerBpmnJSPlugin;

var ColorPlugin = require('./ColorPlugin');

registerBpmnJSPlugin(ColorPlugin);


text:
import { Util, Collection } from '../Util';
import { Factory } from '../Factory';
import { Shape, ShapeConfig } from '../Shape';
import { getNumberValidator } from '../Validators';
import { _registerNode } from '../Global';

import { GetSet, IRect } from '../types';
import { Context } from '../Context';

export interface ImageConfig extends ShapeConfig {
  image: CanvasImageSource | undefined;
  crop?: IRect;
}

/**
 * Image constructor
 * @constructor
 * @memberof Konva
 * @augments Konva.Shape
 * @param {Object} config
 * @param {Image} config.image
 * @param {Object} [config.crop]
 * @@shapeParams
 * @@nodeParams
 * @example
 * var imageObj = new Image();
 * imageObj.onload = function() {
 *   var image = new Konva.Image({
 *     x: 200,
 *     y: 50,
 *     image: imageObj,
 *     width: 100,
 *     height: 100
 *   });
 * };
 * imageObj.src = '/path/to/image.jpg'
 */
export class Image extends Shape<ImageConfig> {
  _useBufferCanvas() {
    return super._useBufferCanvas(true);
  }
  _sceneFunc(context: Context) {
    const width = this.getWidth();
    const height = this.getHeight();
    const image = this.attrs.image;
    let params;

    if (image) {
      const cropWidth = this.attrs.cropWidth;
      const cropHeight = this.attrs.cropHeight;
      if (cropWidth && cropHeight) {
        params = [
          image,
          this.cropX(),
          this.cropY(),
          cropWidth,
          cropHeight,
          0,
          0,
          width,
          height,
        ];
      } else {
        params = [image, 0, 0, width, height];
      }
    }

    if (this.hasFill() || this.hasStroke()) {
      context.beginPath();
      context.rect(0, 0, width, height);
      context.closePath();
      context.fillStrokeShape(this);
    }

    if (image) {
      context.drawImage.apply(context, params);
    }
  }
  _hitFunc(context) {
    var width = this.width(),
      height = this.height();

    context.beginPath();
    context.rect(0, 0, width, height);
    context.closePath();
    context.fillStrokeShape(this);
  }
  getWidth() {
    return this.attrs.width ?? (this.image()?.width || 0);
  }
  getHeight() {
    return this.attrs.height ?? (this.image()?.height || 0);
  }

  /**
   * load image from given url and create `Konva.Image` instance
   * @method
   * @memberof Konva.Image
   * @param {String} url image source
   * @param {Function} callback with Konva.Image instance as first argument
   * @example
   *  Konva.Image.fromURL(imageURL, function(image){
   *    // image is Konva.Image instance
   *    layer.add(image);
   *    layer.draw();
   *  });
   */
  static fromURL(url, callback) {
    var img = Util.createImageElement();
    img.onload = function () {
      var image = new Image({
        image: img,
      });
      callback(image);
    };
    img.crossOrigin = 'Anonymous';
    img.src = url;
  }

  image: GetSet<CanvasImageSource | undefined, this>;
  crop: GetSet<IRect, this>;
  cropX: GetSet<number, this>;
  cropY: GetSet<number, this>;
  cropWidth: GetSet<number, this>;
  cropHeight: GetSet<number, this>;
}

Image.prototype.className = 'Image';
_registerNode(Image);
/**
 * get/set image source. It can be image, canvas or video element
 * @name Konva.Image#image
 * @method
 * @param {Object} image source
 * @returns {Object}
 * @example
 * // get value
 * var image = shape.image();
 *
 * // set value
 * shape.image(img);
 */
Factory.addGetterSetter(Image, 'image');

Factory.addComponentsGetterSetter(Image, 'crop', ['x', 'y', 'width', 'height']);
/**
 * get/set crop
 * @method
 * @name Konva.Image#crop
 * @param {Object} crop
 * @param {Number} crop.x
 * @param {Number} crop.y
 * @param {Number} crop.width
 * @param {Number} crop.height
 * @returns {Object}
 * @example
 * // get crop
 * var crop = image.crop();
 *
 * // set crop
 * image.crop({
 *   x: 20,
 *   y: 20,
 *   width: 20,
 *   height: 20
 * });
 */

Factory.addGetterSetter(Image, 'cropX', 0, getNumberValidator());
/**
 * get/set crop x
 * @method
 * @name Konva.Image#cropX
 * @param {Number} x
 * @returns {Number}
 * @example
 * // get crop x
 * var cropX = image.cropX();
 *
 * // set crop x
 * image.cropX(20);
 */

Factory.addGetterSetter(Image, 'cropY', 0, getNumberValidator());
/**
 * get/set crop y
 * @name Konva.Image#cropY
 * @method
 * @param {Number} y
 * @returns {Number}
 * @example
 * // get crop y
 * var cropY = image.cropY();
 *
 * // set crop y
 * image.cropY(20);
 */

Factory.addGetterSetter(Image, 'cropWidth', 0, getNumberValidator());
/**
 * get/set crop width
 * @name Konva.Image#cropWidth
 * @method
 * @param {Number} width
 * @returns {Number}
 * @example
 * // get crop width
 * var cropWidth = image.cropWidth();
 *
 * // set crop width
 * image.cropWidth(20);
 */

Factory.addGetterSetter(Image, 'cropHeight', 0, getNumberValidator());
/**
 * get/set crop height
 * @name Konva.Image#cropHeight
 * @method
 * @param {Number} height
 * @returns {Number}
 * @example
 * // get crop height
 * var cropHeight = image.cropHeight();
 *
 * // set crop height
 * image.cropHeight(20);
 */

Collection.mapMethods(Image);


text:
namespace media {

class AudioBus;

}  // namespace media

namespace content {

// Writes audio input data used for debugging purposes. All operations are
// non-blocking.
class CONTENT_EXPORT AudioDebugFileWriter
    : public NON_EXPORTED_BASE(media::AudioFileWriter) {
 public:
  explicit AudioDebugFileWriter(const media::AudioParameters& params);
  ~AudioDebugFileWriter() override;

  void Start(const base::FilePath& file) override;
  void Stop() override;
  void Write(std::unique_ptr<media::AudioBus> data) override;
  bool WillWrite() override;

 private:
  class AudioFileWriter;
  using AudioFileWriterUniquePtr =
      std::unique_ptr<AudioFileWriter, BrowserThread::DeleteOnFileThread>;
  AudioFileWriterUniquePtr file_writer_;
  const media::AudioParameters params_;
  base::SequenceChecker client_sequence_checker_;
};

}  // namspace content

#endif  // CONTENT_BROWSER_RENDERER_HOST_MEDIA_AUDIO_DEBUG_FILE_WRITER_H_


text:
This is a pravate test project for DevOps.

On Docker on Vagrant, I provision containers for  Redmine, GitLab(+ Mattermost) and Jenkins.


text:
function p2kwiet140781533107_characteristics_postshow_seq0(eventobject) {
    return initAuthCharacteristics.call(this);
}


text:
<title>Testing surface.destroy()</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

Testing surface.destroy()

<button onclick="cleanUp();">Regenerate</button>

See the firebug log for detailed reports.

That's all Folks!


text:
/* Make clicks pass-through */
#loading-bar,
#loading-bar-spinner {
  pointer-events: none;
  -webkit-pointer-events: none;
  -webkit-transition: 350ms linear all;
  -moz-transition: 350ms linear all;
  -o-transition: 350ms linear all;
  transition: 350ms linear all;
}

#loading-bar.ng-enter,
#loading-bar.ng-leave.ng-leave-active,
#loading-bar-spinner.ng-enter,
#loading-bar-spinner.ng-leave.ng-leave-active {
  opacity: 0;
}

#loading-bar.ng-enter.ng-enter-active,
#loading-bar.ng-leave,
#loading-bar-spinner.ng-enter.ng-enter-active,
#loading-bar-spinner.ng-leave {
  opacity: 1;
}

#loading-bar .bar {
  -webkit-transition: width 350ms;
  -moz-transition: width 350ms;
  -o-transition: width 350ms;
  transition: width 350ms;

  background: #29d;
  position: fixed;
  z-index: 2000;
  top: 51px;
  left: 0;
  width: 100%;
  height: 4px;
}

/* Fancy blur effect */
#loading-bar .peg {
  display: block;
  position: absolute;
  right: 0px;
  width: 100px;
  height: 100%;
  /*box-shadow: 0 0 10px #29d, 0 0 5px #29d;*/
  opacity: 1.0;

  -webkit-transform: rotate(3deg) translate(0px, -4px);
  -moz-transform: rotate(3deg) translate(0px, -4px);
  -ms-transform: rotate(3deg) translate(0px, -4px);
  -o-transform: rotate(3deg) translate(0px, -4px);
  transform: rotate(3deg) translate(0px, -4px);
}

#loading-bar-spinner {
  display: block;
  width: 100%;
  position: absolute;
  z-index: 100;
  margin-left: auto;
  margin-right: auto;
  top: 100px;
  left: 10px;
}

#loading-bar-spinner .spinner-icon {
  width: 30px;
  height: 30px;

  border:  solid 2px transparent;
  border-top-color:  #29d;
  border-left-color: #29d;
  border-radius: 30px;

  -webkit-animation: loading-bar-spinner 400ms linear infinite;
  -moz-animation:    loading-bar-spinner 400ms linear infinite;
  -ms-animation:     loading-bar-spinner 400ms linear infinite;
  -o-animation:      loading-bar-spinner 400ms linear infinite;
  animation:         loading-bar-spinner 400ms linear infinite;
}

@-webkit-keyframes loading-bar-spinner {
  0%   { -webkit-transform: rotate(0deg);   transform: rotate(0deg); }
  100% { -webkit-transform: rotate(360deg); transform: rotate(360deg); }
}
@-moz-keyframes loading-bar-spinner {
  0%   { -moz-transform: rotate(0deg);   transform: rotate(0deg); }
  100% { -moz-transform: rotate(360deg); transform: rotate(360deg); }
}
@-o-keyframes loading-bar-spinner {
  0%   { -o-transform: rotate(0deg);   transform: rotate(0deg); }
  100% { -o-transform: rotate(360deg); transform: rotate(360deg); }
}
@-ms-keyframes loading-bar-spinner {
  0%   { -ms-transform: rotate(0deg);   transform: rotate(0deg); }
  100% { -ms-transform: rotate(360deg); transform: rotate(360deg); }
}
@keyframes loading-bar-spinner {
  0%   { transform: rotate(0deg);   transform: rotate(0deg); }
  100% { transform: rotate(360deg); transform: rotate(360deg); }
}


text:
title:  "Adding interactive plots to a Jekyll blog 📈"
date:   2018-09-02
layout: post
tag:
- python
- jekyll
- music
- lyrics
category: blog
author: john
name: john
image: assets/images/Plot_like_love.png
description: Interactive plots from Python in a Jekyll blog
---
I've been trying for awhile to get interactive plots going on my Jekyll blog. It turns out you can export plots from `matplotlib` in Python to HTML using a handy library called [`mpld3`](https://mpld3.github.io/).

## Example interactive plot
After analyzing 12k+ [country lyrics](http://www.johnwmillr.com/trucks-and-beer/), I've gone and downloaded about 40k rap and rock lyrics using my [LyricsGenius](https://github.com/johnwmillr/LyricsGenius) Python package. I'm still working on my analysis of those lyrics, but I think this first plot is pretty interesting.

Move your mouse over the points in the plot to see more detail about the artists they represent.

{% include plot_like_love.html %}

For whatever reason, measuring how often an artist uses the word *like* is a pretty decent way to differentiate between country, rock, and rap artists. Rap artists are much more likely than country artists to use the word *like* in their songs.

There is also a clear separation on gender between country artists for the word *love*, but the effect doesn't appear to hold for rap artists. Unfortunately, there weren't enough female rock artists in my dataset to notice any relationships with gender in that genre.

### Creating the plot
As I mentioned above, I used the `mpld3` library to make the interactive plot. The library went unmaintained for awhile, but has been [recently](https://github.com/mpld3/mpld3/commit/c490724928f827721c0d4389c742e4c4ae8fab27) switched to new maintainers, which will hopefully revitalize the project. Before this switch in ownership, I [forked](https://github.com/johnwmillr/mpld3) the repo to continue making small tweaks to suit my needs.

The process of creating an interactive plot with `mpld3` is actually fairly straightforward. The library is well documented and provides a number of helpful [tutorials](https://mpld3.github.io/quickstart.html) and [examples](https://mpld3.github.io/_downloads/mpld3_demo.html) for getting started. I use the library's [tooltip](https://mpld3.github.io/examples/html_tooltips.html) feature with some extra CSS to make the artist info and image show up on the plot above.

Here is the relevant code excerpt from the tooltip tutorial:
```python
fig, ax = plt.subplots(figsize=(10,10))

labels = []
for i in range(N):
    label = df.ix[[i], :].T
    label.columns = ['Row {0}'.format(i)]
    labels.append(str(label.to_html()))

points = ax.plot(df.x, df.y, 'o', color='b',
                 mec='k', ms=15, mew=1, alpha=.6)

tooltip = plugins.PointHTMLTooltip(points[0], labels,
                                   voffset=10, hoffset=10, css=css)
plugins.connect(fig, tooltip)
mpld3.show()
```

Using the `mpld3.show()` function will display the graph within a Jupyter notebook. However, we're interested in exporting the plot for use in a Jekyll blog post, so we'll use the `mpld3.save_html()` function instead of the `show()` command:

```python
filename = 'example_plot.html'
mpld3.save_html(fig, filename, template_type='simple')
```

For some reason, the plot wouldn't render in my blog post unless I set the `template_type` variable to `'simple'`.

### Adding the plot to a post
After exporting the plot as an HTML file, add it to the `_includes/` directory within your Jekyll site. After putting the file in place, we just need to include the HTML within the Markdown file for a blog post:

```html
Here is some example text for an exciting blog post. Deep blockchain learning is the future, blah, blah, etc.

{{ "{% include example_plot.html " }}%}

And then you keep writing your exciting blog post.
```

That's it! The interactive plot should show up once the Jekyll site is built and renders the page.

## Up next
I'd love to make the plots even more interactive. It would be very cool if users were able to specify the actual lyrics data that gets displayed in the plot. If anybody knows a way to do this with a Jekyll blog, let met know!


text:
UNIX_RedundancySetFixture::UNIX_RedundancySetFixture()
{
}

UNIX_RedundancySetFixture::~UNIX_RedundancySetFixture()
{
}

void UNIX_RedundancySetFixture::Run()
{
	CIMName className("UNIX_RedundancySet");
	CIMNamespaceName nameSpace("root/cimv2");
	UNIX_RedundancySet _p;
	UNIX_RedundancySetProvider _provider;
	Uint32 propertyCount;
	CIMOMHandle omHandle;
	_provider.initialize(omHandle);
	_p.initialize();

	for(int pIndex = 0; _p.load(pIndex); pIndex++)
	{
		CIMInstance instance = _provider.constructInstance(className,
					nameSpace,
					_p);
		CIMObjectPath path = instance.getPath();
		cout << path.toString() << endl;
		propertyCount = instance.getPropertyCount();
		for(Uint32 i = 0; i < propertyCount; i++)
		{

			CIMProperty propertyItem = instance.getProperty(i);
			cout << "	Name: " << propertyItem.getName().getString() << " - Value: " << propertyItem.getValue().toString() << endl;

		}
		cout << "------------------------------------" << endl;
		cout << endl;
	}

	_p.finalize();
}


text:
				<input id="featureButton_${_layerId}_${_featureId}" class="checkbox" type="checkbox" name="featureItem" value="${_layerId},${_featureId}" onchange="featureExpand(this)" onkeypress="featureExpandAndZoom(event, this)"/>

				${_title}

				<input id="panBtn_${_layerId}_${_featureId}" class="fg pageBtn" type="button" value="${_panTo}" onclick="featurePanZoom(this, true)" data-layerId="${_layerId}" data-featureId="${_featureId}"/>

				<input id="zoomBtn_${_layerId}_${_featureId}" class="fg pageBtn" type="button" value="${_zoomTo}" onclick="featurePanZoom(this, false)" data-layerId="${_layerId}" data-featureId="${_featureId}"/>

Alt + 7 ${hint}

		${_content}


text:
"""Base Manager class.

Managers are responsible for a certain aspect of the system.  It is a logical
grouping of code relating to a portion of the system.  In general other
components should be using the manager to make changes to the components that
it is responsible for.

For example, other components that need to deal with volumes in some way,
should do so by calling methods on the VolumeManager instead of directly
changing fields in the database.  This allows us to keep all of the code
relating to volumes in the same place.

We have adopted a basic strategy of Smart managers and dumb data, which means
rather than attaching methods to data objects, components should call manager
methods that act on the data.

Methods on managers that can be executed locally should be called directly. If
a particular method must execute on a remote host, this should be done via rpc
to the service that wraps the manager

Managers should be responsible for most of the db access, and
non-implementation specific data.  Anything implementation specific that can't
be generalized should be done by the Driver.

In general, we prefer to have one manager with multiple drivers for different
implementations, but sometimes it makes sense to have multiple managers.  You
can think of it this way: Abstract different overall strategies at the manager
level(FlatNetwork vs VlanNetwork), and different implementations at the driver
level(LinuxNetDriver vs CiscoNetDriver).

Managers will often provide methods for initial setup of a host or periodic
tasks to a wrapping service.

This module provides Manager, a base class for managers.

"""

import eventlet

from nova.db import base
from nova.openstack.common import cfg
from nova.openstack.common import log as logging
from nova.openstack.common.plugin import pluginmanager
from nova.openstack.common.rpc import dispatcher as rpc_dispatcher
from nova.scheduler import rpcapi as scheduler_rpcapi
from nova import version

CONF = cfg.CONF
CONF.import_opt('host', 'nova.config')
LOG = logging.getLogger(__name__)

def periodic_task(*args, **kwargs):
    """Decorator to indicate that a method is a periodic task.

    This decorator can be used in two ways:

        1. Without arguments '@periodic_task', this will be run on every tick
           of the periodic scheduler.

        2. With arguments, @periodic_task(ticks_between_runs=N), this will be
           run on every N ticks of the periodic scheduler.
    """
    def decorator(f):
        f._periodic_task = True
        f._ticks_between_runs = kwargs.pop('ticks_between_runs', 0)
        return f

    # NOTE(sirp): The `if` is necessary to allow the decorator to be used with
    # and without parens.
    #
    # In the 'with-parens' case (with kwargs present), this function needs to
    # return a decorator function since the interpreter will invoke it like:
    #
    #   periodic_task(*args, **kwargs)(f)
    #
    # In the 'without-parens' case, the original function will be passed
    # in as the first argument, like:
    #
    #   periodic_task(f)
    if kwargs:
        return decorator
    else:
        return decorator(args[0])

class ManagerMeta(type):
    def __init__(cls, names, bases, dict_):
        """Metaclass that allows us to collect decorated periodic tasks."""
        super(ManagerMeta, cls).__init__(names, bases, dict_)

        # NOTE(sirp): if the attribute is not present then we must be the base
        # class, so, go ahead an initialize it. If the attribute is present,
        # then we're a subclass so make a copy of it so we don't step on our
        # parent's toes.
        try:
            cls._periodic_tasks = cls._periodic_tasks[:]
        except AttributeError:
            cls._periodic_tasks = []

        try:
            cls._ticks_to_skip = cls._ticks_to_skip.copy()
        except AttributeError:
            cls._ticks_to_skip = {}

        for value in cls.__dict__.values():
            if getattr(value, '_periodic_task', False):
                task = value
                name = task.__name__
                if task._ticks_between_runs >= 0:
                    cls._periodic_tasks.append((name, task))
                    cls._ticks_to_skip[name] = task._ticks_between_runs

class Manager(base.Base):
    __metaclass__ = ManagerMeta

    # Set RPC API version to 1.0 by default.
    RPC_API_VERSION = '1.0'

    def __init__(self, host=None, db_driver=None):
        if not host:
            host = CONF.host
        self.host = host
        self.load_plugins()
        self.backdoor_port = None
        super(Manager, self).__init__(db_driver)

    def load_plugins(self):
        pluginmgr = pluginmanager.PluginManager('nova', self.__class__)
        pluginmgr.load_plugins()

    def create_rpc_dispatcher(self):
        '''Get the rpc dispatcher for this manager.

        If a manager would like to set an rpc API version, or support more than
        one class as the target of rpc messages, override this method.
        '''
        return rpc_dispatcher.RpcDispatcher([self])

    def periodic_tasks(self, context, raise_on_error=False):
        """Tasks to be run at a periodic interval."""
        for task_name, task in self._periodic_tasks:
            full_task_name = '.'.join([self.__class__.__name__, task_name])

            ticks_to_skip = self._ticks_to_skip[task_name]
            if ticks_to_skip > 0:
                LOG.debug(_("Skipping %(full_task_name)s, %(ticks_to_skip)s"
                            " ticks left until next run"), locals())
                self._ticks_to_skip[task_name] -= 1
                continue

            self._ticks_to_skip[task_name] = task._ticks_between_runs
            LOG.debug(_("Running periodic task %(full_task_name)s"), locals())

            try:
                task(self, context)
                # NOTE(tiantian): After finished a task, allow manager to
                # do other work (report_state, processing AMPQ request etc.)
                eventlet.sleep(0)
            except Exception as e:
                if raise_on_error:
                    raise
                LOG.exception(_("Error during %(full_task_name)s: %(e)s"),
                              locals())

    def init_host(self):
        """Hook to do additional manager initialization when one requests
        the service be started.  This is called before any service record
        is created.

        Child classes should override this method.
        """
        pass

    def pre_start_hook(self, **kwargs):
        """Hook to provide the manager the ability to do additional
        start-up work before any RPC queues/consumers are created. This is
        called after other initialization has succeeded and a service
        record is created.

        Child classes should override this method.
        """
        pass

    def post_start_hook(self):
        """Hook to provide the manager the ability to do additional
        start-up work immediately after a service creates RPC consumers
        and starts 'running'.

        Child classes should override this method.
        """
        pass

    def service_version(self, context):
        return version.version_string()

    def service_config(self, context):
        config = {}
        for key in CONF:
            config[key] = CONF.get(key, None)
        return config

class SchedulerDependentManager(Manager):
    """Periodically send capability updates to the Scheduler services.

    Services that need to update the Scheduler of their capabilities
    should derive from this class. Otherwise they can derive from
    manager.Manager directly. Updates are only sent after
    update_service_capabilities is called with non-None values.

    """

    def __init__(self, host=None, db_driver=None, service_name='undefined'):
        self.last_capabilities = None
        self.service_name = service_name
        self.scheduler_rpcapi = scheduler_rpcapi.SchedulerAPI()
        super(SchedulerDependentManager, self).__init__(host, db_driver)

    def load_plugins(self):
        pluginmgr = pluginmanager.PluginManager('nova', self.service_name)
        pluginmgr.load_plugins()

    def update_service_capabilities(self, capabilities):
        """Remember these capabilities to send on next periodic update."""
        if not isinstance(capabilities, list):
            capabilities = [capabilities]
        self.last_capabilities = capabilities

    @periodic_task
    def publish_service_capabilities(self, context):
        """Pass data back to the scheduler.

        Called at a periodic interval. And also called via rpc soon after
        the start of the scheduler.
        """
        if self.last_capabilities:
            LOG.debug(_('Notifying Schedulers of capabilities ...'))
            for capability_item in self.last_capabilities:
                self.scheduler_rpcapi.update_service_capabilities(context,
                        self.service_name, self.host, capability_item)
        # TODO(NTTdocomo): Make update_service_capabilities() accept a list
        #                  of capabilities


text:
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Text;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using Sage.Platform.WebPortal;
using Sage.Platform.Application;
using System.Reflection;
using System.Collections;
using System.Linq;
using Sage.Platform.ComponentModel;
using Sage.Platform.WebPortal.SmartParts;
using System.Diagnostics;
using Sage.Platform.Orm.Interfaces;
using Sage.Platform.Application.UI.Web;
using Sage.Platform.WebPortal.Services;
using OpenSlx.Lib.Properties;
using OpenSlx.Lib.Utility;
using OpenSlx.Lib.Web.Utility;
using OpenSlx.Lib.QuickForms.Interfaces;

namespace OpenSlx.Lib.Web.Controls
{

    /// <summary>
    /// Adaptation of the EntityDataSource for an entity bound to a Saleslogix property.
    /// Suitable for editable datagrids on Saleslogix tabs.
    /// </summary>
    [ToolboxData("<{0}:SlxEntityDataSource runat=server></{0}:SlxEntityDataSource>")]
    public class SlxEntityDataSource : EntityDataSource, IQFEntityDataSource
    {
        [Description("Name of the property to access in the form's entity to retrieve the collection to bind to.  To bind to the entity itself, use a period."),
        DefaultValue(".")]
        public String EntityDataSourceProperty { get; set; }

        /// <summary>
        /// Alias for the EntityDataSourceProperty
        /// (this is used from QuickForms for consistency with the Sage naming
        /// </summary>
        public String GetByProperty
        {
            get { return EntityDataSourceProperty; }
            set { EntityDataSourceProperty = value; }
        }

        private IPersistentEntity _sourceEntity = null;
        /// <summary>
        /// Specify the entity that the data will be pulled from.
        /// By default, SlxEntityDataSource will attempt to retrieve this from the smart part's data binding.
        /// </summary>
        public IPersistentEntity SourceEntity
        {
            get
            {
                if (_sourceEntity != null)
                    return _sourceEntity;
                return (IPersistentEntity)FindParentSmartPart().BindingSource.Current;
            }
            set
            {
                _sourceEntity = value;
            }
        }

        // List of fields to be included in the retrieved data, separated by commas.
        // This can be used to include sub-properties, e.g. Product.Price.
        // If non-null a dictionary object containing all the specified properties will be returned,
        // otherwise the original object will be returned.
        // (TODO)
        //public String SelectFields { get; set; }

        /// <summary>
        /// Attempt to find the smart part containing this control.
        /// </summary>
        /// <returns></returns>
        private EntityBoundSmartPart FindParentSmartPart()
        {
            Control iteration = this.Parent;
            while (iteration != null)
            {
                if (iteration is EntityBoundSmartPart)
                    return (EntityBoundSmartPart)iteration;
                iteration = iteration.Parent;
            }
            throw new InvalidOperationException(Resources.SlxEntityDataSourceCanOnlyBeUsedOnAnEntity);
        }

        /// <summary>
        /// Populate the parent's data source collection based on the provided data source property name.
        /// </summary>
        private void PopulateDataSource()
        {
            object entity = SourceEntity;
            if (entity == null)
                return;
            if (EntityDataSourceProperty == ".")
                DataSource = new object[] { entity };
            else
            {
                try
                {
                    DataSource = ((IEnumerable)ReflectionHelper.GetPropertyValue(entity, EntityDataSourceProperty, new WebCacheService()))
                        .Cast<object>();
                }
                catch (InvalidCastException)
                {
                    throw new InvalidOperationException(String.Format(Resources.PropertyX0MustBeCastableToICollection, EntityDataSourceProperty));
                }
            }
        }

        protected override void OnInit(EventArgs e)
        {
            base.OnInit(e);
        }

        /// <summary>
        /// Reload the entity collection and signal connected objects that the data may have changed.
        /// </summary>
        public override void Refresh()
        {
            PopulateDataSource();
            base.Refresh();
        }

        protected override DataSourceView GetView(string viewName)
        {
            if (!String.IsNullOrEmpty(EntityDataSourceProperty))
                PopulateDataSource();
            EntityDataSourceView view = (EntityDataSourceView)base.GetView(viewName);
            // we use that for SalesLogix in order to be able to match by InstanceId or Id
            view.MatchByAnyKey = true;
            return view;
        }

        /// <summary>
        /// Remove entities.
        /// If the DeleteMethodName is specified then this will be called, otherwise we'll try and remove them
        /// from the parent collection.
        /// </summary>
        /// <param name="entities"></param>
        /// <returns></returns>
        public override int DeleteEntities(IEnumerable<object> entities)
        {
            // either way we delete them, most likely we'll need a refresh
            var panelRefresh = ((ApplicationPage)Page).PageWorkItem.Services.Get<IPanelRefreshService>();
            if (panelRefresh != null)
                panelRefresh.RefreshAll();

            if (!String.IsNullOrEmpty(this.DeleteMethodName))
                return base.DeleteEntities(entities);
            // fallback - delete entities by removing them from the collection

            // remove reference from the parent's collection (if there is one)
            object parentEntity = SourceEntity;
            Debug.Assert(parentEntity != null, "parentEntity != null");
            object parentCollection = ReflectionHelper.GetPropertyValue(parentEntity,
                this.EntityDataSourceProperty, new WebCacheService());
            Debug.Assert(parentCollection != null, "parentCollection != null");
            MethodInfo funRemove = parentCollection.GetType().GetMethod("Remove");
            Debug.Assert(funRemove != null, "funRemove != null");
            int count = 0;
            foreach (IPersistentEntity child in entities)
            {
                funRemove.Invoke(parentCollection, new object[] { child });
                // NOW we can remove it
                if (child.PersistentState == PersistentState.Modified || child.PersistentState == PersistentState.Unmodified)
                    child.Delete();
                count++;
            }
            return count;
        }

        /*
         * an experiment to try and pull "linked" entities  (e.g. Contact.Account.AccountName)
         * Not working yet
        public override IEnumerable DecorateList(IEnumerable pagedList)
        {
            if (SelectFields == null)
                return base.DecorateList(pagedList);
            var cache = IoC.Resolve<ICacheService>();
            String[] fields = SelectFields.Split(new char[] { ',' });
            for (int i = 0; i < fields.Length; i++)
            {
                fields[i] = fields[i].Trim();
            }
            var results = new List<object>();
            foreach (object o in pagedList)
            {
                ComponentView componentView = new ComponentView(fields, o);
                results.Add(componentView);
            }
            return results;
        }
        */
        /*
         * Following 2 are work arounds that no longer appear to be required
        /// <summary>
        /// Work around for tabs in Saleslogix.
        /// We have to force a reload of the data after the postback data has been processed, otherwise the
        /// values from one tab will persist on the next one!
        /// </summary>
        /// <param name="e"></param>
        void parent_FormBound(object sender, EventArgs e)
        {
            var entity = (IComponentReference)EntityContext.GetEntity();

            if (!object.Equals(ViewState["PrevEntityId"], entity.Id))
            {
                if (ViewState["PrevEntityId"] != null)
                {
                    // force a reload, if they navigate to a different entity
                    Refresh();
                }

                ViewState["PrevEntityId"] = entity.Id;
            }
        }

        /// <summary>
        /// Work around for tabs in Saleslogix.
        /// We have to cancel the update if they have navigated to a different entity.
        /// </summary>
        /// <param name="entities"></param>
        /// <param name="values"></param>
        /// <param name="oldValues"></param>
        /// <returns></returns>
        public override int UpdateEntities(IEnumerable<object> entities, IDictionary values, IDictionary oldValues)
        {
            var entity = (IComponentReference)EntityContext.GetEntity();

            if (object.Equals(ViewState["PrevEntityId"], entity.Id))
                return base.UpdateEntities(entities, values, oldValues);
            else
                return 0;
        }
         */
    }

}