text:
"""Tests for DNNEstimators."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import json
import tempfile

import numpy as np
import tensorflow as tf

from tensorflow.contrib.learn.python.learn.estimators import _sklearn
from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils
from tensorflow.contrib.learn.python.learn.estimators import test_data
from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec
from tensorflow.python.ops import math_ops

class DNNClassifierTest(tf.test.TestCase):

  def _assertInRange(self, expected_min, expected_max, actual):
    self.assertLessEqual(expected_min, actual)
    self.assertGreaterEqual(expected_max, actual)

  def testEstimatorContract(self):
    estimator_test_utils.assert_estimator_contract(
        self, tf.contrib.learn.DNNClassifier)

  def testLogisticRegression_MatrixData(self):
    """Tests binary classification using matrix data as input."""
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    classifier = tf.contrib.learn.DNNClassifier(
        feature_columns=cont_features,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    input_fn = test_data.iris_input_logistic_fn
    classifier.fit(input_fn=input_fn, steps=5)
    scores = classifier.evaluate(input_fn=input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)

  def testLogisticRegression_MatrixData_Labels1D(self):
    """Same as the last test, but label shape is [100] instead of [100, 1]."""
    def _input_fn():
      iris = test_data.prepare_iris_data_for_logistic_regression()
      return {
          'feature': tf.constant(iris.data, dtype=tf.float32)
      }, tf.constant(iris.target, shape=[100], dtype=tf.int32)

    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    classifier = tf.contrib.learn.DNNClassifier(
        feature_columns=cont_features,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=5)
    scores = classifier.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)

  def testLogisticRegression_NpMatrixData(self):
    """Tests binary classification using numpy matrix data as input."""
    iris = test_data.prepare_iris_data_for_logistic_regression()
    train_x = iris.data
    train_y = iris.target
    feature_columns = [tf.contrib.layers.real_valued_column('', dimension=4)]
    classifier = tf.contrib.learn.DNNClassifier(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(x=train_x, y=train_y, steps=5)
    scores = classifier.evaluate(x=train_x, y=train_y, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])

  def testLogisticRegression_TensorData(self):
    """Tests binary classification using tensor data as input."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[.8], [0.2], [.1]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([[1], [0], [0]], dtype=tf.int32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(language_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=2,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=5)

    scores = classifier.evaluate(input_fn=_input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = list(
        classifier.predict(input_fn=predict_input_fn, as_iterable=True))
    self.assertListEqual(predictions, [1, 0, 0])

  def testLogisticRegression_FloatLabel(self):
    """Tests binary classification with float labels."""
    def _input_fn_float_label(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[50], [20], [10]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      labels = tf.constant([[0.8], [0.], [0.2]], dtype=tf.float32)
      return features, labels

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(language_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=2,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn_float_label, steps=50)

    predict_input_fn = functools.partial(_input_fn_float_label, num_epochs=1)
    predictions_proba = list(
        classifier.predict_proba(input_fn=predict_input_fn, as_iterable=True))
    self.assertEqual(3, len(predictions_proba))
    predictions = list(
        classifier.predict(input_fn=predict_input_fn, as_iterable=True))
    self.assertEqual(3, len(predictions))
    for b in range(3):
      self.assertEqual(2, len(predictions_proba[b]))
      for i in range(2):
        self._assertInRange(0.0, 1.0, predictions_proba[b][i])
      self.assertTrue(predictions[b] in (0, 1))

  def testMultiClass_MatrixData(self):
    """Tests multi-class classification using matrix data as input."""
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=cont_features,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    input_fn = test_data.iris_input_multiclass_fn
    classifier.fit(input_fn=input_fn, steps=200)
    scores = classifier.evaluate(input_fn=input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)

  def testMultiClass_MatrixData_Labels1D(self):
    """Same as the last test, but label shape is [150] instead of [150, 1]."""
    def _input_fn():
      iris = tf.contrib.learn.datasets.load_iris()
      return {
          'feature': tf.constant(iris.data, dtype=tf.float32)
      }, tf.constant(iris.target, shape=[150], dtype=tf.int32)

    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=cont_features,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=200)
    scores = classifier.evaluate(input_fn=_input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])

  def testMultiClass_NpMatrixData(self):
    """Tests multi-class classification using numpy matrix data as input."""
    iris = tf.contrib.learn.datasets.load_iris()
    train_x = iris.data
    train_y = iris.target
    feature_columns = [tf.contrib.layers.real_valued_column('', dimension=4)]
    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    classifier.fit(x=train_x, y=train_y, steps=200)
    scores = classifier.evaluate(x=train_x, y=train_y, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])

  def testLoss(self):
    """Tests loss calculation."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # The logistic prediction should be (y = 0.25).
      labels = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
      }
      return features, labels

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=2,
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn_train, steps=5)
    scores = classifier.evaluate(input_fn=_input_fn_train, steps=1)
    self.assertIn('loss', scores)

  def testLossWithWeights(self):
    """Tests loss calculation with weights."""

    def _input_fn_train():
      # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x))
      # The logistic prediction should be (y = 0.25).
      labels = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, labels

    def _input_fn_eval():
      # 4 rows, with different weights.
      labels = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[7.], [1.], [1.], [1.]])
      }
      return features, labels

    classifier = tf.contrib.learn.DNNClassifier(
        weight_column_name='w',
        n_classes=2,
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn_train, steps=5)
    scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1)
    self.assertIn('loss', scores)

  def testTrainWithWeights(self):
    """Tests training with given weight column."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # First row has more weight than others. Model should fit (y=x) better
      # than (y=Not(x)) due to the relative higher weight of the first row.
      labels = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[100.], [3.], [2.], [2.]])
      }
      return features, labels

    def _input_fn_eval():
      # Create 4 rows (y = x)
      labels = tf.constant([[1], [1], [1], [1]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, labels

    classifier = tf.contrib.learn.DNNClassifier(
        weight_column_name='w',
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    classifier.fit(input_fn=_input_fn_train, steps=5)
    scores = classifier.evaluate(input_fn=_input_fn_eval, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])

  def testPredict_AsIterableFalse(self):
    """Tests predict and predict_prob methods with as_iterable=False."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[.8], [.2], [.1]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([[1], [0], [0]], dtype=tf.int32)

    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1)
    ]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    classifier.fit(input_fn=_input_fn, steps=100)

    scores = classifier.evaluate(input_fn=_input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)
    predictions = classifier.predict(input_fn=_input_fn, as_iterable=False)
    self.assertListEqual(list(predictions), [1, 0, 0])
    predictions = classifier.predict_proba(input_fn=_input_fn,
                                           as_iterable=False)
    self.assertAllClose(
        predictions, [[0., 1., 0.], [1., 0., 0.], [1., 0., 0.]], atol=0.1)

  def testPredict_AsIterable(self):
    """Tests predict and predict_prob methods with as_iterable=True."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[.8], [.2], [.1]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([[1], [0], [0]], dtype=tf.int32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(language_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=200)

    scores = classifier.evaluate(input_fn=_input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = list(
        classifier.predict(input_fn=predict_input_fn, as_iterable=True))
    self.assertListEqual(predictions, [1, 0, 0])
    predictions = list(
        classifier.predict_proba(input_fn=predict_input_fn, as_iterable=True))
    self.assertAllClose(
        predictions, [[0., 1., 0.], [1., 0., 0.], [1., 0., 0.]], atol=0.3)

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""
    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      labels = tf.constant([[1], [0], [0], [0]])
      features = {
          'x': tf.train.limit_epochs(
              tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs),
      }
      return features, labels

    def _my_metric_op(predictions, labels):
      # For the case of binary classification, the 2nd column of "predictions"
      # denotes the model predictions.
      labels = tf.to_float(labels)
      predictions = tf.slice(predictions, [0, 1], [-1, 1])
      labels = math_ops.cast(labels, predictions.dtype)
      return tf.reduce_sum(tf.mul(predictions, labels))

    classifier = tf.contrib.learn.DNNClassifier(
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=5)
    scores = classifier.evaluate(
        input_fn=_input_fn,
        steps=5,
        metrics={
            'my_accuracy': MetricSpec(
                metric_fn=tf.contrib.metrics.streaming_accuracy,
                prediction_key='classes'),
            'my_precision': MetricSpec(
                metric_fn=tf.contrib.metrics.streaming_precision,
                prediction_key='classes'),
            'my_metric': MetricSpec(
                metric_fn=_my_metric_op,
                prediction_key='probabilities')
        })
    self.assertTrue(
        set(['loss', 'my_accuracy', 'my_precision', 'my_metric'
            ]).issubset(set(scores.keys())))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(list(classifier.predict(input_fn=predict_input_fn)))
    self.assertEqual(_sklearn.accuracy_score([1, 0, 0, 0], predictions),
                     scores['my_accuracy'])

    # Test the case where the 2nd element of the key is neither "classes" nor
    # "probabilities".
    with self.assertRaisesRegexp(KeyError, 'bad_type'):
      classifier.evaluate(
          input_fn=_input_fn,
          steps=5,
          metrics={
              'bad_name': MetricSpec(
                  metric_fn=tf.contrib.metrics.streaming_auc,
                  prediction_key='bad_type')})

  def testTrainSaveLoad(self):
    """Tests that insures you can save and reload a trained model."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[.8], [.2], [.1]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([[1], [0], [0]], dtype=tf.int32)

    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1)
    ]

    model_dir = tempfile.mkdtemp()
    classifier = tf.contrib.learn.DNNClassifier(
        model_dir=model_dir,
        n_classes=3,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    classifier.fit(input_fn=_input_fn, steps=5)
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions1 = classifier.predict(input_fn=predict_input_fn)
    del classifier

    classifier2 = tf.contrib.learn.DNNClassifier(
        model_dir=model_dir,
        n_classes=3,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))
    predictions2 = classifier2.predict(input_fn=predict_input_fn)
    self.assertEqual(list(predictions1), list(predictions2))

  def testTrainWithPartitionedVariables(self):
    """Tests training with partitioned variables."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[.8], [.2], [.1]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([[1], [0], [0]], dtype=tf.int32)

    # The given hash_bucket_size results in variables larger than the
    # default min_slice_size attribute, so the variables are partitioned.
    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=2e7)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1)
    ]

    tf_config = {
        'cluster': {
            tf.contrib.learn.TaskType.PS: ['fake_ps_0', 'fake_ps_1']
        }
    }
    with tf.test.mock.patch.dict('os.environ',
                                 {'TF_CONFIG': json.dumps(tf_config)}):
      config = tf.contrib.learn.RunConfig(tf_random_seed=5)
      # Because we did not start a distributed cluster, we need to pass an
      # empty ClusterSpec, otherwise the device_setter will look for
      # distributed jobs, such as "/job:ps" which are not present.
      config._cluster_spec = tf.train.ClusterSpec({})

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=config)

    classifier.fit(input_fn=_input_fn, steps=5)
    scores = classifier.evaluate(input_fn=_input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)

  def testExport(self):
    """Tests export model for servo."""

    def input_fn():
      return {
          'age': tf.constant([1]),
          'language': tf.SparseTensor(values=['english'],
                                      indices=[[0, 0]],
                                      shape=[1, 1])
      }, tf.constant([[1]])

    language = tf.contrib.layers.sparse_column_with_hash_bucket('language', 100)
    feature_columns = [
        tf.contrib.layers.real_valued_column('age'),
        tf.contrib.layers.embedding_column(language, dimension=1)
    ]

    classifier = tf.contrib.learn.DNNClassifier(
        feature_columns=feature_columns,
        hidden_units=[3, 3])
    classifier.fit(input_fn=input_fn, steps=5)

    export_dir = tempfile.mkdtemp()
    classifier.export(export_dir)

  def testEnableCenteredBias(self):
    """Tests that we can enable centered bias."""
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=cont_features,
        hidden_units=[3, 3],
        enable_centered_bias=True,
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    input_fn = test_data.iris_input_multiclass_fn
    classifier.fit(input_fn=input_fn, steps=5)
    self.assertIn('centered_bias_weight', classifier.get_variable_names())
    scores = classifier.evaluate(input_fn=input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)

  def testDisableCenteredBias(self):
    """Tests that we can disable centered bias."""
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    classifier = tf.contrib.learn.DNNClassifier(
        n_classes=3,
        feature_columns=cont_features,
        hidden_units=[3, 3],
        enable_centered_bias=False,
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    input_fn = test_data.iris_input_multiclass_fn
    classifier.fit(input_fn=input_fn, steps=5)
    self.assertNotIn('centered_bias_weight', classifier.get_variable_names())
    scores = classifier.evaluate(input_fn=input_fn, steps=1)
    self._assertInRange(0.0, 1.0, scores['accuracy'])
    self.assertIn('loss', scores)

class DNNRegressorTest(tf.test.TestCase):

  def testEstimatorContract(self):
    estimator_test_utils.assert_estimator_contract(
        self, tf.contrib.learn.DNNRegressor)

  def testRegression_MatrixData(self):
    """Tests regression using matrix data as input."""
    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=cont_features,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    input_fn = test_data.iris_input_logistic_fn
    regressor.fit(input_fn=input_fn, steps=200)
    scores = regressor.evaluate(input_fn=input_fn, steps=1)
    self.assertIn('loss', scores)

  def testRegression_MatrixData_Labels1D(self):
    """Same as the last test, but label shape is [100] instead of [100, 1]."""
    def _input_fn():
      iris = test_data.prepare_iris_data_for_logistic_regression()
      return {
          'feature': tf.constant(iris.data, dtype=tf.float32)
      }, tf.constant(iris.target, shape=[100], dtype=tf.int32)

    cont_features = [
        tf.contrib.layers.real_valued_column('feature', dimension=4)]

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=cont_features,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=200)
    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)

  def testRegression_NpMatrixData(self):
    """Tests binary classification using numpy matrix data as input."""
    iris = test_data.prepare_iris_data_for_logistic_regression()
    train_x = iris.data
    train_y = iris.target
    feature_columns = [tf.contrib.layers.real_valued_column('', dimension=4)]
    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(x=train_x, y=train_y, steps=200)
    scores = regressor.evaluate(x=train_x, y=train_y, steps=1)
    self.assertIn('loss', scores)

  def testRegression_TensorData(self):
    """Tests regression using tensor data as input."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[.8], [.15], [0.]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    language_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(language_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=200)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)

  def testLoss(self):
    """Tests loss calculation."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # The algorithm should learn (y = 0.25).
      labels = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
      }
      return features, labels

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    regressor.fit(input_fn=_input_fn_train, steps=5)
    scores = regressor.evaluate(input_fn=_input_fn_train, steps=1)
    self.assertIn('loss', scores)

  def testLossWithWeights(self):
    """Tests loss calculation with weights."""

    def _input_fn_train():
      # 4 rows with equal weight, one of them (y = x), three of them (y=Not(x))
      # The algorithm should learn (y = 0.25).
      labels = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, labels

    def _input_fn_eval():
      # 4 rows, with different weights.
      labels = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[7.], [1.], [1.], [1.]])
      }
      return features, labels

    regressor = tf.contrib.learn.DNNRegressor(
        weight_column_name='w',
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn_train, steps=5)
    scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1)
    self.assertIn('loss', scores)

  def testTrainWithWeights(self):
    """Tests training with given weight column."""

    def _input_fn_train():
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      # First row has more weight than others. Model should fit (y=x) better
      # than (y=Not(x)) due to the relative higher weight of the first row.
      labels = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[100.], [3.], [2.], [2.]])
      }
      return features, labels

    def _input_fn_eval():
      # Create 4 rows (y = x)
      labels = tf.constant([[1.], [1.], [1.], [1.]])
      features = {
          'x': tf.ones(shape=[4, 1], dtype=tf.float32),
          'w': tf.constant([[1.], [1.], [1.], [1.]])
      }
      return features, labels

    regressor = tf.contrib.learn.DNNRegressor(
        weight_column_name='w',
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    regressor.fit(input_fn=_input_fn_train, steps=5)
    scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1)
    self.assertIn('loss', scores)

  def testPredict_AsIterableFalse(self):
    """Tests predict method with as_iterable=False."""
    labels = [1., 0., 0.2]
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[0.8], [0.15], [0.]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant(labels, dtype=tf.float32)

    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    regressor.fit(input_fn=_input_fn, steps=200)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)
    predictions = regressor.predict(input_fn=_input_fn, as_iterable=False)
    self.assertAllClose(labels, predictions, atol=0.2)

  def testPredict_AsIterable(self):
    """Tests predict method with as_iterable=True."""
    labels = [1., 0., 0.2]
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[0.8], [0.15], [0.]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant(labels, dtype=tf.float32)

    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    regressor.fit(input_fn=_input_fn, steps=200)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = list(
        regressor.predict(input_fn=predict_input_fn, as_iterable=True))
    self.assertAllClose(labels, predictions, atol=0.2)

  def testCustomMetrics(self):
    """Tests custom evaluation metrics."""
    def _input_fn(num_epochs=None):
      # Create 4 rows, one of them (y = x), three of them (y=Not(x))
      labels = tf.constant([[1.], [0.], [0.], [0.]])
      features = {
          'x': tf.train.limit_epochs(
              tf.ones(shape=[4, 1], dtype=tf.float32), num_epochs=num_epochs),
      }
      return features, labels

    def _my_metric_op(predictions, labels):
      return tf.reduce_sum(tf.mul(predictions, labels))

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=[tf.contrib.layers.real_valued_column('x')],
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=1))

    regressor.fit(input_fn=_input_fn, steps=5)
    scores = regressor.evaluate(
        input_fn=_input_fn,
        steps=1,
        metrics={
            'my_error': tf.contrib.metrics.streaming_mean_squared_error,
            'my_metric': _my_metric_op
        })
    self.assertIn('loss', set(scores.keys()))
    self.assertIn('my_error', set(scores.keys()))
    self.assertIn('my_metric', set(scores.keys()))
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = np.array(list(regressor.predict(input_fn=predict_input_fn)))
    self.assertAlmostEqual(
        _sklearn.mean_squared_error(np.array([1, 0, 0, 0]), predictions),
        scores['my_error'])

    # Tests that when the key is a tuple, an error is raised.
    with self.assertRaises(KeyError):
      regressor.evaluate(
          input_fn=_input_fn,
          steps=1,
          metrics={('my_error', 'predictions'):
                   tf.contrib.metrics.streaming_mean_squared_error})

  def testTrainSaveLoad(self):
    """Tests that insures you can save and reload a trained model."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[0.8], [0.15], [0.]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    model_dir = tempfile.mkdtemp()
    regressor = tf.contrib.learn.DNNRegressor(
        model_dir=model_dir,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    regressor.fit(input_fn=_input_fn, steps=5)
    predict_input_fn = functools.partial(_input_fn, num_epochs=1)
    predictions = list(regressor.predict(input_fn=predict_input_fn))
    del regressor

    regressor2 = tf.contrib.learn.DNNRegressor(
        model_dir=model_dir,
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))
    predictions2 = list(regressor2.predict(input_fn=predict_input_fn))
    self.assertAllClose(predictions, predictions2)

  def testTrainWithPartitionedVariables(self):
    """Tests training with partitioned variables."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[0.8], [0.15], [0.]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    # The given hash_bucket_size results in variables larger than the
    # default min_slice_size attribute, so the variables are partitioned.
    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=2e7)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    tf_config = {
        'cluster': {
            tf.contrib.learn.TaskType.PS: ['fake_ps_0', 'fake_ps_1']
        }
    }
    with tf.test.mock.patch.dict('os.environ',
                                 {'TF_CONFIG': json.dumps(tf_config)}):
      config = tf.contrib.learn.RunConfig(tf_random_seed=1)
      # Because we did not start a distributed cluster, we need to pass an
      # empty ClusterSpec, otherwise the device_setter will look for
      # distributed jobs, such as "/job:ps" which are not present.
      config._cluster_spec = tf.train.ClusterSpec({})

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        config=config)

    regressor.fit(input_fn=_input_fn, steps=5)

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)

  def testEnableCenteredBias(self):
    """Tests that we can enable centered bias."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[0.8], [0.15], [0.]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        enable_centered_bias=True,
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    regressor.fit(input_fn=_input_fn, steps=5)
    self.assertIn('centered_bias_weight', regressor.get_variable_names())

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)

  def testDisableCenteredBias(self):
    """Tests that we can disable centered bias."""
    def _input_fn(num_epochs=None):
      features = {
          'age': tf.train.limit_epochs(
              tf.constant([[0.8], [0.15], [0.]]), num_epochs=num_epochs),
          'language': tf.SparseTensor(
              values=tf.train.limit_epochs(
                  ['en', 'fr', 'zh'], num_epochs=num_epochs),
              indices=[[0, 0], [0, 1], [2, 0]],
              shape=[3, 2])
      }
      return features, tf.constant([1., 0., 0.2], dtype=tf.float32)

    sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
        'language', hash_bucket_size=20)
    feature_columns = [
        tf.contrib.layers.embedding_column(sparse_column, dimension=1),
        tf.contrib.layers.real_valued_column('age')
    ]

    regressor = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns,
        hidden_units=[3, 3],
        enable_centered_bias=False,
        config=tf.contrib.learn.RunConfig(tf_random_seed=3))

    regressor.fit(input_fn=_input_fn, steps=5)
    self.assertNotIn('centered_bias_weight', regressor.get_variable_names())

    scores = regressor.evaluate(input_fn=_input_fn, steps=1)
    self.assertIn('loss', scores)

def boston_input_fn():
  boston = tf.contrib.learn.datasets.load_boston()
  features = tf.cast(tf.reshape(tf.constant(boston.data), [-1, 13]), tf.float32)
  labels = tf.cast(tf.reshape(tf.constant(boston.target), [-1, 1]), tf.float32)
  return features, labels

class FeatureColumnTest(tf.test.TestCase):

  def testTrain(self):
    feature_columns = tf.contrib.learn.infer_real_valued_columns_from_input_fn(
        boston_input_fn)
    est = tf.contrib.learn.DNNRegressor(
        feature_columns=feature_columns, hidden_units=[3, 3])
    est.fit(input_fn=boston_input_fn, steps=1)
    _ = est.evaluate(input_fn=boston_input_fn, steps=1)

if __name__ == '__main__':
  tf.test.main()


text:
var EventEmitter = require('events').EventEmitter;
var utils = require('./utils');
var util = require('util');

exports.Channels = Channels;

/**
 * Channel
 * @param channels
 * @param name
 * @returns {Channel}
 * @constructor
 */
function Channel(channels, name, options) {
    if (!(this instanceof Channel)) {
        return new Channel(channels, name, options);
    }
    this.channels = channels;
    this.adapter = channels.adapter;
    this.name = name;
    this.options = options;

    return this;
}

util.inherits(Channel, require('events').EventEmitter);

Channel.prototype.bind = Channel.prototype.on;

Channel.prototype.subscribe = function (cb) {
    var self = this;
    this.adapter.subscribe(this.name, this.options, function (err) {
        if (cb) cb.call(self, err, self);
    });
    return this;
};

/**
 * unsubscribe - unsubscribe from channel
 *
 * @param {Function} [cb] - callback fired on unsuback
 * @returns {Channel} this - for chaining
 * @example channel.unsubscribe('topic');
 * @example channel.unsubscribe('topic', console.log);
 */
Channel.prototype.unsubscribe = function (cb) {
    this.adapter.unsubscribe(this.name, {}, cb);
    return this;
};

Channel.prototype.__handleMessage = function (message) {
    message = JSON.parse(message);
    if (message.__event__ && message.__data__) {
        this.emit(message.__event__, message.__data__);
    }
};

/**
 * Channels
 *
 * @param adapter
 * @constructor
 */
function Channels(adapter) {
    this.adapter = adapter;
    this._channels = {};
}

Channels.prototype.add = function (cname, options) {
    var c = this._channels[cname];
    if (!c) {
        c = new Channel(this, cname, options);
        this._channels[cname] = c;
    }
    return c;
};

Channels.prototype.remove = function (cname) {
    var channel = this._channels[cname];
    delete this._channels[cname];
    return channel;
};

Channels.prototype.channel = function (cname) {
    return this._channels[cname];
};

Channels.prototype.unsubscribeAll = function (cb) {
    if (!this.channels) return cb();
    var invokers = [];
    utils.forEach(this.channels, function (channel) {
        invokers.push(channel.unsubscribe.bind(channel));
    });
    return utils.parallel(invokers, cb);
};

redpajama_set_name: RedPajamaGithub


text:
namespace tg
{
	swift::ScriptHandle GameAssets::getScript(const std::string& n)
	{
		auto s = scripts.find(n);
		if(s != scripts.end())
		{
			auto script = std::make_unique<GameScript>();
			script->loadFromFile(s->second);

			return script;
		}

		throw swift::ResourceNotFound("Could not find script: " + n);
	}
}


text:
Get the union of two sorted streams

```
npm install sorted-union-stream
```

[![build status](https://secure.travis-ci.org/mafintosh/sorted-union-stream.png)](http://travis-ci.org/mafintosh/sorted-union-stream)

## Usage

``` js
var union = require('sorted-union-stream')
var from = require('from2-array')

// es.readArray converts an array into a stream
var sorted1 = from.obj([1,10,24,42,43,50,55])
var sorted2 = from.obj([10,42,53,55,60])

// combine the two streams into a single sorted stream
var u = union(sorted1, sorted2)

u.on('data', function(data) {
  console.log(data)
})
u.on('end', function() {
  console.log('no more data')
})
```

Running the above example will print

```
1
10
24
42
43
50
53
55
60
no more data
```

## Streaming objects

If you are streaming objects sorting is based on `.key`.

If this property is not present you should add a `toKey` function as the third parameter.
`toKey` should return an key representation of the data that can be used to compare objects.

_The keys MUST be sorted_

``` js
var sorted1 = from.obj([{foo:'a'}, {foo:'b'}, {foo:'c'}])
var sorted2 = from.obj([{foo:'b'}, {foo:'d'}])

var u = union(sorted1, sorted2, function(data) {
  return data.foo // the foo property is sorted
})

union.on('data', function(data) {
  console.log(data)
});
```

Running the above will print

```
{foo:'a'}
{foo:'b'}
{foo:'c'}
{foo:'d'}
```

## License

MIT


text:
<?php defined('BASEPATH') or exit('No direct script access allowed'); ?>

                    <?php $countries = get_all_countries(); ?>

                            <?php $value = (isset($credit_note) ? $credit_note->billing_street : ''); ?>
                            <?php echo render_textarea('billing_street','billing_street',$value); ?>
                            <?php $value = (isset($credit_note) ? $credit_note->billing_city : ''); ?>
                            <?php echo render_input('billing_city','billing_city',$value); ?>
                            <?php $value = (isset($credit_note) ? $credit_note->billing_state : ''); ?>
                            <?php echo render_input('billing_state','billing_state',$value); ?>
                            <?php $value = (isset($credit_note) ? $credit_note->billing_zip : ''); ?>
                            <?php echo render_input('billing_zip','billing_zip',$value); ?>
                            <?php $selected = (isset($credit_note) ? $credit_note->billing_country : ''); ?>
                            <?php echo render_select('billing_country',$countries,array('country_id',array('short_name'),'iso2'),'billing_country',$selected); ?>

                        <a href="#" class="pull-right" id="get_shipping_from_customer_profile" data-placement="left" data-toggle="tooltip" title="<?php echo _l('get_shipping_from_customer_profile'); ?>">

                                <input type="checkbox" id="include_shipping" name="include_shipping" <?php if(isset($credit_note) && $credit_note->include_shipping == 1){echo 'checked';} ?>>
                                <?php echo _l('shipping_address'); ?>

                        <div id="shipping_details" class="<?php if((isset($credit_note) && $credit_note->include_shipping != 1) || !isset($credit_note)){echo 'hide';} ?>">

                                <input type="checkbox" id="show_shipping_on_credit_note" name="show_shipping_on_credit_note" <?php if((isset($credit_note) && $credit_note->show_shipping_on_credit_note == 1) || !isset($credit_note)){echo 'checked';} ?>>
                                <?php echo _l('show_shipping_on_credit_note'); ?>

                        <?php $value = (isset($credit_note) ? $credit_note->shipping_street : ''); ?>
                        <?php echo render_textarea('shipping_street','shipping_street',$value); ?>
                        <?php $value = (isset($credit_note) ? $credit_note->shipping_city : ''); ?>
                        <?php echo render_input('shipping_city','shipping_city',$value); ?>
                        <?php $value = (isset($credit_note) ? $credit_note->shipping_state : ''); ?>
                        <?php echo render_input('shipping_state','shipping_state',$value); ?>
                        <?php $value = (isset($credit_note) ? $credit_note->shipping_zip : ''); ?>
                        <?php echo render_input('shipping_zip','shipping_zip',$value); ?>
                        <?php $selected = (isset($credit_note) ? $credit_note->shipping_country : ''); ?>
                        <?php echo render_select('shipping_country',$countries,array('country_id',array('short_name'),'iso2'),'shipping_country',$selected); ?>

            <?php echo _l('apply'); ?>


text:
package com.sun.dao;

import java.util.Map;

public interface LoginDao {

	public Map checkLogin(Map map);

}


text:
package biz.paluch.logging.gelf.intern.sender;

import java.nio.charset.StandardCharsets;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import biz.paluch.logging.gelf.intern.ErrorReporter;
import biz.paluch.logging.gelf.intern.GelfMessage;
import biz.paluch.logging.gelf.intern.GelfSender;

/**
 * {@link GelfSender} using Kafka.
 *
 * @author Rifat Döver
 * @since 1.13
 */
public class KafkaGelfSender implements GelfSender {

    private final KafkaProducer<byte[], byte[]> kafkaProducer;
    private final String topicName;
    private final ErrorReporter errorReporter;

    public KafkaGelfSender(KafkaProducer<byte[], byte[]> kafkaProducer, String topicName, ErrorReporter errorReporter) {
        this.kafkaProducer = kafkaProducer;
        this.topicName = topicName;
        this.errorReporter = errorReporter;
    }

    @Override
    public boolean sendMessage(GelfMessage message) {
        ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topicName,
                message.toJson().getBytes(StandardCharsets.UTF_8));
        boolean hasOffset;
        try {
            Future<RecordMetadata> metadata = kafkaProducer.send(record);
            hasOffset = metadata.get(30, TimeUnit.SECONDS).hasOffset();
        } catch (Exception e) {
            errorReporter.reportError("Error sending log to kafka", e);
            return false;
        }
        return hasOffset;
    }

    @Override
    public void close() {
        kafkaProducer.close();
    }
}


text:
From this PyCon 2017 talk: [Amjith Ramanujam Awesome Command Line Tools PyCon 2017](https://us.pycon.org/2017/schedule/presentation/518/)

<iframe width="560" height="315" src="https://www.youtube.com/embed/hJhZhLg3obk" frameborder="0" allowfullscreen></iframe>

[Slides](https://speakerdeck.com/amjith/awesome-commandline-tools)

Checklist of ideas

 - [ ] persistent history
 - [ ] history search
 - [ ] emacs keybindings
 - [ ] paged output
 - [ ] auto complete
 - [ ] minimal config
 - [ ] syntax coloring

Code snippet repl.py:

```python

from prompt_toolkit import prompt
from prompt_toolkit.history import FileHistory
from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
from prompt_toolkit.contrib.completers import WordCompleter
from pygments.lexers.sql import SqlLexer

SQLCompleter = WordCompleter(['SELECT',
                              'SHOW',
                              'FROM',
                              'WHERE',
                              ], ignore_case=True)

while 1:
    inp = print(u'> ',
                history=FileHistory('history.txt'),
                auto_suggest=AutoSuggestFromHistory(),
                completer=SQLCompleter,
                lexer=SqlLexer,
               )
    print(inp)

```


text:
<?php echo url('Accueil', ''); ?>

<?php echo url('Ajouter commentaire : ', 'livreor/creerCommentaire'); ?>

<?php foreach($liste_commentaires as $message): ?>
<div id="num_<?php echo $message->id; ?>" class="message">

		<a href="#num_<?php echo $message->id; ?>"># Par <?php echo htmlentities($message->pseudo); ?>
		le <?php echo $message->date; ?>

<?php echo nl2br(htmlentities($message->message)); ?>

<?php endforeach; ?>


text:
import os
from mercurial import demandimport
# these do not work with demandimport, blacklist
demandimport.ignore.extend([
        'bzrlib.transactions',
        'bzrlib.urlutils',
        'ElementPath',
    ])

from mercurial.i18n import _
from mercurial import util
from common import NoRepo, commit, converter_source

try:
    # bazaar imports
    from bzrlib import branch, revision, errors
    from bzrlib.revisionspec import RevisionSpec
except ImportError:
    pass

supportedkinds = ('file', 'symlink')

class bzr_source(converter_source):
    """Reads Bazaar repositories by using the Bazaar Python libraries"""

    def __init__(self, ui, path, rev=None):
        super(bzr_source, self).__init__(ui, path, rev=rev)

        if not os.path.exists(os.path.join(path, '.bzr')):
            raise NoRepo(_('%s does not look like a Bazaar repository')
                         % path)

        try:
            # access bzrlib stuff
            branch
        except NameError:
            raise NoRepo(_('Bazaar modules could not be loaded'))

        path = os.path.abspath(path)
        self._checkrepotype(path)
        self.branch = branch.Branch.open(path)
        self.sourcerepo = self.branch.repository
        self._parentids = {}

    def _checkrepotype(self, path):
        # Lightweight checkouts detection is informational but probably
        # fragile at API level. It should not terminate the conversion.
        try:
            from bzrlib import bzrdir
            dir = bzrdir.BzrDir.open_containing(path)[0]
            try:
                tree = dir.open_workingtree(recommend_upgrade=False)
                branch = tree.branch
            except (errors.NoWorkingTree, errors.NotLocalUrl), e:
                tree = None
                branch = dir.open_branch()
            if (tree is not None and tree.bzrdir.root_transport.base !=
                branch.bzrdir.root_transport.base):
                self.ui.warn(_('warning: lightweight checkouts may cause '
                               'conversion failures, try with a regular '
                               'branch instead.\n'))
        except:
            self.ui.note(_('bzr source type could not be determined\n'))

    def before(self):
        """Before the conversion begins, acquire a read lock
        for all the operations that might need it. Fortunately
        read locks don't block other reads or writes to the
        repository, so this shouldn't have any impact on the usage of
        the source repository.

        The alternative would be locking on every operation that
        needs locks (there are currently two: getting the file and
        getting the parent map) and releasing immediately after,
        but this approach can take even 40% longer."""
        self.sourcerepo.lock_read()

    def after(self):
        self.sourcerepo.unlock()

    def getheads(self):
        if not self.rev:
            return [self.branch.last_revision()]
        try:
            r = RevisionSpec.from_string(self.rev)
            info = r.in_history(self.branch)
        except errors.BzrError:
            raise util.Abort(_('%s is not a valid revision in current branch')
                             % self.rev)
        return [info.rev_id]

    def getfile(self, name, rev):
        revtree = self.sourcerepo.revision_tree(rev)
        fileid = revtree.path2id(name.decode(self.encoding or 'utf-8'))
        kind = None
        if fileid is not None:
            kind = revtree.kind(fileid)
        if kind not in supportedkinds:
            # the file is not available anymore - was deleted
            raise IOError(_('%s is not available in %s anymore') %
                    (name, rev))
        if kind == 'symlink':
            target = revtree.get_symlink_target(fileid)
            if target is None:
                raise util.Abort(_('%s.%s symlink has no target')
                                 % (name, rev))
            return target
        else:
            sio = revtree.get_file(fileid)
            return sio.read()

    def getmode(self, name, rev):
        return self._modecache[(name, rev)]

    def getchanges(self, version):
        # set up caches: modecache and revtree
        self._modecache = {}
        self._revtree = self.sourcerepo.revision_tree(version)
        # get the parentids from the cache
        parentids = self._parentids.pop(version)
        # only diff against first parent id
        prevtree = self.sourcerepo.revision_tree(parentids[0])
        return self._gettreechanges(self._revtree, prevtree)

    def getcommit(self, version):
        rev = self.sourcerepo.get_revision(version)
        # populate parent id cache
        if not rev.parent_ids:
            parents = []
            self._parentids[version] = (revision.NULL_REVISION,)
        else:
            parents = self._filterghosts(rev.parent_ids)
            self._parentids[version] = parents

        return commit(parents=parents,
                date='%d %d' % (rev.timestamp, -rev.timezone),
                author=self.recode(rev.committer),
                # bzr returns bytestrings or unicode, depending on the content
                desc=self.recode(rev.message),
                rev=version)

    def gettags(self):
        if not self.branch.supports_tags():
            return {}
        tagdict = self.branch.tags.get_tag_dict()
        bytetags = {}
        for name, rev in tagdict.iteritems():
            bytetags[self.recode(name)] = rev
        return bytetags

    def getchangedfiles(self, rev, i):
        self._modecache = {}
        curtree = self.sourcerepo.revision_tree(rev)
        if i is not None:
            parentid = self._parentids[rev][i]
        else:
            # no parent id, get the empty revision
            parentid = revision.NULL_REVISION

        prevtree = self.sourcerepo.revision_tree(parentid)
        changes = [e[0] for e in self._gettreechanges(curtree, prevtree)[0]]
        return changes

    def _gettreechanges(self, current, origin):
        revid = current._revision_id
        changes = []
        renames = {}
        for (fileid, paths, changed_content, versioned, parent, name,
            kind, executable) in current.iter_changes(origin):

            if paths[0] == u'' or paths[1] == u'':
                # ignore changes to tree root
                continue

            # bazaar tracks directories, mercurial does not, so
            # we have to rename the directory contents
            if kind[1] == 'directory':
                if kind[0] not in (None, 'directory'):
                    # Replacing 'something' with a directory, record it
                    # so it can be removed.
                    changes.append((self.recode(paths[0]), revid))

                if None not in paths and paths[0] != paths[1]:
                    # neither an add nor an delete - a move
                    # rename all directory contents manually
                    subdir = origin.inventory.path2id(paths[0])
                    # get all child-entries of the directory
                    for name, entry in origin.inventory.iter_entries(subdir):
                        # hg does not track directory renames
                        if entry.kind == 'directory':
                            continue
                        frompath = self.recode(paths[0] + '/' + name)
                        topath = self.recode(paths[1] + '/' + name)
                        # register the files as changed
                        changes.append((frompath, revid))
                        changes.append((topath, revid))
                        # add to mode cache
                        mode = ((entry.executable and 'x')
                                or (entry.kind == 'symlink' and 's')
                                or '')
                        self._modecache[(topath, revid)] = mode
                        # register the change as move
                        renames[topath] = frompath

                # no futher changes, go to the next change
                continue

            # we got unicode paths, need to convert them
            path, topath = [self.recode(part) for part in paths]

            if topath is None:
                # file deleted
                changes.append((path, revid))
                continue

            # renamed
            if path and path != topath:
                renames[topath] = path
                changes.append((path, revid))

            # populate the mode cache
            kind, executable = [e[1] for e in (kind, executable)]
            mode = ((executable and 'x') or (kind == 'symlink' and 'l')
                    or '')
            self._modecache[(topath, revid)] = mode
            changes.append((topath, revid))

        return changes, renames

    def _filterghosts(self, ids):
        """Filters out ghost revisions which hg does not support, see
        <http://bazaar-vcs.org/GhostRevision>
        """
        parentmap = self.sourcerepo.get_parent_map(ids)
        parents = tuple([parent for parent in ids if parent in parentmap])
        return parents

    def recode(self, s, encoding=None):
        """This version of recode tries to encode unicode to bytecode,
        and preferably using the UTF-8 codec.
        Other types than Unicode are silently returned, this is by
        intention, e.g. the None-type is not going to be encoded but instead
        just passed through
        """
        if not encoding:
            encoding = self.encoding or 'utf-8'

        if isinstance(s, unicode):
            return s.encode(encoding)
        else:
            # leave it alone
            return s


text:
namespace CivGen
{
    using System;
    using System.Collections.Generic;

    public partial class Color
    {
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2214:DoNotCallOverridableMethodsInConstructors")]
        public Color()
        {
            this.PlayerColors = new HashSet<PlayerColor>();
            this.PlayerColors1 = new HashSet<PlayerColor>();
            this.PlayerColors2 = new HashSet<PlayerColor>();
        }

        public string Type { get; set; }
        public double Red { get; set; }
        public double Green { get; set; }
        public double Blue { get; set; }
        public double Alpha { get; set; }

        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
        public virtual ICollection<PlayerColor> PlayerColors { get; set; }
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
        public virtual ICollection<PlayerColor> PlayerColors1 { get; set; }
        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
        public virtual ICollection<PlayerColor> PlayerColors2 { get; set; }
    }
}


text:
@implementation FetchMoviesRequest

- (NSURLRequest *) request {
    NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:SERVER_URL_FETCH_MOVIES]];
    [request setValue:HTTP_ACCEPT_HEADER_FIELD_VALUE forHTTPHeaderField:HTTP_ACCEPT_HEADER_FIELD_KEY];
    [request setValue:HTTP_CONTENT_TYPE_HEADER_FIELD_VALUE forHTTPHeaderField:HTTP_CONTENT_TYPE_HEADER_FIELD_KEY];
    return request;
}

- (void) executeWithCompletion:(void(^)(NSArray<Movie*> *movies, NSError *error))completion {
    NSURLSession *session = [NSURLSession sharedSession];
    NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:[self request] completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
        NSMutableArray<Movie*> *movies = [NSMutableArray array];
        if (error == nil) {
            NSError *jsonParseError;
            NSDictionary *json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&jsonParseError];
            for (NSDictionary *jsonMovie in json[@"movies"]) {
                [movies addObject:[[Movie alloc] initWithDictionary:jsonMovie]];
            }
        }
        completion(movies, error);
    }];
    [dataTask resume];
}

@end


text:
module Vault
  module Finders
    delegate :size, :to => :store
    alias_method :count, :size

    def all(query={})
      query.delete(key)
      store.filter(query).map do |key_value, properties|
        build(key_value, properties)
      end
    end

    def [](key_value)
      properties = store[key_value]
      build(key_value, properties) if properties
    end

    private

    def build(key_value, properties)
      new(properties.update(key => key_value))
    end
  end
end


text:
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Azure.Core.TestFramework;
using Azure.Storage.Files.Shares.Models;
using Azure.Storage.Sas;
using Azure.Storage.Test;
using Moq;
using NUnit.Framework;

namespace Azure.Storage.Files.Shares.Tests
{
    public class ServiceClientTests : FileTestBase
    {
        public ServiceClientTests(bool async, ShareClientOptions.ServiceVersion serviceVersion)
            : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */)
        {
        }

        [RecordedTest]
        public void Ctor_ConnectionString()
        {
            var accountName = "accountName";
            var accountKey = Convert.ToBase64String(new byte[] { 0, 1, 2, 3, 4, 5 });

            var credentials = new StorageSharedKeyCredential(accountName, accountKey);
            var fileEndpoint = new Uri("http://127.0.0.1/" + accountName);
            var fileSecondaryEndpoint = new Uri("http://127.0.0.1/" + accountName + "-secondary");

            var connectionString = new StorageConnectionString(credentials, (default, default), (default, default), (default, default), (fileEndpoint, fileSecondaryEndpoint));

            ShareServiceClient service = InstrumentClient(new ShareServiceClient(connectionString.ToString(true), GetOptions()));

            var builder = new ShareUriBuilder(service.Uri);

            Assert.AreEqual("", builder.ShareName);
            Assert.AreEqual("", builder.DirectoryOrFilePath);
            //Assert.AreEqual("accountName", builder.AccountName);
        }

        [RecordedTest]
        public async Task Ctor_AzureSasCredential()
        {
            // Arrange
            string sas = GetNewAccountSasCredentials(resourceTypes: AccountSasResourceTypes.All, permissions: AccountSasPermissions.All).ToString();
            Uri uri = SharesClientBuilder.GetServiceClient_SharedKey().Uri;

            // Act
            var sasClient = InstrumentClient(new ShareServiceClient(uri, new AzureSasCredential(sas), GetOptions()));
            ShareServiceProperties properties = await sasClient.GetPropertiesAsync();

            // Assert
            Assert.IsNotNull(properties);
        }

        [RecordedTest]
        public void Ctor_AzureSasCredential_VerifyNoSasInUri()
        {
            // Arrange
            string sas = GetNewAccountSasCredentials(resourceTypes: AccountSasResourceTypes.All, permissions: AccountSasPermissions.All).ToString();
            Uri uri = SharesClientBuilder.GetServiceClient_SharedKey().Uri;
            uri = new Uri(uri.ToString() + "?" + sas);

            // Act
            TestHelper.AssertExpectedException<ArgumentException>(
                () => new ShareClient(uri, new AzureSasCredential(sas)),
                e => e.Message.Contains($"You cannot use {nameof(AzureSasCredential)} when the resource URI also contains a Shared Access Signature"));
        }

        [RecordedTest]
        public async Task GetPropertiesAsync()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();

            // Act
            Response<ShareServiceProperties> properties = await service.GetPropertiesAsync();

            // Assert
            Assert.IsNotNull(properties);
            var accountName = new ShareUriBuilder(service.Uri).AccountName;
            TestHelper.AssertCacheableProperty(accountName, () => service.AccountName);
        }

        [RecordedTest]
        public async Task GetPropertiesAsync_Error()
        {
            // Arrange
            ShareServiceClient service = InstrumentClient(
                new ShareServiceClient(
                    s_invalidUri,
                    new StorageSharedKeyCredential(
                        TestConfigDefault.AccountName,
                        TestConfigDefault.AccountKey),
                    GetOptions()));

            // Act
            await TestHelper.AssertExpectedExceptionAsync<RequestFailedException>(
                service.GetPropertiesAsync(),
                e => Assert.AreEqual(ShareErrorCode.AuthenticationFailed.ToString(), e.ErrorCode));
        }

        [RecordedTest]
        [NonParallelizable]
        [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/15505")]
        public async Task SetPropertiesAsync()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();
            Response<ShareServiceProperties> properties = await service.GetPropertiesAsync();
            _ = properties.Value.Cors.ToArray();
            properties.Value.Cors.Clear();
            properties.Value.Cors.Add(
                new ShareCorsRule
                {
                    MaxAgeInSeconds = 1000,
                    AllowedHeaders = "x-ms-meta-data*,x-ms-meta-target*,x-ms-meta-abc",
                    AllowedMethods = "PUT,GET",
                    AllowedOrigins = "*",
                    ExposedHeaders = "x-ms-meta-*"
                });

            // Act
            await service.SetPropertiesAsync(properties: properties);

            // Assert
            properties = await service.GetPropertiesAsync();
            Assert.AreEqual(1, properties.Value.Cors.Count());
            Assert.IsTrue(properties.Value.Cors[0].MaxAgeInSeconds == 1000);
        }

        [RecordedTest]
        [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2019_12_12)]
        [NonParallelizable]
        public async Task GetSetServicePropertiesAsync_SmbMultiChannel()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_PremiumFile();

            // Act
            Response<ShareServiceProperties> propertiesResponse = await service.GetPropertiesAsync();
            ShareServiceProperties properties = propertiesResponse.Value;

            // Assert
            Assert.IsFalse(properties.Protocol.Smb.Multichannel.Enabled);

            // Act
            properties.Protocol.Smb.Multichannel.Enabled = true;
            await service.SetPropertiesAsync(properties);
            propertiesResponse = await service.GetPropertiesAsync();
            properties = propertiesResponse.Value;

            // Assert
            Assert.IsTrue(properties.Protocol.Smb.Multichannel.Enabled);

            // Cleanup
            properties.Protocol.Smb.Multichannel.Enabled = false;
            await service.SetPropertiesAsync(properties);
        }

        [RecordedTest]
        public async Task SetPropertiesAsync_Error()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();
            Response<ShareServiceProperties> properties = await service.GetPropertiesAsync();
            ShareServiceClient fakeService = InstrumentClient(
                new ShareServiceClient(
                    new Uri("https://error.file.core.windows.net"),
                    new StorageSharedKeyCredential(
                        TestConfigDefault.AccountName,
                        TestConfigDefault.AccountKey),
                    GetOptions()));

            // Act
            await TestHelper.AssertExpectedExceptionAsync<RequestFailedException>(
                fakeService.SetPropertiesAsync(properties),
                e => Assert.AreEqual(ShareErrorCode.AuthenticationFailed.ToString(), e.ErrorCode));
        }

        [RecordedTest]
        public async Task ListSharesSegmentAsync()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();

            // Ensure at least one share
            await using DisposingShare test = await GetTestShareAsync(service);
            ShareClient share = test.Share;

            var shares = new List<ShareItem>();
            await foreach (Page<ShareItem> page in service.GetSharesAsync().AsPages())
            {
                shares.AddRange(page.Values);
            }

            // Assert
            Assert.AreNotEqual(0, shares.Count);
            Assert.AreEqual(shares.Count, shares.Select(c => c.Name).Distinct().Count());
            Assert.IsTrue(shares.Any(c => share.Uri == service.GetShareClient(c.Name).Uri));
            Assert.IsTrue(shares.All(c => c.Properties.Metadata == null));
        }

        [RecordedTest]
        [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2019_07_07)]
        [Ignore("#10044: Re-enable failing Storage tests")]
        public async Task ListSharesSegmentAsync_Premium()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_PremiumFile();
            string shareName = GetNewShareName();

            // Ensure at least one premium share
            await using DisposingShare test = await GetTestShareAsync(
                service: service,
                shareName: shareName);
            ShareClient share = test.Share;

            var shares = new List<ShareItem>();
            await foreach (Page<ShareItem> page in service.GetSharesAsync().AsPages())
            {
                shares.AddRange(page.Values);
            }

            // Assert
            ShareItem premiumShareItem = shares.Where(r => r.Name == shareName).First();
            Assert.IsNotNull(premiumShareItem.Properties.ETag);
            Assert.IsNotNull(premiumShareItem.Properties.LastModified);
            Assert.IsNotNull(premiumShareItem.Properties.NextAllowedQuotaDowngradeTime);
            Assert.IsNotNull(premiumShareItem.Properties.ProvisionedEgressMBps);
            Assert.IsNotNull(premiumShareItem.Properties.ProvisionedIngressMBps);
            Assert.IsNotNull(premiumShareItem.Properties.ProvisionedIops);
            Assert.IsNotNull(premiumShareItem.Properties.QuotaInGB);
        }

        [RecordedTest]
        public async Task ListSharesSegmentAsync_Metadata()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();
            IDictionary<string, string> metadata = BuildMetadata();

            // Ensure at least one share
            await using DisposingShare test = await GetTestShareAsync(service, metadata: metadata);
            ShareClient share = test.Share;

            var shares = new List<ShareItem>();
            await foreach (Page<ShareItem> page in service.GetSharesAsync(ShareTraits.Metadata).AsPages())
            {
                shares.AddRange(page.Values);
            }

            // Assert
            Assert.AreNotEqual(0, shares.Count);
            Assert.AreEqual(shares.Count, shares.Select(c => c.Name).Distinct().Count());
            Assert.IsTrue(shares.Any(c => share.Uri == service.GetShareClient(c.Name).Uri));
            AssertDictionaryEquality(
                metadata,
                shares.Where(s => s.Name == test.Share.Name).FirstOrDefault().Properties.Metadata);
        }

        [RecordedTest]
        [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2019_12_12)]
        public async Task ListSharesSegmentAsync_Deleted()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SoftDelete();
            ShareClient share = InstrumentClient(service.GetShareClient(GetNewShareName()));
            await share.CreateAsync();
            await share.DeleteAsync();

            // Act
            IList<ShareItem> shares = await service.GetSharesAsync(states: ShareStates.Deleted).ToListAsync();

            // Assert
            ShareItem shareItem = shares.Where(s => s.Name == share.Name).FirstOrDefault();
            Assert.IsTrue(shareItem.IsDeleted);
            Assert.IsNotNull(shareItem.VersionId);
        }

        [RecordedTest]
        [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2019_12_12)]
        public async Task ListSharesSegmentAsync_AccessTier()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();

            // Ensure at least one share
            await using DisposingShare test = await GetTestShareAsync(service);
            ShareClient share = test.Share;

            ShareSetPropertiesOptions options = new ShareSetPropertiesOptions
            {
                AccessTier = ShareAccessTier.Hot
            };

            await test.Share.SetPropertiesAsync(options);

            var shares = new List<ShareItem>();
            await foreach (ShareItem shareItem in service.GetSharesAsync())
            {
                shares.Add(shareItem);
            }

            ShareItem shareItemForShare = shares.FirstOrDefault(r => r.Name == test.Share.Name);

            // Assert
            Assert.AreEqual(ShareAccessTier.Hot.ToString(), shareItemForShare.Properties.AccessTier);
            Assert.IsNotNull(shareItemForShare.Properties.AccessTierChangeTime);
            Assert.AreEqual("pending-from-transactionOptimized", shareItemForShare.Properties.AccessTierTransitionState);
        }

        [RecordedTest]
        public async Task ListShareSegmentAsync_Error()
        {
            // Arrange
            ShareServiceClient service = InstrumentClient(
                new ShareServiceClient(
                    new Uri("https://error.file.core.windows.net"),
                    new StorageSharedKeyCredential(
                        TestConfigDefault.AccountName,
                        TestConfigDefault.AccountKey),
                    GetOptions()));

            // Act
            await TestHelper.AssertExpectedExceptionAsync<RequestFailedException>(
                service.GetSharesAsync().ToListAsync(),
                e => Assert.AreEqual(ShareErrorCode.AuthenticationFailed.ToString(), e.ErrorCode));
        }

        [RecordedTest]
        [PlaybackOnly("https://github.com/Azure/azure-sdk-for-net/issues/17262")]
        [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2020_04_08)]
        public async Task ListSharesSegmentAsync_EnabledProtocolsAndRootSquash()
        {
            // Arrange
            var shareName = GetNewShareName();
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_PremiumFile();
            ShareClient share = InstrumentClient(service.GetShareClient(shareName));
            ShareCreateOptions options = new ShareCreateOptions
            {
                Protocols = ShareProtocols.Nfs,
                RootSquash = ShareRootSquash.AllSquash,
            };

            await share.CreateAsync(options);

            // Act
            IList<ShareItem> shares = await service.GetSharesAsync().ToListAsync();

            // Assert
            ShareItem shareItem = shares.Where(s => s.Name == share.Name).FirstOrDefault();
            Assert.AreEqual(ShareProtocols.Nfs, shareItem.Properties.Protocols);
            Assert.AreEqual(ShareRootSquash.AllSquash, shareItem.Properties.RootSquash);
        }

        [RecordedTest]
        public async Task CreateShareAsync()
        {
            var name = GetNewShareName();
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();
            try
            {
                ShareClient share = InstrumentClient((await service.CreateShareAsync(name)).Value);
                Response<ShareProperties> properties = await share.GetPropertiesAsync();
                Assert.AreNotEqual(0, properties.Value.QuotaInGB);
            }
            finally
            {
                await service.DeleteShareAsync(name, false);
            }
        }

        [RecordedTest]
        public async Task DeleteShareAsync()
        {
            var name = GetNewShareName();
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SharedKey();
            ShareClient share = InstrumentClient((await service.CreateShareAsync(name)).Value);

            await service.DeleteShareAsync(name, false);
            Assert.ThrowsAsync<RequestFailedException>(
                async () => await share.GetPropertiesAsync());
        }

        [RecordedTest]
        [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2019_12_12)]
        public async Task UndeleteShareAsync()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SoftDelete();
            string shareName = GetNewShareName();
            ShareClient share = InstrumentClient(service.GetShareClient(shareName));
            await share.CreateAsync();
            await share.DeleteAsync();
            IList<ShareItem> shares = await service.GetSharesAsync(states: ShareStates.Deleted).ToListAsync();
            ShareItem shareItem = shares.Where(s => s.Name == shareName).FirstOrDefault();

            // It takes some time for the Share to be deleted.
            await Delay(30000);

            // Act
            Response<ShareClient> response = await service.UndeleteShareAsync(
                shareItem.Name,
                shareItem.VersionId);

            // Assert
            await response.Value.GetPropertiesAsync();

            // Cleanup
            await share.DeleteAsync();
        }

        [RecordedTest]
        [ServiceVersion(Min = ShareClientOptions.ServiceVersion.V2019_12_12)]
        public async Task UndeleteShareAsync_Error()
        {
            // Arrange
            ShareServiceClient service = SharesClientBuilder.GetServiceClient_SoftDelete();
            ShareClient share = InstrumentClient(service.GetShareClient(GetNewShareName()));
            string fakeVersion = "01D60F8BB59A4652";

            // Act
            await TestHelper.AssertExpectedExceptionAsync<RequestFailedException>(
                service.UndeleteShareAsync(GetNewShareName(), fakeVersion),
                e => Assert.AreEqual(ShareErrorCode.ShareNotFound.ToString(), e.ErrorCode));
        }

        #region GenerateSasTests
        [RecordedTest]
        public void CanGenerateSas_ClientConstructors()
        {
            // Arrange
            var constants = TestConstants.Create(this);
            var blobEndpoint = new Uri("https://127.0.0.1/" + constants.Sas.Account);
            var blobSecondaryEndpoint = new Uri("https://127.0.0.1/" + constants.Sas.Account + "-secondary");
            var storageConnectionString = new StorageConnectionString(constants.Sas.SharedKeyCredential, fileStorageUri: (blobEndpoint, blobSecondaryEndpoint));
            string connectionString = storageConnectionString.ToString(true);

            // Act - ShareServiceClient(string connectionString)
            ShareServiceClient share = InstrumentClient(new ShareServiceClient(
                connectionString));
            Assert.IsTrue(share.CanGenerateAccountSasUri);

            // Act - ShareServiceClient(string connectionString, string blobContainerName, BlobClientOptions options)
            ShareServiceClient share2 = InstrumentClient(new ShareServiceClient(
                connectionString,
                GetOptions()));
            Assert.IsTrue(share2.CanGenerateAccountSasUri);

            // Act - ShareServiceClient(Uri blobContainerUri, BlobClientOptions options = default)
            ShareServiceClient share3 = InstrumentClient(new ShareServiceClient(
                blobEndpoint,
                GetOptions()));
            Assert.IsFalse(share3.CanGenerateAccountSasUri);

            // Act - ShareServiceClient(Uri blobContainerUri, StorageSharedKeyCredential credential, BlobClientOptions options = default)
            ShareServiceClient share4 = InstrumentClient(new ShareServiceClient(
                blobEndpoint,
                constants.Sas.SharedKeyCredential,
                GetOptions()));
            Assert.IsTrue(share4.CanGenerateAccountSasUri);
        }

        [RecordedTest]
        public void CanGenerateSas_GetShareClient()
        {
            // Arrange
            var constants = TestConstants.Create(this);
            var blobEndpoint = new Uri("https://127.0.0.1/" + constants.Sas.Account);
            var blobSecondaryEndpoint = new Uri("https://127.0.0.1/" + constants.Sas.Account + "-secondary");
            var storageConnectionString = new StorageConnectionString(constants.Sas.SharedKeyCredential, fileStorageUri: (blobEndpoint, blobSecondaryEndpoint));
            string connectionString = storageConnectionString.ToString(true);

            // Act - ShareServiceClient(string connectionString)
            ShareServiceClient service = InstrumentClient(new ShareServiceClient(
                connectionString));
            ShareClient share = service.GetShareClient(GetNewShareName());
            Assert.IsTrue(share.CanGenerateSasUri);

            // Act - ShareServiceClient(string connectionString, string blobContainerName, BlobClientOptions options)
            ShareServiceClient service2 = InstrumentClient(new ShareServiceClient(
                connectionString,
                GetOptions()));
            ShareClient share2 = service2.GetShareClient(GetNewShareName());
            Assert.IsTrue(share2.CanGenerateSasUri);

            // Act - ShareServiceClient(Uri blobContainerUri, BlobClientOptions options = default)
            ShareServiceClient service3 = InstrumentClient(new ShareServiceClient(
                blobEndpoint,
                GetOptions()));
            ShareClient share3 = service3.GetShareClient(GetNewShareName());
            Assert.IsFalse(share3.CanGenerateSasUri);

            // Act - ShareServiceClient(Uri blobContainerUri, StorageSharedKeyCredential credential, BlobClientOptions options = default)
            ShareServiceClient service4 = InstrumentClient(new ShareServiceClient(
                blobEndpoint,
                constants.Sas.SharedKeyCredential,
                GetOptions()));
            ShareClient share4 = service4.GetShareClient(GetNewShareName());
            Assert.IsTrue(share4.CanGenerateSasUri);
        }

        [RecordedTest]
        public void CanGenerateAccountSas_Mockable()
        {
            // Act
            var directory = new Mock<ShareServiceClient>();
            directory.Setup(x => x.CanGenerateAccountSasUri).Returns(false);

            // Assert
            Assert.IsFalse(directory.Object.CanGenerateAccountSasUri);

            // Act
            directory.Setup(x => x.CanGenerateAccountSasUri).Returns(true);

            // Assert
            Assert.IsTrue(directory.Object.CanGenerateAccountSasUri);
        }

        [RecordedTest]
        public void GenerateAccountSas_RequiredParameters()
        {
            // Arrange
            TestConstants constants = TestConstants.Create(this);
            Uri serviceUri = new Uri($"https://{constants.Sas.Account}.file.core.windows.net");
            DateTimeOffset expiresOn = Recording.UtcNow.AddHours(+1);
            AccountSasPermissions permissions = AccountSasPermissions.Read | AccountSasPermissions.Write;
            AccountSasResourceTypes resourceTypes = AccountSasResourceTypes.All;
            ShareServiceClient serviceClient = InstrumentClient(
                new ShareServiceClient(
                    serviceUri,
                    constants.Sas.SharedKeyCredential,
                    GetOptions()));

            // Act
            Uri sasUri = serviceClient.GenerateAccountSasUri(
                permissions: permissions,
                expiresOn: expiresOn,
                resourceTypes: resourceTypes);

            // Assert
            AccountSasBuilder sasBuilder = new AccountSasBuilder(permissions, expiresOn, AccountSasServices.Files, resourceTypes);
            ShareUriBuilder expectedUri = new ShareUriBuilder(serviceUri)
            {
                Sas = sasBuilder.ToSasQueryParameters(constants.Sas.SharedKeyCredential)
            };
            Assert.AreEqual(expectedUri.ToUri(), sasUri);
        }

        [RecordedTest]
        public void GenerateAccountSas_Builder()
        {
            TestConstants constants = TestConstants.Create(this);
            Uri serviceUri = new Uri($"https://{constants.Sas.Account}.file.core.windows.net");
            AccountSasPermissions permissions = AccountSasPermissions.Read | AccountSasPermissions.Write;
            DateTimeOffset expiresOn = Recording.UtcNow.AddHours(+1);
            AccountSasServices services = AccountSasServices.Files;
            AccountSasResourceTypes resourceTypes = AccountSasResourceTypes.All;
            ShareServiceClient serviceClient = InstrumentClient(
                new ShareServiceClient(
                    serviceUri,
                    constants.Sas.SharedKeyCredential,
                    GetOptions()));

            AccountSasBuilder sasBuilder = new AccountSasBuilder(permissions, expiresOn, services, resourceTypes);

            // Act
            Uri sasUri = serviceClient.GenerateAccountSasUri(sasBuilder);

            // Assert
            AccountSasBuilder sasBuilder2 = new AccountSasBuilder(permissions, expiresOn, services, resourceTypes);
            ShareUriBuilder expectedUri = new ShareUriBuilder(serviceUri)
            {
                Sas = sasBuilder2.ToSasQueryParameters(constants.Sas.SharedKeyCredential)
            };
            Assert.AreEqual(expectedUri.ToUri(), sasUri);
        }

        [RecordedTest]
        public void GenerateAccountSas_WrongService_Service()
        {
            TestConstants constants = TestConstants.Create(this);
            Uri serviceUri = new Uri($"https://{constants.Sas.Account}.file.core.windows.net");
            AccountSasPermissions permissions = AccountSasPermissions.Read | AccountSasPermissions.Write;
            DateTimeOffset expiresOn = Recording.UtcNow.AddHours(+1);
            AccountSasServices services = AccountSasServices.Blobs; // Wrong Service
            AccountSasResourceTypes resourceTypes = AccountSasResourceTypes.All;
            ShareServiceClient serviceClient = InstrumentClient(
                new ShareServiceClient(
                    serviceUri,
                    constants.Sas.SharedKeyCredential,
                    GetOptions()));

            AccountSasBuilder sasBuilder = new AccountSasBuilder(permissions, expiresOn, services, resourceTypes)
            {
                IPRange = new SasIPRange(System.Net.IPAddress.None, System.Net.IPAddress.None),
                StartsOn = Recording.UtcNow.AddHours(-1)
            };

            // Act
            TestHelper.AssertExpectedException(
                () => serviceClient.GenerateAccountSasUri(sasBuilder),
                new InvalidOperationException("SAS Uri cannot be generated. builder.Services does specify Files. builder.Services must either specify Files or specify all Services are accessible in the value."));
        }
        #endregion

        [RecordedTest]
        public void CanMockClientConstructors()
        {
            // One has to call .Object to trigger constructor. It's lazy.
            var mock = new Mock<ShareServiceClient>(TestConfigDefault.ConnectionString, new ShareClientOptions()).Object;
            mock = new Mock<ShareServiceClient>(TestConfigDefault.ConnectionString).Object;
            mock = new Mock<ShareServiceClient>(new Uri("https://test/test/test"), new ShareClientOptions()).Object;
            mock = new Mock<ShareServiceClient>(new Uri("https://test/test/test"), Tenants.GetNewSharedKeyCredentials(), new ShareClientOptions()).Object;
            mock = new Mock<ShareServiceClient>(new Uri("https://test/test/test"), new AzureSasCredential("foo"), new ShareClientOptions()).Object;
        }
    }
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<DocSetNodes version="1.0">
    <TOC>
        <Node type="folder">
            <Name>PaperTrailLumberjackiOSExample</Name>
            <Path>index.html</Path>
			<Subnodes>

				<Node type="folder">
					<Name>Classes</Name>
					<Path>index.html</Path>
					<Subnodes>
						<NodeRef refid="1"/>
						<NodeRef refid="2"/>

					</Subnodes>
				</Node>

			</Subnodes>
        </Node>
    </TOC>
	<Library>

		<Node id="1" type="file" documentType="reference">
            <Name>RMPaperTrailLogger</Name>
            <Path>Classes/RMPaperTrailLogger.html</Path>

                <Subnodes>
                <Node type="section">
                    <Path>Classes/RMPaperTrailLogger.html</Path>
                    <Name>Overview</Name>
                    <Anchor>overview</Anchor>
                </Node>
                <Node type="section">
                    <Path>Classes/RMPaperTrailLogger.html</Path>
                    <Name>Tasks</Name>
                    <Anchor>tasks</Anchor>
                </Node>

                <Node type="section">
                    <Path>Classes/RMPaperTrailLogger.html</Path>
                    <Name>Properties</Name>
                    <Anchor>properties</Anchor>
                </Node>

                <Node type="section">
                    <Path>Classes/RMPaperTrailLogger.html</Path>
                    <Name>Class Methods</Name>
                    <Anchor>class_methods</Anchor>
                </Node>

                <Node type="section">
                    <Path>Classes/RMPaperTrailLogger.html</Path>
                    <Name>Instance Methods</Name>
                    <Anchor>instance_methods</Anchor>
                </Node>

            </Subnodes>

        </Node>
		<Node id="2" type="file" documentType="reference">
            <Name>RMSyslogFormatter</Name>
            <Path>Classes/RMSyslogFormatter.html</Path>

        </Node>

	</Library>
</DocSetNodes>


text:
The primary thing you need to do to get started with tastypie, is define a resource which will expose the URIs for your API. The Base resource is set up with 2 primary actions ( `list` and `detail` ) and allows the base CRUD http verbs - `GET`, `PUT`, `POST`, `DELETE`, and optionally `PATCH`.

## Define A Resource

A functional resource, by convention, should define method handlers for each of the actions & verbs where it makes sense - where the resource method name is `<VERB>_<ACTION>`.

```
var tastypie = require('tastypie')
var Resource = tastypie.Resource;
var http     = tasypie.http
var Simple;

Simple = Resource.extend({
    fields:{
        key:{type:'char'}
    }
    ,constructor:function( options ){
        this.parent('constructor', options)
    }
    ,get_list: function( bundle ){
        // the data property is what gets returnedt
        bundle.data = { key:'value' };

        // use the respond method if you
        // want serialization, status code, etc...
        return this.respond( bundle )
    }
    , get_detail: function( bundle ){
        bundle.data = {success:1}
        return this.respond( bundle )
    }
    , put_detail: function( bundle ){
        budnel.data = {key:'updated'}
        return this.respond( bundle, http.accepted )
    }
    , post_list: function( bundle ){
        var data = bundle.req.payload;
        // do something with the data.
        return this.respond( bundle, http.created)
    }
    , delete_detail: function( bundle ){
        bundle.data = null;
        return this.respond( bundle, http.noContent )
    }
    , patch_detail: function( bundle ){
        // or just send a straight response.
        return bundle.res({any:'data you want'}).code( 201 )
    }
});

module.exports = Simple;
```

That is pretty much all there is to it. You can use the {@link module:tastypie/lib/resource#respond|respond} method on the resource if you would like to have some level of consistency. The respond method will take care of Content negotiation, serialization, and set some headers for you. If you want to *skip* all of that, you can just use the `res` method on the `bundle`, which is a [hapi reply function](http://hapijs.com/api#replyerr-result).

## Registring Resource With Hapi

Once you have a resource, you need to add it to an {@link module:tastypie/lib/api|API Namespace} and register with a [Hapi Server](http://hapijs.com/api#server) instance

```
var Hapi = require('hapi');
var Api  = require('tastypie').Api;
var Simple = require('./simple');
var server, v1

server = new Hapi.Server();
server.connection({
    host:'0.0.0.0',
    port:3000,
    labels:['api']
});

v1 = new Api('api/v1');
v1.use('simple', new Simple())

server.register(v1,function(){
    server.start(function(){
        console.log('server running at http://localhost:3000/')
    });
})
```

Now we have three urls registered **/api/v1/simple**, **/api/v1/simple/{id}**, and **/api/v1/simple/schema**, which support has support for the main CRUD Verbs.

## API Level Serializer

A common _gotcha_ when attempting to respond directly to requests ( by using `bundle.res()` ) is that you lose content negotiation and proper serialization. As a remedy, you can specify a serializer {@link module:tastypie/lib/api|on an api instance} and it will attempt to serialize object appropriately.

```
var Api = require('tastypie').Api;
var Serializer = require('tastypie').Serializer;
var server = new( require('hapi') ).Server();
var v1 = new Api('api/v1', {
    serializer: new Serializer()
});

v1.use('simple', new Simple());
server.register( v1, console.log );
```

When a serializer is present, the `Api` instance will inject itself into the request life-cycle and do the heavy lifting. Now you can use reply with plain old JSON objects ( `return bundle.res({})` ). Really nice for prototyping or quick iterations with front-end teams.

## Final Source

Here is the example code in its entirety

```
var tastypie = require('tastypie')
var Resource = tastypie.Resource;
var http     = tastypie.http
var Hapi     = require('hapi');
var Api      = require('tastypie').Api;
var server, v1, Simple;

// resource
Simple = Resource.extend({

     get_list: function( bundle ){
        // the data property is what gets returned
        bundle.data = {'key': 'value'};

        // use the respond method if you
        // want serialization, status code, etc...
        return this.respond( bundle )
    }
    , get_detail: function( bundle ){
        bundle.data = {success:1}
        return this.respond( bundle )
    }
    , put_detail: function( bundle ){
        budnel.data = {key:'updated'}
        return this.respond( bundle, http.accepted )
    }
    , post_list: function( bundle ){
        var data = bundle.req.payload;
        // do something with the data.
        return this.respond( bundle, http.created)
    }
    , delete_detail: function( bundle ){
        bundle.data = null;
        return this.respond( bundle, http.noContent )
    }
    , patch_detail: function( bundle ){
        // or just send a straight response.
        return bundle.res({any:'data you want'}).code( 201 )
    }
});

// server setup
server = new Hapi.Server();
server.connection({
    host:'0.0.0.0',
    port:3000,
    labels:['api']
});

// namespace setup
v1 = new Api('api/v1');
v1.use('simple', new Simple())

server.register(v1,function(){
    server.start(function(){
        console.log('server running at http://localhost:3000/')
    });
})
```