# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests feeding functions using arrays and `DataFrames`."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections

import numpy as np

from tensorflow.python.estimator.inputs.queues import feeding_functions as ff
from tensorflow.python.platform import test

try:
  # pylint: disable=g-import-not-at-top
  import pandas as pd
  HAS_PANDAS = True
except IOError:
  # Pandas writes a temporary file during import. If it fails, don't use pandas.
  HAS_PANDAS = False
except ImportError:
  HAS_PANDAS = False


def vals_to_list(a):
  return {
      key: val.tolist() if isinstance(val, np.ndarray) else val
      for key, val in a.items()
  }


class _FeedingFunctionsTestCase(test.TestCase):
  """Tests for feeding functions."""

  def testArrayFeedFnBatchOne(self):
    array = np.arange(32).reshape([16, 2])
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, 1)

    # cycle around a couple times
    for x in range(0, 100):
      i = x % 16
      expected = {
          "index_placeholder": [i],
          "value_placeholder": [[2 * i, 2 * i + 1]]
      }
      actual = aff()
      self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchFive(self):
    array = np.arange(32).reshape([16, 2])
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, 5)

    # cycle around a couple times
    for _ in range(0, 101, 2):
      aff()

    expected = {
        "index_placeholder": [15, 0, 1, 2, 3],
        "value_placeholder": [[30, 31], [0, 1], [2, 3], [4, 5], [6, 7]]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchTwoWithOneEpoch(self):
    array = np.arange(5) + 10
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, batch_size=2, num_epochs=1)

    expected = {
        "index_placeholder": [0, 1],
        "value_placeholder": [10, 11]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [2, 3],
        "value_placeholder": [12, 13]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [4],
        "value_placeholder": [14]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchOneHundred(self):
    array = np.arange(32).reshape([16, 2])
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, 100)

    expected = {
        "index_placeholder":
            list(range(0, 16)) * 6 + list(range(0, 4)),
        "value_placeholder":
            np.arange(32).reshape([16, 2]).tolist() * 6 +
            [[0, 1], [2, 3], [4, 5], [6, 7]]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testArrayFeedFnBatchOneHundredWithSmallerArrayAndMultipleEpochs(self):
    array = np.arange(2) + 10
    placeholders = ["index_placeholder", "value_placeholder"]
    aff = ff._ArrayFeedFn(placeholders, array, batch_size=100, num_epochs=2)

    expected = {
        "index_placeholder": [0, 1, 0, 1],
        "value_placeholder": [10, 11, 10, 11],
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchOne(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 64)
    array2 = np.arange(64, 96)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 128))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, 1)

    # cycle around a couple times
    for x in range(0, 100):
      i = x % 32
      expected = {
          "index_placeholder": [i + 96],
          "a_placeholder": [32 + i],
          "b_placeholder": [64 + i]
      }
      actual = aff()
      self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchFive(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 64)
    array2 = np.arange(64, 96)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 128))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, 5)

    # cycle around a couple times
    for _ in range(0, 101, 2):
      aff()

    expected = {
        "index_placeholder": [127, 96, 97, 98, 99],
        "a_placeholder": [63, 32, 33, 34, 35],
        "b_placeholder": [95, 64, 65, 66, 67]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchTwoWithOneEpoch(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 37)
    array2 = np.arange(64, 69)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 101))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, batch_size=2, num_epochs=1)

    expected = {
        "index_placeholder": [96, 97],
        "a_placeholder": [32, 33],
        "b_placeholder": [64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [98, 99],
        "a_placeholder": [34, 35],
        "b_placeholder": [66, 67]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [100],
        "a_placeholder": [36],
        "b_placeholder": [68]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchOneHundred(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 64)
    array2 = np.arange(64, 96)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 128))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, 100)

    expected = {
        "index_placeholder": list(range(96, 128)) * 3 + list(range(96, 100)),
        "a_placeholder": list(range(32, 64)) * 3 + list(range(32, 36)),
        "b_placeholder": list(range(64, 96)) * 3 + list(range(64, 68))
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testPandasFeedFnBatchOneHundredWithSmallDataArrayAndMultipleEpochs(self):
    if not HAS_PANDAS:
      return
    array1 = np.arange(32, 34)
    array2 = np.arange(64, 66)
    df = pd.DataFrame({"a": array1, "b": array2}, index=np.arange(96, 98))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._PandasFeedFn(placeholders, df, batch_size=100, num_epochs=2)

    expected = {
        "index_placeholder": [96, 97, 96, 97],
        "a_placeholder": [32, 33, 32, 33],
        "b_placeholder": [64, 65, 64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testOrderedDictNumpyFeedFnBatchTwoWithOneEpoch(self):
    a = np.arange(32, 37)
    b = np.arange(64, 69)
    x = {"a": a, "b": b}
    ordered_dict_x = collections.OrderedDict(
        sorted(x.items(), key=lambda t: t[0]))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._OrderedDictNumpyFeedFn(
        placeholders, ordered_dict_x, batch_size=2, num_epochs=1)

    expected = {
        "index_placeholder": [0, 1],
        "a_placeholder": [32, 33],
        "b_placeholder": [64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [2, 3],
        "a_placeholder": [34, 35],
        "b_placeholder": [66, 67]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

    expected = {
        "index_placeholder": [4],
        "a_placeholder": [36],
        "b_placeholder": [68]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testOrderedDictNumpyFeedFnLargeBatchWithSmallArrayAndMultipleEpochs(self):
    a = np.arange(32, 34)
    b = np.arange(64, 66)
    x = {"a": a, "b": b}
    ordered_dict_x = collections.OrderedDict(
        sorted(x.items(), key=lambda t: t[0]))
    placeholders = ["index_placeholder", "a_placeholder", "b_placeholder"]
    aff = ff._OrderedDictNumpyFeedFn(
        placeholders, ordered_dict_x, batch_size=100, num_epochs=2)

    expected = {
        "index_placeholder": [0, 1, 0, 1],
        "a_placeholder": [32, 33, 32, 33],
        "b_placeholder": [64, 65, 64, 65]
    }
    actual = aff()
    self.assertEqual(expected, vals_to_list(actual))

  def testFillArraySmall(self):
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[64, 36], dtype=np.int32)
    ff._fill_array(actual, a)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testFillArrayLarge(self):
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    ff._fill_array(actual, a)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testFillArraySmallWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[64, 36], dtype=np.int32)
    ff._fill_array(actual, a, fill_value)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testFillArrayLargeWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    actual = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    ff._fill_array(actual, a, fill_value)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededSmall(self):
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededLarge(self):
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = 0
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededSmallWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[32, 32], dtype=np.int32).tolist() +
         np.ones(shape=[32, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[64, 36], dtype=np.int32)
    expected[:32, 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededLargeWithSpecifiedValue(self):
    fill_value = 8
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.int32).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.int32).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.int32)
    expected[:8, ..., 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededSmallWithSpecifiedNonNumericValue(self):
    fill_value = False
    a = (np.ones(shape=[32, 32], dtype=np.bool).tolist() +
         np.ones(shape=[32, 36], dtype=np.bool).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[64, 36], dtype=np.bool)
    expected[:32, 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())

  def testPadIfNeededLargeWithSpecifiedNonNumericValue(self):
    fill_value = False
    a = (np.ones(shape=[8, 8, 8, 8, 32], dtype=np.bool).tolist() +
         np.ones(shape=[8, 8, 8, 8, 36], dtype=np.bool).tolist())
    a = list(map(np.array, a))
    actual = ff._pad_if_needed(a, fill_value)
    expected = np.ones(shape=[16, 8, 8, 8, 36], dtype=np.bool)
    expected[:8, ..., 32:] = fill_value
    self.assertEqual(expected.tolist(), actual.tolist())


if __name__ == "__main__":
  test.main()
