text:
"""Multi-GPU tests for MirroredStrategy."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys

from tensorflow.contrib.distribute.python import mirrored_strategy
from tensorflow.contrib.distribute.python import strategy_test_lib
from tensorflow.contrib.distribute.python import values
from tensorflow.core.protobuf import config_pb2
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.eager import backprop
from tensorflow.python.eager import context
from tensorflow.python.eager import function
from tensorflow.python.eager import test
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.layers import core
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import rnn
from tensorflow.python.ops import rnn_cell_impl
from tensorflow.python.ops import variable_scope
from tensorflow.python.ops import variables
from tensorflow.python.training import device_util
from tensorflow.python.training import distribute as distribute_lib

GPU_TEST = "test_gpu" in sys.argv[0]

class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):

  def _get_distribution_strategy(self):
    devices = ["/device:CPU:0", "/device:GPU:0"]
    if GPU_TEST:
      self.assertGreater(context.num_gpus(), 0)
      if context.num_gpus() > 1:
        devices = ["/device:GPU:0", "/device:GPU:1"]
    print(self.id().split(".")[-1], "devices:", ", ".join(devices))
    return mirrored_strategy.MirroredStrategy(devices)

  def testMinimizeLossEager(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")
    self._test_minimize_loss_eager(self._get_distribution_strategy())

  def testMinimizeLossGraph(self):
    soft_placement = not GPU_TEST
    print("testMinimizeLossGraph soft_placement:", soft_placement)
    self._test_minimize_loss_graph(
        self._get_distribution_strategy(), soft_placement=soft_placement)

  def testMapReduce(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")
    self._test_map_reduce(self._get_distribution_strategy())

  def testDeviceIndex(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")
    self._test_device_index(self._get_distribution_strategy())

  def testTowerId(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")
    self._test_tower_id(self._get_distribution_strategy())

  def testNumTowers(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")
    self.assertEqual(2, self._get_distribution_strategy().num_towers)

  @test_util.run_in_graph_and_eager_modes
  def testCallAndMergeExceptions(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")
    self._test_call_and_merge_exceptions(self._get_distribution_strategy())

  @test_util.run_in_graph_and_eager_modes
  def testRunRegroupError(self):

    def run_fn(device_id):
      # Generates a list with different lengths on different devices.
      # Will fail in _regroup() (if more than one device).
      return list(range(device_id))

    dist = self._get_distribution_strategy()
    with dist.scope(), self.assertRaises(AssertionError):
      dist.call_for_each_tower(run_fn, dist.worker_device_index)

  @test_util.run_in_graph_and_eager_modes
  def testReduceToCpu(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")

    def run_fn(device_id):
      return device_id

    dist = self._get_distribution_strategy()
    with dist.scope():
      result = dist.call_for_each_tower(run_fn, dist.worker_device_index)
      reduced = dist.reduce(
          variable_scope.VariableAggregation.SUM,
          result,
          destinations="/device:CPU:0")
      unwrapped = dist.unwrap(reduced)
      self.assertEqual(1, len(unwrapped))
      expected = sum(range(len(dist.worker_devices)))
      self.assertEqual(expected, self.evaluate(unwrapped[0]))

  @test_util.run_in_graph_and_eager_modes()
  def testReduceToMultipleDestinations(self):
    if not GPU_TEST:
      self.skipTest("Not GPU test")

    devices = ["/device:GPU:0"]
    if GPU_TEST:
      self.assertGreater(context.num_gpus(), 0)
    print(self.id().split(".")[-1], "devices:", ", ".join(devices))

    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      reduced = dist.reduce(
          variable_scope.VariableAggregation.SUM,
          1.0,
          destinations=["/device:CPU:0", "/device:GPU:0"])
      unwrapped = dist.unwrap(reduced)
      self.assertEqual(2, len(unwrapped))
      self.assertEqual(1.0, self.evaluate(unwrapped[0]))

class MirroredStrategyVariableCreationTest(test.TestCase):

  config = config_pb2.ConfigProto()
  config.allow_soft_placement = True

  def _skip_eager_if_gpus_less_than(self, num_gpus):
    if context.num_gpus() < num_gpus and context.executing_eagerly():
      self.skipTest("Enough GPUs not available for this test in eager mode.")

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testSingleVariable(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn():
      # This variable should be created only once across the threads because of
      # special variable_creator functions used by `dist.call_for_each_tower`.
      v = variable_scope.variable(1.0, name="foo")
      distribute_lib.get_tower_context().merge_call(lambda _: _)
      return v

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      self.assertIsInstance(result, values.MirroredVariable)
      self.assertEquals("foo:0", result.name)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testUnnamedVariable(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn():
      v = variable_scope.variable(1.0)
      distribute_lib.get_tower_context().merge_call(lambda _: _)
      return v

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      self.assertIsInstance(result, values.MirroredVariable)
      # Default name of "Variable" will be used.
      self.assertEquals("Variable:0", result.name)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testMultipleVariables(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn():
      vs = []
      for i in range(5):
        vs.append(variable_scope.variable(1.0, name="foo" + str(i)))
      distribute_lib.get_tower_context().merge_call(lambda _: _)
      return vs

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      for i, v in enumerate(result):
        self.assertIsInstance(v, values.MirroredVariable)
        self.assertEquals("foo" + str(i) + ":0", v.name)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testMultipleVariablesWithSameCanonicalName(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn():
      vs = []
      vs.append(variable_scope.variable(1.0, name="foo/bar"))
      vs.append(variable_scope.variable(1.0, name="foo_1/bar"))
      vs.append(variable_scope.variable(1.0, name="foo_1/bar_1"))
      vs.append(variable_scope.variable(1.0, name="foo/bar_1"))
      distribute_lib.get_tower_context().merge_call(lambda _: _)
      return vs

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      for v in result:
        self.assertIsInstance(v, values.MirroredVariable)
      self.assertEquals(4, len(result))
      self.assertEquals("foo/bar:0", result[0].name)
      self.assertEquals("foo_1/bar:0", result[1].name)
      self.assertEquals("foo_1/bar_1:0", result[2].name)
      self.assertEquals("foo/bar_1:0", result[3].name)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testVariableWithSameCanonicalNameAcrossThreads(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn(device_id):
      v = variable_scope.variable(1.0, name="foo_" + str(device_id))
      distribute_lib.get_tower_context().merge_call(lambda _: _)
      return v

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      result = dist.call_for_each_tower(
          model_fn, dist.worker_device_index, run_concurrently=False)
      self.assertIsInstance(result, values.MirroredVariable)
      # The resulting mirrored variable will use the name from the first device.
      self.assertEquals("foo_0:0", result.name)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testWithLayers(self):
    self._skip_eager_if_gpus_less_than(1)
    def model_fn(features):
      with variable_scope.variable_scope("common"):
        layer1 = core.Dense(1)
        layer1(features)
        layer2 = core.Dense(1)
        layer2(features)
        # This will pause the current thread, and execute the other thread.
        distribute_lib.get_tower_context().merge_call(lambda _: _)
        layer3 = core.Dense(1)
        layer3(features)
        return [(layer1.kernel, layer1.bias),
                (layer2.kernel, layer2.bias),
                (layer3.kernel, layer3.bias)]

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])
    features = dist.distribute_dataset(
        lambda: dataset_ops.Dataset.from_tensors([[1.]]).repeat(10)
    ).make_one_shot_iterator().get_next()

    with dist.scope():
      result = dist.call_for_each_tower(
          model_fn, features, run_concurrently=False)
      suffixes = ["", "_1", "_2"]
      for (kernel, bias), suffix in zip(result, suffixes):
        self.assertIsInstance(kernel, values.MirroredVariable)
        self.assertEquals("common/dense" + suffix + "/kernel:0", kernel.name)
        self.assertIsInstance(bias, values.MirroredVariable)
        self.assertEquals("common/dense" + suffix + "/bias:0", bias.name)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testWithVariableAndVariableScope(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn():
      v0 = variable_scope.variable(1.0, name="var0", aggregation=None)
      with variable_scope.variable_scope("common"):
        v1 = variable_scope.variable(1.0, name="var1")
        # This will pause the current thread, and execute the other thread.
        distribute_lib.get_tower_context().merge_call(lambda _: _)
        v2 = variable_scope.variable(
            1.0,
            name="var2",
            synchronization=variable_scope.VariableSynchronization.ON_READ,
            aggregation=variable_scope.VariableAggregation.SUM)
        v3 = variable_scope.variable(
            1.0,
            name="var3",
            synchronization=variable_scope.VariableSynchronization.ON_WRITE,
            aggregation=variable_scope.VariableAggregation.MEAN)

      return v0, v1, v2, v3

    devices = ["/device:CPU:0", "/device:GPU:0"]
    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      v = variable_scope.variable(1.0, name="var-main0")
      self.assertEquals("var-main0:0", v.name)

      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      self.assertEquals(4, len(result))
      v0, v1, v2, v3 = result
      self.assertIsInstance(v0, values.MirroredVariable)
      self.assertEquals("var0:0", v0.name)
      self.assertIsInstance(v1, values.MirroredVariable)
      self.assertEquals("common/var1:0", v1.name)
      self.assertIsInstance(v2, values.TowerLocalVariable)
      self.assertEquals("common/var2:0", v2.name)
      self.assertEquals(variable_scope.VariableAggregation.SUM, v2.aggregation)
      self.assertIsInstance(v3, values.MirroredVariable)
      self.assertEquals("common/var3:0", v3.name)
      self.assertEquals(variable_scope.VariableAggregation.MEAN, v3.aggregation)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testWithGetVariableAndVariableScope(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn():
      v0 = variable_scope.get_variable("var0", [1])
      with variable_scope.variable_scope("common"):
        v1 = variable_scope.get_variable("var1", [1])
        # This will pause the current thread, and execute the other thread.
        distribute_lib.get_tower_context().merge_call(lambda _: _)
        v2 = variable_scope.get_variable(
            "var2", [1],
            synchronization=variable_scope.VariableSynchronization.ON_READ,
            aggregation=variable_scope.VariableAggregation.SUM)
        v3 = variable_scope.get_variable(
            "var3", [1],
            synchronization=variable_scope.VariableSynchronization.ON_WRITE,
            aggregation=variable_scope.VariableAggregation.MEAN)

      return v0, v1, v2, v3

    devices = ["/device:CPU:0", "/device:GPU:0"]
    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      with variable_scope.variable_scope("main"):
        v = variable_scope.get_variable("var-main0", [1])
        self.assertEquals("main/var-main0:0", v.name)

        result = dist.call_for_each_tower(model_fn, run_concurrently=False)
        self.assertEquals(4, len(result))
        v0, v1, v2, v3 = result
        self.assertIsInstance(v0, values.MirroredVariable)
        self.assertEquals("main/var0:0", v0.name)
        self.assertIsInstance(v1, values.MirroredVariable)
        self.assertEquals("main/common/var1:0", v1.name)
        self.assertIsInstance(v2, values.TowerLocalVariable)
        self.assertEquals("main/common/var2:0", v2.name)
        self.assertEquals(variable_scope.VariableAggregation.SUM,
                          v2.aggregation)
        self.assertIsInstance(v3, values.MirroredVariable)
        self.assertEquals("main/common/var3:0", v3.name)
        self.assertEquals(variable_scope.VariableAggregation.MEAN,
                          v3.aggregation)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testNoneSynchronizationWithGetVariable(self):
    self._skip_eager_if_gpus_less_than(1)
    devices = ["/device:CPU:0", "/device:GPU:0"]
    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      with self.assertRaisesRegexp(
          ValueError, "`NONE` variable synchronization mode is not "
          "supported with `Mirrored` distribution strategy. Please change "
          "the `synchronization` for variable: v"):
        variable_scope.get_variable(
            "v", [1],
            synchronization=variable_scope.VariableSynchronization.NONE)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testNoneSynchronizationWithVariable(self):
    self._skip_eager_if_gpus_less_than(1)
    devices = ["/device:CPU:0", "/device:GPU:0"]
    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      with self.assertRaisesRegexp(
          ValueError, "`NONE` variable synchronization mode is not "
          "supported with `Mirrored` distribution strategy. Please change "
          "the `synchronization` for variable: v"):
        variable_scope.variable(
            1.0,
            name="v",
            synchronization=variable_scope.VariableSynchronization.NONE)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testInvalidSynchronizationWithVariable(self):
    self._skip_eager_if_gpus_less_than(1)
    devices = ["/device:CPU:0", "/device:GPU:0"]
    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      with self.assertRaisesRegexp(
          ValueError, "Invalid variable synchronization mode: Invalid for "
          "variable: v"):
        variable_scope.variable(1.0, name="v", synchronization="Invalid")

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testInvalidAggregationWithGetVariable(self):
    self._skip_eager_if_gpus_less_than(1)
    devices = ["/device:CPU:0", "/device:GPU:0"]
    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      with self.assertRaisesRegexp(
          ValueError, "Invalid variable aggregation mode: invalid for "
          "variable: v"):
        variable_scope.get_variable(
            "v", [1],
            synchronization=variable_scope.VariableSynchronization.ON_WRITE,
            aggregation="invalid")

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testInvalidAggregationWithVariable(self):
    self._skip_eager_if_gpus_less_than(1)
    devices = ["/device:CPU:0", "/device:GPU:0"]
    dist = mirrored_strategy.MirroredStrategy(devices)
    with dist.scope():
      with self.assertRaisesRegexp(
          ValueError, "Invalid variable aggregation mode: invalid for "
          "variable: v"):
        variable_scope.variable(
            1.0,
            name="v",
            synchronization=variable_scope.VariableSynchronization.ON_WRITE,
            aggregation="invalid")

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testThreeDevices(self):
    self._skip_eager_if_gpus_less_than(2)

    def model_fn():
      v = variable_scope.variable(1.0, name="foo")
      distribute_lib.get_tower_context().merge_call(lambda _: _)
      return v

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"])

    with dist.scope():
      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      self.assertIsInstance(result, values.MirroredVariable)
      self.assertEquals("foo:0", result.name)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testNonMatchingVariableCreation(self):
    self._skip_eager_if_gpus_less_than(1)

    def model_fn(name):
      v = variable_scope.variable(1.0, name=name)
      distribute_lib.get_tower_context().merge_call(lambda _: _)
      return v

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      names = values.DistributedValues({
          "/device:CPU:0": "foo",
          "/device:GPU:0": "bar"
      })
      with self.assertRaises(RuntimeError):
        _ = dist.call_for_each_tower(model_fn, names, run_concurrently=False)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testTowerLocalVariable(self):
    self._skip_eager_if_gpus_less_than(1)

    all_v_sum = {}
    all_v_mean = {}
    components_sum = {}
    components_mean = {}

    def model_fn(device_id):
      v_sum = variable_scope.variable(
          1.0,
          synchronization=variable_scope.VariableSynchronization.ON_READ,
          aggregation=variable_scope.VariableAggregation.SUM)
      v_mean = variable_scope.variable(
          4.0,
          synchronization=variable_scope.VariableSynchronization.ON_READ,
          aggregation=variable_scope.VariableAggregation.MEAN)
      self.assertTrue(isinstance(v_sum, values.TowerLocalVariable))
      self.assertTrue(isinstance(v_mean, values.TowerLocalVariable))
      updates = [v_sum.assign_add(2.0 + device_id),
                 v_mean.assign(6.0 * device_id)]
      all_v_sum[device_id] = v_sum
      all_v_mean[device_id] = v_mean
      c_sum = v_sum.get()
      c_mean = v_mean.get()
      components_sum[device_id] = c_sum
      components_mean[device_id] = c_mean
      self.assertIsNot(v_sum, c_sum)
      self.assertIsNot(v_mean, c_mean)
      return updates, v_sum, v_mean, c_sum, c_mean

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      # Create "sum" and "mean" versions of TowerLocalVariables.
      ret_ops, ret_v_sum, ret_v_mean, regrouped_sum, regrouped_mean = (
          dist.call_for_each_tower(
              model_fn, dist.worker_device_index, run_concurrently=False))
      # Should see the same wrapping instance in all towers.
      self.assertIs(all_v_sum[0], ret_v_sum)
      self.assertIs(all_v_mean[0], ret_v_mean)
      self.assertIs(all_v_sum[0], all_v_sum[1])
      self.assertIs(all_v_mean[0], all_v_mean[1])

      # Regroup should recover the same wrapper.
      self.assertIs(ret_v_sum, regrouped_sum)
      self.assertIs(ret_v_mean, regrouped_mean)
      self.assertIsNot(components_sum[0], components_sum[1])
      self.assertIsNot(components_mean[0], components_mean[1])

      # Apply updates
      self.evaluate(variables.global_variables_initializer())
      self.evaluate([y for x in ret_ops for y in dist.unwrap(x)])
      expected_sum = 0.0
      expected_mean = 0.0
      for i, d in enumerate(dist.worker_devices):
        # Should see different values on different devices.
        v_sum_value = self.evaluate(ret_v_sum.get(d).read_value())
        v_mean_value = self.evaluate(ret_v_mean.get(d).read_value())
        expected = i + 3.0
        self.assertEqual(expected, v_sum_value)
        expected_sum += expected
        expected = i * 6.0
        self.assertEqual(expected, v_mean_value)
        expected_mean += expected
      expected_mean /= len(dist.worker_devices)

      # Without get(device), should return the value you get by
      # applying the reduction across all towers (whether you use
      # read_var(), get(), or nothing).
      self.assertEqual(expected_sum, self.evaluate(dist.read_var(ret_v_sum)))
      self.assertEqual(expected_mean, self.evaluate(dist.read_var(ret_v_mean)))
      self.assertEqual(expected_sum, self.evaluate(ret_v_sum.get()))
      self.assertEqual(expected_mean, self.evaluate(ret_v_mean.get()))
      self.assertEqual(expected_sum, self.evaluate(ret_v_sum))
      self.assertEqual(expected_mean, self.evaluate(ret_v_mean))

  # NOTE(priyag): Names and name scopes are ignored in eager, hence we are not
  # testing this in eager mode.

  def testNameScope(self):
    def model_fn():
      with ops.name_scope("foo"):
        a = constant_op.constant(1.0, name="a")
        distribute_lib.get_tower_context().merge_call(lambda _: _)
        b = constant_op.constant(1.0, name="b")
      return a, b

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with context.graph_mode(), dist.scope():
      with ops.name_scope("main"):
        result = dist.call_for_each_tower(model_fn, run_concurrently=False)
        self.assertEquals(2, len(result))
        for v, name in zip(result, ["a", "b"]):
          self.assertIsInstance(v, values.DistributedValues)
          v0, v1 = dist.unwrap(v)
          self.assertEquals("main/foo/" + name + ":0", v0.name)
          self.assertEquals("main/tower_1/foo/" + name + ":0", v1.name)

  def testWithDefaultName(self):
    def model_fn():
      with ops.name_scope(None, "foo"):
        a = constant_op.constant(1.0, name="a")
        distribute_lib.get_tower_context().merge_call(lambda _: _)
        b = constant_op.constant(2.0, name="b")
      return a, b

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with context.graph_mode(), dist.scope():
      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      self.assertEquals(2, len(result))
      for v, name in zip(result, ["a", "b"]):
        self.assertIsInstance(v, values.DistributedValues)
        v0, v1 = dist.unwrap(v)
        self.assertEquals("foo/" + name + ":0", v0.name)
        self.assertEquals("tower_1/foo/" + name + ":0", v1.name)

  # variable_scope.variable() respects name scopes when creating
  # variables. On the other hand variable_scope.get_variable() ignores name
  # scopes when creating variables. We test both methods of creating variables
  # to make sure that we have the same variable names in both cases.
  def testNameScopeWithVariable(self):
    def in_cross_tower(_):
      c = variable_scope.variable(1.0, name="c")
      return c

    def model_fn():
      b = variable_scope.variable(1.0, name="b")
      with ops.name_scope("foo"):
        c = distribute_lib.get_tower_context().merge_call(in_cross_tower)
      return b, c

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with context.graph_mode(), dist.scope():
      with ops.name_scope("main"):
        a = variable_scope.variable(1.0, name="a")
        result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      result_b = result[0]
      result_c = result[1]
      self.assertIsInstance(result_b, values.DistributedValues)
      self.assertIsInstance(result_c, values.DistributedValues)
      a0, a1 = dist.unwrap(a)
      b0, b1 = dist.unwrap(result_b)
      c0, c1 = dist.unwrap(result_c)
      self.assertEquals("main/a:0", a0.name)
      self.assertEquals("main/a/replica_1:0", a1.name)
      self.assertEquals("main/b:0", b0.name)
      self.assertEquals("main/b/replica_1:0", b1.name)
      self.assertEquals("main/foo/c:0", c0.name)
      self.assertEquals("main/foo/c/replica_1:0", c1.name)

  def testNameScopeWithGetVariable(self):
    def in_cross_tower(_):
      c = variable_scope.get_variable("c", [1])
      return c

    def model_fn():
      b = variable_scope.get_variable("b", [1])
      with ops.name_scope("foo"):
        c = distribute_lib.get_tower_context().merge_call(in_cross_tower)
      return b, c

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with context.graph_mode(), dist.scope():
      with ops.name_scope("main"):
        a = variable_scope.get_variable("a", [1])
        result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      result_b = result[0]
      result_c = result[1]
      self.assertIsInstance(result_b, values.DistributedValues)
      self.assertIsInstance(result_c, values.DistributedValues)
      a0, a1 = dist.unwrap(a)
      b0, b1 = dist.unwrap(result_b)
      c0, c1 = dist.unwrap(result_c)
      self.assertEquals("a:0", a0.name)
      self.assertEquals("a/replica_1:0", a1.name)
      self.assertEquals("b:0", b0.name)
      self.assertEquals("b/replica_1:0", b1.name)
      self.assertEquals("c:0", c0.name)
      self.assertEquals("c/replica_1:0", c1.name)

  def testDynamicRnnVariables(self):
    def model_fn():
      inputs = constant_op.constant(2 * [2 * [[0.0, 1.0, 2.0, 3.0, 4.0]]])
      cell_fw = rnn_cell_impl.LSTMCell(300)
      cell_bw = rnn_cell_impl.LSTMCell(300)
      (outputs, _) = rnn.bidirectional_dynamic_rnn(
          cell_fw,
          cell_bw,
          inputs,
          dtype=dtypes.float32)
      return outputs

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with context.graph_mode(), dist.scope():
      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
      # Two variables are created by the RNN layer.
      self.assertEquals(2, len(result))
      for v in result:
        self.assertIsInstance(v, values.DistributedValues)
        _, v1 = dist.unwrap(v)
        self.assertStartsWith(v1.name, "tower_1/")

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testTowerLocalVariableUpdate(self):
    with context.graph_mode():

      def model_fn():
        v_sum = variable_scope.variable(
            1.0,
            synchronization=variable_scope.VariableSynchronization.ON_READ,
            aggregation=variable_scope.VariableAggregation.SUM)
        self.assertTrue(isinstance(v_sum, values.TowerLocalVariable))
        return v_sum

      dist = mirrored_strategy.MirroredStrategy(
          ["/device:GPU:0", "/device:GPU:1"])

      def update(var, value):
        return var.assign(value)

      with dist.scope():
        ret_v_sum = dist.call_for_each_tower(model_fn, run_concurrently=False)
        update_ops = dist.unwrap(dist.update(ret_v_sum, update, 5.0))

        # Initialize variables.
        self.evaluate(variables.global_variables_initializer())
        # Assert that the aggregated value of the tower local vars is the sum of
        # the individual values before running the update ops.
        self.assertEquals(1.0, self.evaluate(
            ret_v_sum.get(dist._devices[0]).read_value()))
        self.assertEquals(2.0, self.evaluate(ret_v_sum))

        # Apply updates.
        self.evaluate(update_ops)
        # Assert that the aggregated value of the tower local vars is the sum of
        # the individual values after running the update ops.
        self.assertEquals(5.0, self.evaluate(
            ret_v_sum.get(dist._devices[0]).read_value()))
        self.assertEquals(10.0, self.evaluate(ret_v_sum))

class MirroredVariableUpdateTest(test.TestCase):
  # The following tests check assign, assign_add and assign_sub on Mirrored
  # variables in tower and cross tower context.
  config = config_pb2.ConfigProto()
  config.allow_soft_placement = True

  def _skip_eager_if_gpus_less_than(self, num_gpus):
    if context.num_gpus() < num_gpus and context.executing_eagerly():
      self.skipTest("Enough GPUs not available for this test in eager mode.")

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignMirroredVarTowerContextWithoutAggregationType(self):
    # Test that we always have an aggregation type set on the mirrored variable
    # if we assign to it in tower mode.
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      v = variable_scope.variable(1.0, name="foo")
      return v

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())

      def model_fn():
        return mirrored_var.assign(5.0)

      with self.assertRaisesRegexp(
          ValueError, "You must specify an aggregation method to update a "
                      "MirroredVariable in Tower Context."):
        self.evaluate(dist.unwrap(dist.call_for_each_tower(model_fn)))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignMirroredVarTowerContextWithSum(self):
    # Test that we don't reduce a non-per-device value with the "sum"
    # aggregation type.
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      v = variable_scope.variable(
          1.0, name="foo", aggregation=variable_scope.VariableAggregation.SUM)
      return v

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())

      def model_fn():
        return mirrored_var.assign(5.0)

      with self.assertRaisesRegexp(
          ValueError, "A non-DistributedValues value cannot be reduced with "
          "the given aggregation."):
        self.evaluate(dist.unwrap(dist.call_for_each_tower(model_fn)))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignMirroredVarCrossTowerContext(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(1.0, name="foo")

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(1.0, self.evaluate(mirrored_var))
      mirrored_var_result = self.evaluate(mirrored_var.assign(6.0))
      self.assertEquals(6.0, mirrored_var_result)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignMirroredVarTowerContext(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(
          1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(1.0, self.evaluate(mirrored_var))

      def model_fn():
        value = math_ops.cast(distribute_lib.get_tower_context().tower_id,
                              mirrored_var.dtype)
        return mirrored_var.assign(value)

      self.evaluate(dist.unwrap(dist.call_for_each_tower(
          model_fn, run_concurrently=False)))
      self.assertEquals(0.5, self.evaluate(mirrored_var))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignMirroredVarTowerContextWithSingleValue(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(
          1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(1.0, self.evaluate(mirrored_var))

      def model_fn():
        return mirrored_var.assign(5.0)

      self.evaluate(dist.unwrap(dist.call_for_each_tower(
          model_fn, run_concurrently=False)))
      self.assertEquals(5.0, self.evaluate(mirrored_var))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignAddMirroredVarCrossTowerContext(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(1.0, name="foo")

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(1.0, self.evaluate(mirrored_var))
      mirrored_var_result = self.evaluate(mirrored_var.assign_add(6.0))
      self.assertEquals(7.0, mirrored_var_result)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignAddMirroredVarTowerContext(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(
          1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(1.0, self.evaluate(mirrored_var))

      def model_fn():
        value = math_ops.cast(distribute_lib.get_tower_context().tower_id,
                              mirrored_var.dtype)
        return mirrored_var.assign_add(value)

      self.evaluate(dist.unwrap(dist.call_for_each_tower(
          model_fn, run_concurrently=False)))
      self.assertEquals(1.5, self.evaluate(mirrored_var))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignAddMirroredVarTowerContextWithSingleValue(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(
          1.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(1.0, self.evaluate(mirrored_var))

      def model_fn():
        return mirrored_var.assign_add(5.0)

      self.evaluate(dist.unwrap(dist.call_for_each_tower(
          model_fn, run_concurrently=False)))
      self.assertEquals(6.0, self.evaluate(mirrored_var))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignSubMirroredVarCrossTowerContext(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(5.0, name="foo")

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(5.0, self.evaluate(mirrored_var))
      mirrored_var_result = self.evaluate(mirrored_var.assign_sub(2.0))
      self.assertEquals(3.0, mirrored_var_result)

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignSubMirroredVarTowerContext(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(
          5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(5.0, self.evaluate(mirrored_var))

      def model_fn():
        value = math_ops.cast(distribute_lib.get_tower_context().tower_id,
                              mirrored_var.dtype)
        return mirrored_var.assign_sub(value)

      self.evaluate(dist.unwrap(dist.call_for_each_tower(
          model_fn, run_concurrently=False)))
      self.assertEquals(4.5, self.evaluate(mirrored_var))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignSubMirroredVarTowerContextWithSingleValue(self):
    self._skip_eager_if_gpus_less_than(1)
    def var_fn():
      return variable_scope.variable(
          5.0, name="foo", aggregation=variable_scope.VariableAggregation.MEAN)

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      mirrored_var = dist.call_for_each_tower(var_fn, run_concurrently=False)
      self.assertIsInstance(mirrored_var, values.MirroredVariable)
      self.evaluate(variables.global_variables_initializer())
      self.assertEquals(5.0, self.evaluate(mirrored_var))

      def model_fn():
        return mirrored_var.assign_sub(1.0)

      self.evaluate(dist.unwrap(dist.call_for_each_tower(
          model_fn, run_concurrently=False)))
      self.assertEquals(4.0, self.evaluate(mirrored_var))

class MirroredAndTowerLocalVariableInitializerTest(test.TestCase):
  config = config_pb2.ConfigProto()
  config.allow_soft_placement = True

  def testAssignMirroredVarInitializer(self):
    # This test is not eager compatible since in eager variables are initialized
    # upon construction instead of once the initialization op is run.
    with context.graph_mode():
      def var_fn():
        v = variable_scope.variable(1.0, name="foo")
        return v

      dist = mirrored_strategy.MirroredStrategy(
          ["/device:GPU:0", "/device:CPU:0"])

      with dist.scope():
        mirrored_var = dist.call_for_each_tower(var_fn)
        self.assertIsInstance(mirrored_var, values.MirroredVariable)
        self.assertFalse(self.evaluate(mirrored_var.is_initialized()))
        self.evaluate(mirrored_var.initializer)
        self.assertTrue(self.evaluate(mirrored_var.is_initialized()))

  def testAssignTowerLocalVarInitializer(self):
    # This test is not eager compatible since in eager variables are initialized
    # upon construction instead of once the initialization op is run.
    with context.graph_mode():
      def model_fn():
        v_sum = variable_scope.variable(
            1.0,
            synchronization=variable_scope.VariableSynchronization.ON_READ,
            aggregation=variable_scope.VariableAggregation.SUM)
        self.assertTrue(isinstance(v_sum, values.TowerLocalVariable))
        return v_sum

      dist = mirrored_strategy.MirroredStrategy(
          ["/device:GPU:0", "/device:CPU:0"])

      with dist.scope():
        tower_local_var = dist.call_for_each_tower(model_fn)
        self.assertTrue(isinstance(tower_local_var, values.TowerLocalVariable))
        self.assertFalse(self.evaluate(tower_local_var.is_initialized()))
        self.evaluate(tower_local_var.initializer)
        self.assertTrue(self.evaluate(tower_local_var.is_initialized()))

class TowerLocalVariableAssignTest(test.TestCase):
  config = config_pb2.ConfigProto()
  config.allow_soft_placement = True

  def _skip_eager_if_gpus_less_than(self, num_gpus):
    if context.num_gpus() < num_gpus and context.executing_eagerly():
      self.skipTest("Not enough GPUs available for this test in eager mode.")

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignTowerLocalVarSumAggregation(self):
    self._skip_eager_if_gpus_less_than(1)
    def model_fn():
      v_sum = variable_scope.variable(
          1.0,
          synchronization=variable_scope.VariableSynchronization.ON_READ,
          aggregation=variable_scope.VariableAggregation.SUM)
      return v_sum

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      tower_local_var = dist.call_for_each_tower(model_fn,
                                                 run_concurrently=False)
      self.assertTrue(isinstance(tower_local_var, values.TowerLocalVariable))
      self.evaluate(variables.global_variables_initializer())
      # Each tower has a value of 1.0 assigned to it in tower context.
      # When we read the value using `read_var` we should see the SUM of each of
      # values on each of the towers.
      self.assertEqual(2.0, self.evaluate(dist.read_var(tower_local_var)))
      # Assigning 6.0 in cross tower context will assign a value of
      # 6.0/num_towers to each tower.
      tlv_ops = tower_local_var.assign(6.0)
      self.evaluate(tlv_ops)
      # On reading the tower local var we should get the assigned value back.
      # The value on all the towers are added before being returned by
      # `read_var`.
      self.assertEqual(6.0, self.evaluate(dist.read_var(tower_local_var)))

  @test_util.run_in_graph_and_eager_modes(config=config)
  def testAssignTowerLocalVarMeanAggregation(self):
    self._skip_eager_if_gpus_less_than(1)
    def model_fn():
      v_sum = variable_scope.variable(
          1.0,
          synchronization=variable_scope.VariableSynchronization.ON_READ,
          aggregation=variable_scope.VariableAggregation.MEAN)
      return v_sum

    dist = mirrored_strategy.MirroredStrategy(
        ["/device:GPU:0", "/device:CPU:0"])

    with dist.scope():
      tower_local_var = dist.call_for_each_tower(model_fn,
                                                 run_concurrently=False)
      self.assertTrue(isinstance(tower_local_var, values.TowerLocalVariable))
      self.evaluate(variables.global_variables_initializer())
      # Each tower has a value of 1.0 assigned to it in tower context.
      # When we read the value using `read_var` we should see the MEAN of values
      # on all towers which is the value assigned in tower context.
      self.assertEqual(1.0, self.evaluate(dist.read_var(tower_local_var)))
      tlv_ops = tower_local_var.assign(6.0)
      self.evaluate(tlv_ops)
      # On reading the tower local var we should get the MEAN of all values
      # which is equal to the value assigned.
      self.assertEqual(6.0, self.evaluate(dist.read_var(tower_local_var)))

class MockModel(object):

  def __init__(self, two_variables=False):
    self.variables = []
    self.variables.append(variable_scope.variable(1.25, name="dummy_var1"))
    if two_variables:
      self.variables.append(variable_scope.variable(2.0, name="dummy_var2"))

  def __call__(self, factor=2):
    x = factor * self.variables[0]
    if len(self.variables) > 1:
      x += self.variables[1]
    return x

class MirroredStrategyDefunTest(test.TestCase):

  def _skip_eager_if_gpus_less_than(self, num_gpus):
    if context.num_gpus() < num_gpus and context.executing_eagerly():
      self.skipTest("Not enough GPUs available for this test in eager mode.")

  def _call_and_check(self, model_fn, inputs, expected_result, defuns,
                      two_variables=False):
    cpu_dev = device_util.canonicalize("CPU:0")
    gpu_dev = device_util.canonicalize("GPU:0")
    devices = [cpu_dev, gpu_dev]
    dist = mirrored_strategy.MirroredStrategy(devices)

    with dist.scope():
      mock_model = MockModel(two_variables)
      self.evaluate(variables.global_variables_initializer())

      result = dist.call_for_each_tower(model_fn, mock_model, *inputs,
                                        run_concurrently=False)
      for device in devices:
        device_result = values.select_device(device, result)
        device_expected_result = values.select_device(device, expected_result)
        self.assertAllClose(device_expected_result,
                            self.evaluate(device_result))

      for defun in defuns:
        self.assertEqual(set(mock_model.variables), set(defun.variables))

  @test_util.run_in_graph_and_eager_modes()
  def testVariableInDefun(self):
    self._skip_eager_if_gpus_less_than(1)

    @function.defun
    def times_two(mock_model):
      return mock_model()

    def model_fn(mock_model):
      return times_two(mock_model)

    self._call_and_check(model_fn, [], 2.5, [times_two])

  @test_util.run_in_graph_and_eager_modes()
  def testVariableInNestedDefun(self):
    self._skip_eager_if_gpus_less_than(1)

    @function.defun
    def times_two(mock_model):
      return mock_model()

    @function.defun
    def two_x_plus_one(mock_model):
      return times_two(mock_model) + 1

    def model_fn(mock_model):
      return two_x_plus_one(mock_model)

    self._call_and_check(model_fn, [], 3.5, [times_two, two_x_plus_one])

  @test_util.run_in_graph_and_eager_modes()
  def testTwoVariablesInNestedDefun(self):
    self._skip_eager_if_gpus_less_than(1)

    @function.defun
    def fn1(mock_model):
      return mock_model()

    @function.defun
    def fn2(mock_model):
      return fn1(mock_model) + 1

    def model_fn(mock_model):
      return fn2(mock_model)

    self._call_and_check(model_fn, [], 5.5, [fn1, fn2], two_variables=True)

  @test_util.run_in_graph_and_eager_modes()
  def testGradientTapeOverNestedDefuns(self):
    self._skip_eager_if_gpus_less_than(1)

    @function.defun
    def fn1(mock_model):
      return mock_model()

    @function.defun
    def fn2(mock_model):
      return fn1(mock_model) + 1

    def model_fn(mock_model):
      with backprop.GradientTape(persistent=True) as gtape:
        result = fn2(mock_model)
      grads = gtape.gradient(result,
                             [v.get() for v in mock_model.variables])
      return grads

    self._call_and_check(model_fn, [], [2.0, 1.0], [fn1, fn2],
                         two_variables=True)

  @test_util.run_in_graph_and_eager_modes()
  def testPassPerDevice(self):
    self._skip_eager_if_gpus_less_than(1)

    @function.defun
    def fn1(mock_model, factor):
      return mock_model(factor)

    factors = values.PerDevice({"CPU:0": 5.0, "GPU:0": 3.0})
    expected_result = values.PerDevice({"CPU:0": 5.0 * 1.25,
                                        "GPU:0": 3.0 * 1.25})
    self._call_and_check(fn1, [factors], expected_result, [fn1])

if __name__ == "__main__":
  test.main()


text:
Used tools:
-----------

* Sublime Text 3 with my repo: https://github.com/karolyi/st3-config
  * csslint
  * jshint
  * htmllint
* jQuery UI Widget Factory: http://jqueryui.com/widget/

Since a grid system is not provided, no grid system is used by default. All the positioned elements are pixel positioned.

---

The data is loaded from the `data/cart.json` file at load time, and pushing the "Buy Now" button alerts the JSON format which would be posted to the REST API.

redpajama_set_name: RedPajamaGithub


text:
package nl.vu.cs.dynamite;

import java.io.File;
import java.io.IOException;
import java.util.List;

import nl.vu.cs.ajira.Ajira;
import nl.vu.cs.ajira.actions.ActionConf;
import nl.vu.cs.ajira.actions.ActionFactory;
import nl.vu.cs.ajira.actions.ActionSequence;
import nl.vu.cs.ajira.actions.WriteToFiles;
import nl.vu.cs.ajira.datalayer.InputLayer;
import nl.vu.cs.ajira.exceptions.ActionNotConfiguredException;
import nl.vu.cs.ajira.mgmt.WebServer;
import nl.vu.cs.ajira.submissions.Job;
import nl.vu.cs.ajira.submissions.Submission;
import nl.vu.cs.ajira.utils.Configuration;
import nl.vu.cs.ajira.utils.Consts;
import nl.vu.cs.dynamite.io.DBHandler;
import nl.vu.cs.dynamite.reasoner.actions.common.ActionsHelper;
import nl.vu.cs.dynamite.reasoner.rules.Rule;
import nl.vu.cs.dynamite.reasoner.rules.RuleParser;
import nl.vu.cs.dynamite.reasoner.rules.Ruleset;
import nl.vu.cs.dynamite.reasoner.support.ParamHandler;
import nl.vu.cs.dynamite.storage.BTreeInterface;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ListDB {
	static final Logger log = LoggerFactory.getLogger(ListDB.class);

	private static List<Rule> rules;
	private static boolean compressKeys = false;
	private static String storageName = BTreeInterface.defaultStorageBtree;
	private static Class<? extends InputLayer> storageClass;

	private static int nProcThreads = 4;

	public static void main(String[] args) {
		if (args.length < 3) {
			System.out.println("Usage: ListDB <KB_dir> <ruleset> <destination-dir> [ options ]");
			return;
		}

		parseArgs(args);
		try {
			storageClass = Class.forName(storageName).asSubclass(InputLayer.class);
		} catch(Exception e) {
			log.error("--storage parameter used incorrectly.", e);
			System.exit(1);
		}

		try {
			Ajira arch = new Ajira();
			initAjira(args[0], arch);
			arch.startup();
			readRules(args[1]);
			initGlobalContext(arch);
			launchLister(arch, args[2]);
			closeGlobalContext(arch);
			arch.shutdown();
		} catch (Exception e) {
			log.error("Error in the execution", e);
		}
	}

	private static void initGlobalContext(Ajira arch) {
		Ruleset set = new Ruleset(rules);
		ReasoningContext.getInstance().setRuleset(set);
		ReasoningContext.getInstance().setKB(arch.getContext().getInputLayer(storageClass));
		ReasoningContext.getInstance().setDBHandler(new DBHandler());
		ReasoningContext.getInstance().init();
	}

	private static void closeGlobalContext(Ajira arch) {
		ReasoningContext.getInstance().getKB().close();
		ReasoningContext.getInstance().getDBHandler().close();
	}

	private static void launchLister(Ajira arch, String dest) throws ActionNotConfiguredException {
		Job job = new Job();
		ActionSequence actions = new ActionSequence();
		ActionsHelper.readEverythingFromBTree(actions);
		ActionConf c = ActionFactory.getActionConf(WriteToFiles.class);
		c.setParamString(WriteToFiles.S_PATH, dest);
		actions.add(c);
		job.setActions(actions);

		if (arch.amItheServer()) {
			try {
				Submission s = arch.waitForCompletion(job);
				s.printStatistics();
			} catch (Exception e) {
				log.error("The job is failed!", e);
			}
		}
	}

	private static void initAjira(String kbDir, Ajira arch) throws IOException {

		// Check whether the input dir exists
		if (!new File(kbDir).exists()) {
			throw new IOException("Input dir " + kbDir + " does not exist!");
		}

		Configuration conf = arch.getConfiguration();
		if (kbDir != null) {
			conf.set(BTreeInterface.DB_INPUT, kbDir);
		}
		InputLayer.setDefaultInputLayerClass(storageClass, conf);
		conf.setInt(Consts.N_PROC_THREADS, nProcThreads);
		conf.setInt(WebServer.WEBSERVER_PORT, 50080);
	}

	private static void parseArgs(String[] args) {
		for (int i = 0; i < args.length; ++i) {
			if (args[i].equals("--countDerivations")) {
				ParamHandler.get().setUsingCount(true);
			} else if (args[i].equals("--storage")) {
				storageName = args[++i];
			} else if (args[i].equals("--procs")) {
				nProcThreads = Integer.parseInt(args[++i]);
			} else if (args[i].equals("--compressKeys")) {
				compressKeys = true;
			}
		}
	}

	private static void readRules(String fileName) {
		try {
			rules = new RuleParser().parseRules(fileName);
		} catch (Exception e) {
			log.error("Error parsing... ", e);
			log.error("Failed parsing the ruleset file. Exiting... ");
			System.exit(1);
		}
	}
}


text:
''' Gathers buildbot build statistics to an ElasticSerach database '''

import argparse
import cPickle
import datetime
import logging
import os
import re
import sys

import dateutil.tz
import pytz

import elasticsearch
import elasticsearch.connection
import elasticsearch.helpers

import buildbot.status.builder

_LOGGER = logging.getLogger('elastic-bot')

def main():
    ''' Entry Point '''
    args = _load_arguments()
    _init_logging(args.verbose)

    tz_name = args.buildbot_timezone
    timezone = dateutil.tz.tzlocal() if not tz_name else pytz.timezone(tz_name)

    database = elasticsearch.Elasticsearch(args.nodes)
    actions = _get_bulk_actions(database,
                                args.index,
                                args.builders_dir,
                                args.overwrite,
                                timezone)

    bulk = elasticsearch.helpers.parallel_bulk
    error = False
    for success, result in bulk(database, actions, thread_count=args.threads):
        doc_id = result['index']['_id']
        if not success:
            _LOGGER.error('Error indexing object %s : %s', doc_id, result)
            error = True
        else:
            _LOGGER.info('Indexed item %s', doc_id)

    return 1 if error else 0

def _init_logging(verbose):
    _LOGGER.setLevel(logging.DEBUG if verbose else logging.INFO)
    handler = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter(fmt=('[%(asctime)s] [%(levelname)s] '
                                       '%(message)s'),
                                  datefmt='%Y-%m-%d %H:%M:%S')
    handler.setFormatter(formatter)
    _LOGGER.addHandler(handler)

def _load_arguments():
    parser = argparse.ArgumentParser(description=('Gathers buildbot build '
                                                  'statistics and stores them '
                                                  'in an Elasticsearch database'
                                                  '.'))
    parser.add_argument('--verbose',
                        '-v',
                        action='store_true',
                        help='Enable verbose output')

    parser.add_argument('--builders-dir',
                        metavar='<dir>',
                        default='.',
                        help='Directory where are stored the buildbot logs')

    parser.add_argument('--nodes',
                        metavar='<node>',
                        default=['localhost:9200'],
                        nargs='*',
                        help='Elasticsearch database hosts')

    parser.add_argument('--index',
                        metavar='<index>',
                        default='buildbot',
                        help='Elasticsearch index where to store statistics')

    parser.add_argument('--overwrite',
                        action='store_true',
                        help=('Overwrites builds already indexed'))

    parser.add_argument('--threads',
                        '-t',
                        type=int,
                        default=1,
                        help=('Number of database threads to create.'))

    parser.add_argument('--buildbot-timezone',
                        type=str,
                        default=None,
                        help=('Timezone of timestamps stored in buildbot pickles'))

    return parser.parse_args()

def _get_bulk_actions(database, index, builders_dir, overwrite, timezone):
    try:
        last_builds = {} if overwrite else _get_last_builds(database, index)

        for builder_name, builder_dir in _discover_builders(builders_dir):
            if builder_name in last_builds:
                last_build = last_builds[builder_name]
            else:
                last_build = -1

            for build in _load_builds(builder_name, builder_dir, last_build):
                build_id = '_'.join([builder_name, str(build.number)])
                build_properties = _get_build_properties(build, timezone)
                _LOGGER.debug('Created build %s:%s document',
                              builder_name,
                              build.number)
                for key, value in build_properties.iteritems():
                    _LOGGER.debug('%s : %s', key, value)

                for step in build.steps:
                    if step.started is None:
                        continue

                    step_id = '_'.join([build_id, str(step.step_number)])
                    step_properties = _get_step_properties(build, step, timezone)
                    _LOGGER.debug('Created step %s:%s:%s document',
                                  builder_name,
                                  build.number,
                                  step.step_number)
                    for key, value in step_properties.iteritems():
                        _LOGGER.debug('%s : %s', key, value)
                    yield _get_action(index, 'step', step_id, step_properties)

                yield _get_action(index, 'build', build_id, build_properties)

                _LOGGER.debug('Loaded build %s:%s', builder_name, build.number)
    #pylint: disable=broad-except
    except Exception as ex:
        _LOGGER.error('An error occured during build informations gathering %s',
                      ex)

def _get_last_builds(database, index):
    body = {
        "query": {
            'match_all' : {}
        },
        "aggs" : {
            "builders": {
                "terms": {"field": "buildername", 'size':0},
                "aggs": {
                    "last_build": {
                        "max": {"field": "buildnumber"}
                    }
                }
            }
        }
    }

    page = database.search(index=index, doc_type='build', body=body)
    last_builds = {}
    for bucket in page['aggregations']['builders']['buckets']:
        builder_name = bucket['key']
        last_build = bucket['last_build']['value']
        if last_build is None:
            last_build = -1
        last_builds[builder_name] = int(last_build)

    for builder_name, last_build in last_builds.iteritems():
        logging.debug('Builder %s last indexed build is %s',
                      builder_name,
                      last_build)

    return last_builds

def _discover_builders(root_directory):
    for root, _, files in os.walk(root_directory):
        for file_name in files:
            if file_name == 'builder':
                builder_pickle_path = os.path.join(root, file_name)
                try:
                    with open(builder_pickle_path, 'r') as builder_pickle:
                        builder = cPickle.load(builder_pickle)
                #pylint: disable=broad-except
                except cPickle.UnpicklingError as ex:
                    _LOGGER.warn('Error unpickling builder file %s : %s.'
                                 ' this builder will be ignored.',
                                 builder_pickle_path, ex)
                    continue

                builder_dir = os.path.dirname(builder_pickle_path)
                builder_dir = os.path.abspath(builder_dir)
                builder_dir = os.path.normpath(builder_dir)
                _LOGGER.debug('Found new builder directory %s.', builder_dir)

                yield builder.name, builder_dir

def _load_builds(builder_name, builder_dir, last_indexed_build):
    for root, _, files in os.walk(builder_dir):
        for file_name in files:
            if re.match(r'^\d+$', file_name):
                build_number = int(file_name)
                build_pickle_path = os.path.join(root, file_name)

                if build_number <= last_indexed_build:
                    _LOGGER.debug('Ignoring already indexed build  %s',
                                  build_pickle_path)
                    continue

                try:
                    with open(build_pickle_path, 'r') as file_content:
                        build = cPickle.load(file_content)
                #pylint: disable=broad-except
                except cPickle.UnpicklingError as ex:
                    _LOGGER.warn(('Error while loading build pickle %s : '
                                  '%s. This build will be discarded.'),
                                 build_pickle_path, ex)
                    continue

                _LOGGER.debug('Loaded %s build pickle', build_pickle_path)

                if build.results is None:
                    _LOGGER.info(('Build %s:%s is not finished yet, '
                                  'ignoring it for now'),
                                 builder_name, build.number)
                    continue

                yield build

def _get_build_properties(build, timezone):
    document = _get_properties('build', build, build, timezone)

    trigger_date = 0
    has_changes = False
    if hasattr(build, 'sources'):
        for source_stamp in build.sources:
            if hasattr(source_stamp, 'changes'):
                for change in source_stamp.changes:
                    has_changes = True
                    if trigger_date < change.when:
                        trigger_date = change.when

    if has_changes:
        document['waiting_duration'] = build.started - trigger_date
        document['total_duration'] = build.finished  - trigger_date

    return document

def _get_step_properties(build, step, timezone):
    document = _get_properties('step', build, step, timezone)
    document['step_name'] = step.name
    document['step_number'] = step.step_number
    return document

def _get_properties(doc_type, build, build_or_step, timezone):
    start = datetime.datetime.fromtimestamp(build_or_step.started,
                                            tz=timezone)
    end = datetime.datetime.fromtimestamp(build_or_step.finished,
                                          tz=timezone)

    document = {
        'type' : doc_type,
        'blamelist' : '-'.join(build.blamelist),
        'start' : start,
        'end' : end,
        'duration' : build_or_step.finished - build_or_step.started,
        'result': buildbot.status.builder.Results[build_or_step.results],
    }

    for key, value in build.properties.asDict().iteritems():
        if key in ['workdir', 'scheduler', 'builddir']:
            continue
        value = value[0]
        if value is not None and value != '':
            document[key] = value

    return document

def _get_action(index, doc_type, doc_id, body):
    return {
        "_index": index,
        "_type": doc_type,
        "_id": doc_id,
        "_source": body
    }

if __name__ == '__main__':
    sys.exit(main())


text:
// Copyright (c) 2012 Ecma International.  All rights reserved.
// This code is governed by the BSD license found in the LICENSE file.

/*---
es5id: 15.2.3.7-6-a-306
description: >
    Object.defineProperties - 'O' is an Arguments object, 'P' is
    generic own data property of 'O', test TypeError is thrown when
    updating the [[Value]] attribute value of 'P' which is not
    writable and not configurable (10.6 [[DefineOwnProperty]] step 4)
includes: [propertyHelper.js]
---*/

var arg = (function () {
    return arguments;
}(1, 2, 3));

Object.defineProperty(arg, "genericProperty", {
    value: 1001,
    writable: false,
    configurable: false
});

try {
    Object.defineProperties(arg, {
        "genericProperty": {
            value: 1002
        }
    });

    $ERROR("Expected an exception.");
} catch (e) {
    verifyEqualTo(arg, "genericProperty", 1001);

    verifyNotWritable(arg, "genericProperty");

    verifyNotEnumerable(arg, "genericProperty");

    verifyNotConfigurable(arg, "genericProperty");

    if (!(e instanceof TypeError)) {
        $ERROR("Expected TypeError, got " + e);
    }

}


text:
var expect = require('expect');
var {
    CHANGE_MAP_VIEW,
    ERROR_FEATURE_INFO,
    EXCEPTIONS_FEATURE_INFO,
    LOAD_FEATURE_INFO,
    CHANGE_MAPINFO_STATE,
    NEW_MAPINFO_REQUEST,
    PURGE_MAPINFO_RESULTS,
    getFeatureInfo,
    changeMapView,
    changeMapInfoState,
    newMapInfoRequest,
    purgeMapInfoResults
} = require('../map');

describe('Test correctness of the map actions', () => {

    it('changeMapVeiw', () => {
        const testCenter = 0;
        const testZoom = 3;
        const testBbox = 6;
        const testSize = 9;
        var retval = changeMapView(testCenter, testZoom, testBbox, testSize);

        expect(retval).toExist();
        expect(retval.type).toBe(CHANGE_MAP_VIEW);
        expect(retval.center).toBe(testCenter);
        expect(retval.zoom).toBe(testZoom);
        expect(retval.bbox).toBe(testBbox);
        expect(retval.size).toBe(testSize);
    });

    it('get feature info data', (done) => {
        getFeatureInfo('base/web/client/test-resources/featureInfo-response.json', {})((e) => {
            try {
                expect(e).toExist();
                expect(e.type).toBe(LOAD_FEATURE_INFO);
                done();
            } catch(ex) {
                done(ex);
            }
        });
    });

    it('get feature info exception', (done) => {
        getFeatureInfo('base/web/client/test-resources/featureInfo-exception.json', {})((e) => {
            try {
                expect(e).toExist();
                expect(e.type).toBe(EXCEPTIONS_FEATURE_INFO);
                done();
            } catch(ex) {
                done(ex);
            }
        });
    });

    it('get feature info error', (done) => {
        getFeatureInfo('requestError.json', {})((e) => {
            try {
                expect(e).toExist();
                expect(e.type).toBe(ERROR_FEATURE_INFO);
                done();
            } catch(ex) {
                done(ex);
            }
        });
    });

    it('change map info state', () => {
        const testVal = "val";
        const retval = changeMapInfoState(testVal);

        expect(retval.type).toBe(CHANGE_MAPINFO_STATE);
        expect(retval.enabled).toExist();
        expect(retval.enabled).toBe(testVal);
    });

    it('add new info request', () => {
        const testVal = "val";
        const retval = newMapInfoRequest(testVal);

        expect(retval.type).toBe(NEW_MAPINFO_REQUEST);
        expect(retval.request).toExist();
        expect(retval.request).toBe(testVal);
    });

    it('delete all results', () => {
        const retval = purgeMapInfoResults();

        expect(retval.type).toBe(PURGE_MAPINFO_RESULTS);
    });
});


text:
package com.braintreepayments.cardform.view;

import com.google.android.material.textfield.TextInputLayout;
import android.text.Editable;
import android.text.InputType;
import android.text.method.PasswordTransformationMethod;
import android.text.method.SingleLineTransformationMethod;

import com.braintreepayments.cardform.R;
import com.braintreepayments.cardform.test.TestActivity;
import com.braintreepayments.cardform.utils.CardType;

import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.Robolectric;
import org.robolectric.RobolectricTestRunner;
import org.robolectric.RuntimeEnvironment;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;

@RunWith(RobolectricTestRunner.class)
public class CvvEditTextTest {

    private CvvEditText mView;

    @Before
    public void setup() {
        mView = Robolectric.setupActivity(TestActivity.class)
                .findViewById(R.id.bt_card_form_cvv);
    }

    @Test
    public void defaultLimitIs3() {
        type("123").assertTextIs("123");
        type("4").assertTextIs("123");
    }

    @Test
    public void customLimits() {
        for (CardType type : CardType.values()) {
            mView.setCardType(type);
            if (type == CardType.AMEX) {
                type("1234").assertTextIs("1234");
                type("5").assertTextIs("1234");
            } else {
                type("123").assertTextIs("123");
                type("4").assertTextIs("123");
            }
            mView.getText().clear();
        }
    }

    @Test
    public void hintDefaultsToCvv() {
        assertEquals(((TextInputLayout) mView.getParent().getParent()).getHint(),
                RuntimeEnvironment.application.getString(R.string.bt_cvv));
    }

    @Test
    public void hintChangesForCardType() {
        for (CardType cardType : CardType.values()) {
            mView.setCardType(cardType);

            assertEquals(RuntimeEnvironment.application.getString(cardType.getSecurityCodeName()),
                    mView.getTextInputLayoutParent().getHint());
        }
    }

    @Test
    public void onInit_setsInputTypeToNumberAndKeepsDefaultTransformationMethod() {
        assertEquals(InputType.TYPE_CLASS_NUMBER, mView.getInputType());
        assertSame(mView.getTransformationMethod(), SingleLineTransformationMethod.getInstance());
    }

    @Test
    public void setMaskToTrue_appliesPasswordTransformationMethod() {
        mView.setMask(true);
        assertSame(mView.getTransformationMethod(), PasswordTransformationMethod.getInstance());
    }

    @Test
    public void setMaskToFalse_doesNotApplyPasswordTransformationMethod() {
        mView.setMask(false);
        assertSame(mView.getTransformationMethod(), SingleLineTransformationMethod.getInstance());
    }

    @Test
    public void isValid_returnsFalseForInvalidCvv() {
        assertFalse(mView.isValid());
    }

    @Test
    public void isValid_returnsTrueForValidCvv() {
        type("123");

        assertTrue(mView.isValid());
    }

    @Test
    public void isValid_returnsTrueForInvalidCvvWhenOptional() {
        mView.setOptional(true);

        assertTrue(mView.isValid());
    }

    @Test
    public void getErrorMessage_returnsErrorMessageForNoCardTypeWhenEmpty() {
        String expectedMessage = RuntimeEnvironment.application.getString(R.string.bt_cvv_required,
                RuntimeEnvironment.application.getString(R.string.bt_cvv));

        assertEquals(expectedMessage, mView.getErrorMessage());
    }

    @Test
    public void getErrorMessage_returnsCorrectErrorMessageForCardTypeWhenEmpty() {
        for (CardType cardType : CardType.values()) {
            mView.setCardType(cardType);

            String expectedMessage = RuntimeEnvironment.application.getString(R.string.bt_cvv_required,
                    RuntimeEnvironment.application.getString(cardType.getSecurityCodeName()));
            assertEquals(expectedMessage, mView.getErrorMessage());
        }
    }

    @Test
    public void getErrorMessage_returnsErrorMessageForNoCardTypeWhenNotEmpty() {
        type("4");
        String expectedMessage = RuntimeEnvironment.application.getString(R.string.bt_cvv_invalid,
                RuntimeEnvironment.application.getString(R.string.bt_cvv));

        assertEquals(expectedMessage, mView.getErrorMessage());
    }

    @Test
    public void getErrorMessage_returnsCorrectErrorMessageForCardTypeWhenNotEmpty() {
        type("4");

        for (CardType cardType : CardType.values()) {
            mView.setCardType(cardType);

            String expectedMessage = RuntimeEnvironment.application.getString(R.string.bt_cvv_invalid,
                    RuntimeEnvironment.application.getString(cardType.getSecurityCodeName()));
            assertEquals(expectedMessage, mView.getErrorMessage());
        }
    }

    private CvvEditTextTest type(String text) {
        Editable editable = mView.getText();
        for (char c : text.toCharArray()) {
            editable.append(c);
        }
        return this;
    }

    private void assertTextIs(String expected) {
        assertEquals(expected, mView.getText().toString());
    }
}


text:
package com.ea.orbit.async.test;

import com.ea.orbit.concurrent.Task;

import org.junit.Test;

import java.lang.reflect.InvocationTargetException;

import static com.ea.orbit.async.Await.await;
import static com.ea.orbit.concurrent.Task.done;
import static com.ea.orbit.concurrent.Task.fromValue;
import static org.junit.Assert.assertEquals;

public class LocalVarsTest extends BaseTest
{

    @Test
    public void testRepeatedLocalVarsNames() throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException
    {
        final Task<Integer> res = done().thenCompose(o -> {
            {
                String a = "a1";
            }
            await(getBlockedTask());
            {
                String a = "a2";
            }
            return fromValue(10);
        });
        completeFutures();
        assertEquals((Integer) 10, res.join());
    }

    @Test
    public void testRepeatedLocalVarsNames2() throws IllegalAccessException, InstantiationException, InvocationTargetException, NoSuchMethodException
    {
        final Task<Integer> res = done().thenCompose(o -> {
            {
                String a = "a1";
                String b = "b1";
                int c = 1;
            }
            await(getBlockedTask());
            {
                String a = "a2";
                String b = "b2";
                int c = 10;
                return fromValue(c);
            }
        });
        completeFutures();
        assertEquals((Integer) 10, res.join());
    }
}


text:
@interface BoxPreviewViewController : UIViewController <UIAlertViewDelegate>

@property (nonatomic, readwrite, strong) NSString *fileID;
@property (nonatomic, readwrite, strong) NSString *filename;
@property (nonatomic, readwrite, strong) NSString *parentFolderID;
@property (nonatomic, readwrite, strong) NSData *data;
@property (nonatomic, readwrite, strong) NSString *MIMEType;

- (id)initWithFileID:(NSString *)fileID filename:(NSString *)filename parentFolderID:(NSString *)parentFolderID data:(NSData *)data MIMEType:(NSString *)MIMEType;

@end


text:
package org.apache.ibatis.lang;

import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
 *

 * Indicates that the element uses Java 8 API.

 * It can be used with JUnit's Category annotation.
 *

 */
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target({ ElementType.TYPE, ElementType.METHOD, ElementType.FIELD })
public @interface UsesJava8 {
}


text:
"""
OpenStreetMap XML file parsing
==============================

This module defined classes for nodes, ways and a shallow data structure for
all of these raw information from OSM XML file, as well as a parser based on
the standard expat library.

It contains classes for holding information about the raw map

.. autosummary::
    :toctree: generated
    :template: classtempl.rstt

    Node
    Way
    RawOSM

and one function for parsing the raw OSM XML file in the data structure

.. autosummary::
    :toctree: generated

   read_osm

"""

# pylint: disable=too-few-public-methods
# pylint: disable=too-many-branches

import xml.parsers.expat as expat

import numpy as np

#
# Class definitions
# -----------------
#

class Node(object):

    """Nodes in the OSM XML file

    It mainly consists of the following attributes,

    .. py:attribute:: coord

        The coordinates for the node, in latitudes and longitudes.

    .. py:attribute:: tags

        A dictionary for the tags, using the same key and value as in the XML
        file.

    """

    __slots__ = [
        "coord",
        "tags",
        ]

    def __init__(self, attrs):

        """Initializes a node based on the attributes in XML

        The coordinates are going to be set, with the tags initialized to an
        empty dictionary.

        """

        self.coord = np.array(
            (attrs['lat'], attrs['lon']),
            dtype=np.float64
            )
        self.tags = {}

class Way(object):

    """Ways from the OSM XML file

    It primarily consists of the following attributes

    .. py:attribute:: nodes

        A list of identity references for the nodes

    .. py:attribute:: tags

        The tags dictionary, the same as that for the nodes

    """

    __slots__ = [
        "nodes",
        "tags",
        ]

    def __init__(self):

        """Initializes an instance

        The initialization is going to be trivial, just the lists and
        dictionaries set to empty ones.

        """

        self.nodes = []
        self.tags = {}

class RawOSM(object):

    """Raw GIS data from OpenStreetMap

    The XML input is going to be parsed into an instance of this project. The
    primary fields are

    .. py:attribute:: nodes

        A dictionary of nodes, with the identity integer as the key as the
        :py:class:`Node` instances as values.

    .. py:attribute:: ways

        A similar dictionary of ways, with identity as the key and the
        :py:class:`Way` instances as the values.

    """

    __slots__ = [
        "nodes",
        "ways",
        ]

    def __init__(self):

        """Initializes the instance

        It just sets the two dictionaries into empty ones.

        """

        self.nodes = {}
        self.ways = {}

#
# The parser function
# -------------------
#

def read_osm(file_name):

    """Reads the OSM XML file with given name

    :param file_name: The name of the OSM XML file
    :returns: A :py:class:`RawOSM` instance for the data
    :raises: :py:exc:`ValueError` if something went wrong

    """

    raw_osm = RawOSM()

    # The current state, to be used as a stack
    # Its entries should be pair of element id and element for nodes and ways
    current_state = []

    # The closures for the call-backs

    def start_element(name, attrs):

        """Call-back at the start of element"""

        # At top level
        if name == 'osm':
            pass
        # For a node
        elif name == 'node':
            current_state.append(
                (int(attrs['id']), Node(attrs))
                )
        # For a way
        elif name == 'way':
            current_state.append(
                (int(attrs['id']), Way())
                )
        # For a node in a way
        elif name == 'nd':
            parent = current_state[-1]
            if isinstance(parent, tuple) and isinstance(parent[1], Way):
                parent[1].nodes.append(int(attrs['ref']))
            else:
                pass
        # if a tag
        elif name == 'tag':
            parent = current_state[-1]
            if isinstance(parent, tuple) and (
                    isinstance(parent[1], Node) or isinstance(parent[1], Way)
                    ):
                (parent[1].tags)[attrs['k']] = attrs['v']
            else:
                pass
        # For unused relation
        elif name == 'relation':
            current_state.append(None)
        elif name == 'member':
            pass
        elif name == 'bounds':
            pass
        else:
            raise ValueError('Unrecognized XML node %s' % name)

    def end_element(name):

        """Call back at the end of elements"""

        if name in ['osm', 'tag', 'member', 'nd', 'bounds']:
            pass
        elif name == 'node':
            new_node = current_state.pop()
            raw_osm.nodes[new_node[0]] = new_node[1]
        elif name == 'way':
            new_way = current_state.pop()
            raw_osm.ways[new_way[0]] = new_way[1]
        elif name == 'relation':
            current_state.pop()
        else:
            raise ValueError('Unrecognized XML node %s' % name)

    parser = expat.ParserCreate()
    parser.StartElementHandler = start_element
    parser.EndElementHandler = end_element
    try:
        input_file = open(file_name, 'r')
        parser.ParseFile(input_file)
    except IOError:
        raise ValueError('Input file %s unable to be opened' % file_name)
    except expat.ExpatError as err:
        raise ValueError(
            'Expat parsing failure at line %d column %d of file %s' % (
                err.lineno, err.offset, input_file
                )
            )

    return raw_osm


text:
require 'delegate'

module Travis::Yaml
  class Matrix < DelegateClass(Array)
    EXPAND_KEYS = [
      :compiler, :gemfile, :ghc, :go, :jdk, :lein, :node_js, :otp_release,
      :perl, :php, :python, :ruby, :scala, :xcode_scheme, :xcode_sdk, :os
    ]

    KEYS = EXPAND_KEYS + [:env]

    class Entry < DelegateClass(Nodes::Root)
      attr_reader :env, :matrix_attributes

      def initialize(root, matrix_attributes)
        super(root)

        @matrix_attributes = matrix_attributes
        @env               = Nodes::Env.new(self)
        inherited_env      = root.env.global if root.env
        @env.global        = [matrix_attributes[:env], *inherited_env].compact
      end

      EXPAND_KEYS.each do |key|
        define_method(key) { @matrix_attributes.fetch(key, super()) }
      end

      def inspect
        "#<#{self.class}: #{matrix_attributes}>"
      end
    end

    attr_reader :root

    def initialize(root)
      @root = root
      super(entries)
    end

    def axes
      @axes ||= KEYS.select do |key|
        next true if values_for(key) and values_for(key).size > 1
        root.matrix.include.any? { |i| i[key] } if root.matrix and root.matrix.include
      end
    end

    def entries
      @entries ||= begin
        first, *rest = axes.map { |k| values_for(k) || [nil] }
        entries      = Array(first).product(*rest).map { |list| Hash[axes.zip(list)] }
        if m = root.matrix
          entries.delete_if { |e| m.exclude.any? { |p| p.all? { |k,v| e[k.to_sym] == v } } } if m.exclude
          m.include.each { |i| entries << Hash[axes.map { |k| [k, i[k]] }] } if m.include
        end
        entries.map! { |attributes| Entry.new(root, attributes) }
        entries.any? ? entries : [root]
      end
    end

    def values_for(key)
      return root[key] unless key == :env
      root.env.matrix if root.env
    end
  end
end


text:
.class Lcom/zte/fingerprint/setting/GoodixSetting$1;
.super Ljava/lang/Object;
.source "GoodixSetting.java"

# interfaces
.implements Lcom/zte/mifavor/preference/Preference$OnPreferenceClickListener;

# annotations
.annotation system Ldalvik/annotation/EnclosingMethod;
    value = Lcom/zte/fingerprint/setting/GoodixSetting;->initComponent()V
.end annotation

.annotation system Ldalvik/annotation/InnerClass;
    accessFlags = 0x0
    name = null
.end annotation

# instance fields
.field final synthetic this$0:Lcom/zte/fingerprint/setting/GoodixSetting;

.field final synthetic val$currentCount:I

.field final synthetic val$prfScreenLock:Lcom/zte/mifavor/preference/SwitchPreference;

# direct methods
.method constructor <init>(Lcom/zte/fingerprint/setting/GoodixSetting;ILcom/zte/mifavor/preference/SwitchPreference;)V
    .locals 0
    .param p1, "this$0"    # Lcom/zte/fingerprint/setting/GoodixSetting;

    .prologue
    .line 86
    iput-object p1, p0, Lcom/zte/fingerprint/setting/GoodixSetting$1;->this$0:Lcom/zte/fingerprint/setting/GoodixSetting;

    iput p2, p0, Lcom/zte/fingerprint/setting/GoodixSetting$1;->val$currentCount:I

    iput-object p3, p0, Lcom/zte/fingerprint/setting/GoodixSetting$1;->val$prfScreenLock:Lcom/zte/mifavor/preference/SwitchPreference;

    invoke-direct {p0}, Ljava/lang/Object;-><init>()V

    return-void
.end method

# virtual methods
.method public onPreferenceClick(Lcom/zte/mifavor/preference/Preference;)Z
    .locals 2
    .param p1, "preference"    # Lcom/zte/mifavor/preference/Preference;

    .prologue
    .line 89
    iget v0, p0, Lcom/zte/fingerprint/setting/GoodixSetting$1;->val$currentCount:I

    if-nez v0, :cond_0

    .line 90
    iget-object v0, p0, Lcom/zte/fingerprint/setting/GoodixSetting$1;->this$0:Lcom/zte/fingerprint/setting/GoodixSetting;

    iget-object v1, p0, Lcom/zte/fingerprint/setting/GoodixSetting$1;->val$prfScreenLock:Lcom/zte/mifavor/preference/SwitchPreference;

    # invokes: Lcom/zte/fingerprint/setting/GoodixSetting;->showAddFingerDialog(Lcom/zte/mifavor/preference/SwitchPreference;)V
    invoke-static {v0, v1}, Lcom/zte/fingerprint/setting/GoodixSetting;->access$000(Lcom/zte/fingerprint/setting/GoodixSetting;Lcom/zte/mifavor/preference/SwitchPreference;)V

    .line 92
    :cond_0
    const/4 v0, 0x0

    return v0
.end method


text:
#include "config.h"
#include "core/animation/ElementAnimations.h"

#include "core/layout/LayoutObject.h"

namespace blink {

ElementAnimations::ElementAnimations()
    : m_animationStyleChange(false)
{
}

ElementAnimations::~ElementAnimations()
{
#if !ENABLE(OILPAN)
    for (KeyframeEffect* effect : m_effects)
        effect->notifyElementDestroyed();
    m_effects.clear();
#endif
}

void ElementAnimations::updateAnimationFlags(ComputedStyle& style)
{
    for (const auto& entry : m_animations) {
        const Animation& animation = *entry.key;
        ASSERT(animation.effect());
        // FIXME: Needs to consider AnimationGroup once added.
        ASSERT(animation.effect()->isAnimation());
        const KeyframeEffect& effect = *toKeyframeEffect(animation.effect());
        if (effect.isCurrent()) {
            if (effect.affects(PropertyHandle(CSSPropertyOpacity)))
                style.setHasCurrentOpacityAnimation(true);
            if (effect.affects(PropertyHandle(CSSPropertyTransform))
                || effect.affects(PropertyHandle(CSSPropertyRotate))
                || effect.affects(PropertyHandle(CSSPropertyScale))
                || effect.affects(PropertyHandle(CSSPropertyTranslate)))
                style.setHasCurrentTransformAnimation(true);
            if (effect.affects(PropertyHandle(CSSPropertyWebkitFilter)))
                style.setHasCurrentFilterAnimation(true);
        }
    }

    if (style.hasCurrentOpacityAnimation())
        style.setIsRunningOpacityAnimationOnCompositor(m_defaultStack.hasActiveAnimationsOnCompositor(CSSPropertyOpacity));
    if (style.hasCurrentTransformAnimation())
        style.setIsRunningTransformAnimationOnCompositor(m_defaultStack.hasActiveAnimationsOnCompositor(CSSPropertyTransform));
    if (style.hasCurrentFilterAnimation())
        style.setIsRunningFilterAnimationOnCompositor(m_defaultStack.hasActiveAnimationsOnCompositor(CSSPropertyWebkitFilter));
}

void ElementAnimations::restartAnimationOnCompositor()
{
    for (const auto& entry : m_animations)
        entry.key->restartAnimationOnCompositor();
}

DEFINE_TRACE(ElementAnimations)
{
#if ENABLE(OILPAN)
    visitor->trace(m_cssAnimations);
    visitor->trace(m_defaultStack);
    visitor->trace(m_animations);
#endif
}

const ComputedStyle* ElementAnimations::baseComputedStyle() const
{
#if !ENABLE(ASSERT)
    if (isAnimationStyleChange())
        return m_baseComputedStyle.get();
#endif
    return nullptr;
}

void ElementAnimations::updateBaseComputedStyle(const ComputedStyle* computedStyle)
{
    if (!isAnimationStyleChange()) {
        m_baseComputedStyle = nullptr;
        return;
    }
#if ENABLE(ASSERT)
    if (m_baseComputedStyle && computedStyle)
        ASSERT(*m_baseComputedStyle == *computedStyle);
#endif
    m_baseComputedStyle = ComputedStyle::clone(*computedStyle);
}

void ElementAnimations::clearBaseComputedStyle()
{
    m_baseComputedStyle = nullptr;
}

bool ElementAnimations::isAnimationStyleChange() const
{
    // TODO(rune@opera.com): The FontFaceCache version number may be increased without forcing
    // a style recalc (see crbug.com/471079). ComputedStyle objects created with different cache
    // versions will not be considered equal as Font::operator== will compare versions, hence
    // ComputedStyle::operator== will return false. We avoid using baseComputedStyle (the check for
    // isFallbackValid()) in that case to avoid triggering the ComputedStyle comparison ASSERT
    // in updateBaseComputedStyle.
    return m_animationStyleChange && (!m_baseComputedStyle || m_baseComputedStyle->font().isFallbackValid());
}

} // namespace blink


text:
package com.pinterest.secor.util;

import com.google.common.util.concurrent.RateLimiter;
import com.pinterest.secor.common.SecorConfig;

/**
 * Rate limit util wraps around a rate limiter shared across consumer threads.  The rate limiting
 * mechanism does not prevent temporary bursts of the load on the broker caused by the message
 * prefetching mechanism of the native kafka client.
 *
 * @author Pawel Garbacki (pawel@pinterest.com)
 */
public class RateLimitUtil {
    private static RateLimiter mRateLimiter = null;

    public static void configure(SecorConfig config) {
        // Lazy initialization of the rate limiter would have to be in a synchronized block so
        // creating it here makes things simple.
        mRateLimiter = RateLimiter.create(config.getMessagesPerSecond());
    }

    public static void acquire(int n) {
        mRateLimiter.acquire(n);
    }
}