text:
""" Training script for the retrieval solution.
"""

import functools
import itertools
import json
import logging
import os
import subprocess
import tempfile
import time
from typing import Any, Callable, Dict, List, Optional

from absl import app
from absl import flags
from absl import logging as absl_logging
import colorama
import constants
import numpy as np
import task_specific
import tensor2tensor.utils.adafactor
import tensorflow as tf
import tensorflow.python.distribute.values as values
import tensorflow.python.framework.ops as ops
import tf_utils
import transformers
import utils

LOGGER = logging.getLogger(__name__)
SCRIPT_DIRECTORY = os.path.realpath(os.path.dirname(__file__))

LOGGER.debug(
    "############################################################"
    ">>>>>>>>>>>>>>> Tensorflow version: %s <<<<<<<<<<<<<<<<"
    "############################################################",
    str(tf.__version__)
)

################################################################################
# Flag Definitions
################################################################################
FLAGS = flags.FLAGS

# It is now recommended that one uses the return values of DEFINE_* calls
# because they can by pytype-checked and the intellisense/linter can know
# if the wrong variable name is called, contrarily to the FLAGS.* case.
FLAG_APPROACH_TYPE = flags.DEFINE_enum(
    "approach_type",
    None,
    constants.ApproachTypeChoices.choices(),
    "Type of approach to use.\n"
)
FLAG_MODEL_KEY = flags.DEFINE_string(
    "model_key",
    None,
    "Hugging Face key associated to the pre-trained model."
)
FLAG_RUN_NAME = flags.DEFINE_string(
    "run_name",
    None,
    "Name of the run. Can be anything."
)
FLAG_MODEL_LOAD_PATH = flags.DEFINE_string(
    "model_load_path",
    None,
    "Directory where the pre-trained model was saved."
)
FLAG_OUTPUT_DIR = flags.DEFINE_string(
    "output_dir",
    None,
    "Where to save the results to."
)
FLAG_BATCH_SIZE = flags.DEFINE_integer(
    "batch_size",
    None,
    "Inference batch size."
)
FLAG_BATCH_SPLIT = flags.DEFINE_integer(
    "batch_split",
    None,
    "Used for manual_improved. Sub-batch size."
)
FLAG_TASK = flags.DEFINE_enum(
    "task",
    constants.TaskChoices.train,
    constants.TaskChoices.choices(),
    "Whether to train or to evaluate the mode."
)
FLAG_RANDOM_SEED = flags.DEFINE_integer(
    "random_seed", 0,
    "Random seed used used for the random elements of the script."
)
FLAG_DB_PATH = flags.DEFINE_string(
    "db_path",
    None,
    "Path to the h5 file containing the dataset prepared with query_cacher.py"
)

# TPU Specific Args
FLAG_EXPERIMENTAL_COMPILE = flags.DEFINE_bool(
    "experimental_compile",
    False,
    "Whether to use experimental compile with the train and eval functions."
)
FLAG_DISTRIBUTE_MODE = flags.DEFINE_enum(
    "distribute_mode",
    constants.DistributeModeChoices.split_vertically,
    constants.DistributeModeChoices.choices(),
    "What type of infrastructure to use to distribute the work."
)
FLAG_NUM_REPLICAS = flags.DEFINE_integer(
    "num_replicas",
    1,
    "Number of replicas to use fordata parallelism."
)

# Training specific flags
FLAG_MODEL_OUTPUT_PATH = flags.DEFINE_string(
    "model_output_path",
    None,
    "Where to save the model."
)
FLAG_LEARNING_RATE = flags.DEFINE_float(
    "learning_rate",
    None,
    "Learning rate for the optimizer."
)
FLAG_BATCHES_BETWEEN_EVALS = flags.DEFINE_integer(
    "batches_between_evals",
    5,
    "Number of batches between eval passes."
)
FLAG_NUMBER_EVAL_BATCHES = flags.DEFINE_integer(
    "number_eval_batches",
    1,
    "Number of eval batches when doing an eval pass."
)
FLAG_USE_HELPER_WORDS = flags.DEFINE_boolean(
    "use_helper_words",
    True,
    "Whether to add guiding words in the inputs, like `Question:`,"
    " `Answer:` and `Context:`. ")

# Retriever specific flags
FLAG_QUERY_END = flags.DEFINE_integer(
    "query_end",
    256,
    "When querying once, length of the query being taken from the inputs."
)
FLAG_RETRIEVER_CONFIG_PATH = flags.DEFINE_string(
    "retriever_config_path",
    None,
    "Path to the configuration file for the retrievers."
)
FLAG_SCANN_CONFIG_PATH = flags.DEFINE_string(
    "scann_config_path",
    os.path.join(
        SCRIPT_DIRECTORY, "configs", "scann_configs", "default_config.json"
    ),
    "Configuration file for the ScaNN MIPS library."
)
FLAG_NUM_RETRIEVALS = flags.DEFINE_integer(
    "num_retrievals",
    None,
    "Number of neighbors to get with each retrieval."
    )
FLAG_RETRIEVAL_TEMPERATURE = flags.DEFINE_float(
    "retrieval_temperature",
    None,
    "Temperature to be used with the sampling in the softmax of certain "
    "retrievers (just retrievers.FullyCacherRetriever currently)."
)
FLAG_FULLYCACHED_H5_PATH = flags.DEFINE_string(
    "fullycached_h5_path",
    None,
    "Path to the .h5 file to be used by the fully cached retriever."
)
FLAG_RETRIEVAL_BANK_SIZE = flags.DEFINE_integer(
    "retrieval_bank_size",
    10,
    "Number of segments to sample from for the retrievals."
)

# Dataset specific flags
FLAG_DATASET_DEBUG = flags.DEFINE_boolean(
    "dataset_debug",
    False,
    "Whether to enable costly runtime checks for the dataset."
)
FLAG_INPUT_FIXED_SIZE = flags.DEFINE_boolean(
    "input_fixed_sized",
    True,
    "Whether to pad all inputs to the same size.")
FLAG_DATASET_NAME = flags.DEFINE_enum(
    "dataset_name",
    None,
    constants.DatasetNameChoices.choices(),
    "Name or TFDS key of the dataset we want"
)
FLAG_DATASET_PATH_ROOT = flags.DEFINE_string(
    "dataset_paths_root",
    None,
    "Where to look for datasets."
)
FLAG_HF_DATASETS_CACHE_DIR = flags.DEFINE_string(
    "hf_datasets_cache_dir",
    None,
    "Path to the cache of the huggingface datasets."
)
FLAG_USE_SUBSET = flags.DEFINE_bool(
    "use_subset",
    False,
    "Whether to just use a subset of the data."
)
FLAG_SUBSET_SIZE = flags.DEFINE_integer(
    "subset_size",
    1000,
    "If we are using a subset of the data number of samples to use."
)

FLAG_DATASET_TYPE = flags.DEFINE_enum(
    "dataset_type",
    constants.DatasetTypeChoices.hdf5,
    constants.DatasetTypeChoices.choices(),
    "Whether to use tfrecords or hdf5 as an input method."
)
FLAG_QTY_SHUFFLE = flags.DEFINE_integer(
    "qty_shuffle",
    100,
    "Shuffle how many samples every time."
)
FLAG_TFR_PREFIX = flags.DEFINE_string(
    "tfr_prefix",
    None,
    "Prefix of the location of the tf record dataset.",
)
FLAG_MAX_LENGTH_GENERATION = flags.DEFINE_integer(
    "max_length_generation",
    None,
    "Maximum length of the generation."
)

################################################################################
# Training and evaluation step functions.
################################################################################
# With tf.function, one can't pass non-tensor objects. This makes it so all
# non-tensor objects need to be passed through non-local references, making
# the step functions closures. In order to make our code cleaner / make
# dependencies more explicit, we build the closures with builder functions that
# explicitly show each step function's dependencies.
def build_regular_training_step(
    model,
    optimizer,
    strategy,
    tf_function_kwargs = None
):
  """Build the training step that is used in all cases but vertical mod. par."""
  tf_function_kwargs = {} if tf_function_kwargs is None else tf_function_kwargs

  @tf.function(**tf_function_kwargs)
  def training_step(input_ids, label_ids):
    """Computes the loss, backpropagates gradients, updates weights."""
    losses = []

    # According to the TF2 guide, there are advantages to doing multiple
    # batches in the same tf.function call
    with tf.GradientTape() as tape:
      partial_loss = model(
          input_ids,
          labels=label_ids,
          training=True,
          return_dict=True).loss
      if isinstance(partial_loss, values.PerReplica):
        average_loss = strategy.reduce(
            tf.distribute.ReduceOp.MEAN, partial_loss, axis=None
        )
      else:
        average_loss = tf.math.reduce_mean(partial_loss)

      losses.append(average_loss)
      grads = tape.gradient(average_loss, model.trainable_variables)
      optimizer.apply_gradients(zip(grads, model.trainable_variables))

    if (FLAG_DISTRIBUTE_MODE.value ==
        constants.DistributeModeChoices.split_vertically):
      with tf.device("/job:localhost/replica:0/task:0/device:CPU:0"):
        return tf.math.reduce_mean(losses)
    else:
      return tf.math.reduce_mean(losses)
  return training_step

def build_manual_data_parallel_training_step(
    models,
    optimizer,
    tf_function_kwargs = None
):
  """Data parallel training step without using tf.distribute.Strategies."""

  tf_function_kwargs = {} if tf_function_kwargs is None else tf_function_kwargs
  train_vars = models[0].trainable_variables

  @tf.function(**tf_function_kwargs)
  def fn(input_ids, label_ids):
    losses = []
    accum_gradient = [tf.zeros_like(this_var) for this_var in train_vars]

    # Get some gradients per model instance
    for i in range(0, FLAG_BATCH_SIZE.value // FLAG_BATCH_SPLIT.value):
      with tf.GradientTape() as tape:
        start = i * FLAG_BATCH_SPLIT.value
        end = (i + 1) * FLAG_BATCH_SPLIT.value
        partial_loss = models[i](
            input_ids[start:end],
            labels=label_ids[start:end],
            training=True,
            return_dict=True
        ).loss
        gradients = tape.gradient(partial_loss, models[i].trainable_variables)
        accum_gradient = [acum_grad + grad for acum_grad, grad
                          in zip(accum_gradient, gradients)]
      losses.append(partial_loss)

    accum_gradient = [this_grad / len(models) for this_grad in accum_gradient]
    # Apply to first model
    optimizer.apply_gradients(
        zip(accum_gradient, models[0].trainable_variables)
        )

    with tf.device("/job:localhost/replica:0/task:0/device:CPU:0"):
      return tf.math.reduce_mean(losses)
  return fn

def build_evaluation_step(
    model,
    tf_function_kwargs = None,
):
  # Can't assign {} to the default value, as assigning mutable values to
  # default value is a bad practice, warned against by the linter
  tf_function_kwargs = {} if tf_function_kwargs is None else tf_function_kwargs

  @tf.function(**tf_function_kwargs)
  def fn(input_ids, label_ids):
    losses = []
    for i in range(0, FLAG_BATCH_SIZE.value // FLAG_BATCH_SPLIT.value):
      start = i * FLAG_BATCH_SPLIT.value
      end = (i + 1) * FLAG_BATCH_SPLIT.value
      loss = model(
          input_ids[start:end],
          labels=label_ids[start:end],
          training=False,
          return_dict=True).loss
      losses.append(loss)

    if (
        FLAG_DISTRIBUTE_MODE.value ==
        constants.DistributeModeChoices.split_vertically
    ):
      with tf.device("/job:localhost/replica:0/task:0/device:CPU:0"):
        return tf.math.reduce_mean(losses)
    else:
      return tf.math.reduce_mean(losses)
  return fn

def save_model(
    *,
    train_steps,
    model_or_replicas,
    instance_output_dir
):
  """Save the model and log the flags, locally, then copy over to GS."""
  with tempfile.TemporaryDirectory() as tmp:
    save_directory = os.path.join(
        tmp,
        time.strftime(f"{train_steps}_ckpt_%Y%m%d-%H%M%S")
    )
    model_or_replicas.save_pretrained(os.path.join(save_directory, "model"))
    subprocess.run(
        [
            "/root/google-cloud-sdk/bin/gsutil",
            "-m",
            "cp",
            "-R",
            save_directory,
            instance_output_dir,
        ],
        check=True
    )

def main(argv):
  #######################################################################
  # Initial Setup. Logging, Flags, Random seeds.
  #######################################################################
  if len(argv) > 1:
    raise app.UsageError("Too many command-line arguments.")
  absl_logging.use_python_logging()
  flags_dict = {
      flag.name: flag.value
      for flag in FLAGS.flags_by_module_dict()[argv[0]]
  }

  if FLAGS.use_subset:
    message = (f"{colorama.Back.RED}{colorama.Fore.WHITE}"
               f"{colorama.Style.BRIGHT}USING A SUBSET OF THE DATASET"
               f"{colorama.Style.RESET_ALL}")
    LOGGER.warning(
        message
    )

  utils.log_module_args(LOGGER, argv[0])
  if not FLAGS.output_dir.startswith("gs://"):
    utils.check_exists(FLAG_OUTPUT_DIR.value)
    if not tf.io.gfile.isdir(FLAG_OUTPUT_DIR.value):
      raise RuntimeError("Output dir needs to be a directory.")

  tf.random.set_seed(FLAG_RANDOM_SEED.value)
  np.random.seed(FLAG_RANDOM_SEED.value)

  # Prepare the instance output directory path and save the config there
  folder_name = time.strftime(
      f"{FLAG_RUN_NAME.value}_{FLAG_APPROACH_TYPE.value}_%Y%m%d-%H%M%S"
  )
  instance_output_dir = os.path.join(FLAG_OUTPUT_DIR.value, folder_name).strip()
  if not instance_output_dir.endswith("/"):
    instance_output_dir += "/"
  json_target = os.path.join(instance_output_dir, "training_params.json")
  if not json_target.strip().startswith("gs://"):
    subprocess.check_call(["mkdir", "-p", instance_output_dir])
  utils.to_json_file(json_target, instance_output_dir)

  ##############################################################################
  # Initialization and Configuration of the Devices.
  ##############################################################################
  tpu_setup = None
  # current_acelerator_type is always "CPU" in the beginning with TPUs
  if tf_utils.current_accelerator_type() == "CPU":
    tpu_setup = tf_utils.init_tpus()

  LOGGER.debug("Devices we are computing on:\n%s",
               utils.wrap_iterable(map(str, tf_utils.devices_to_use())))
  LOGGER.debug("All devices:")
  LOGGER.debug(tf_utils.device_mapping())

  if tf_utils.current_accelerator_type() == "GPU":
    tf.config.set_soft_device_placement(True)

  if tf_utils.current_accelerator_type() != "TPU":
    tf.debugging.set_log_device_placement(True)

  if FLAG_DISTRIBUTE_MODE.value in constants.PURE_DATA_PARALLEL_STRATEGIES:
    actual_num_replicas = len(tf_utils.devices_to_use())
  elif FLAG_DISTRIBUTE_MODE.value in constants.DATA_PARALLEL_DMC:
    actual_num_replicas = FLAG_NUM_REPLICAS.value
  else:
    actual_num_replicas = 1

  ##############################################################################
  # We load the retriever model if it is needed.
  ##############################################################################
  # Not currently used.

  retriever = None
  # if (FLAG_APPROACH_TYPE.value ==
  #     constants.ApproachTypeChoices.lm_and_realm):
  #   raise NotImplementedError("This part needs to be tested anew.")
    # config_path = FLAG_RETRIEVER_CONFIG_PATH.value
    # realm_save = tf_utils.REALMSave(**utils.from_json_file(config_path))
    #
    # # Approx 15 min when not in dev mode, on CPU
    # with utils.log_duration(LOGGER, "main",
    #                         "whole of BERTScaNNRetriever.__init__",
    #                         logging.INFO):
    #   scann_config = retrievers.ScannConfig(
    #       **utils.from_json_file(FLAG_SCANN_CONFIG_PATH.value))
    #   retriever = retrievers.BERTScaNNRetriever(
    #       retriever_module_path=realm_save.query_embedder_path,
    #       block_records_path=realm_save.text_records,
    #       num_block_records=realm_save.num_block_records,
    #       mode=tf.estimator.ModeKeys.EVAL,
    #       scann_config=scann_config)

  # elif (FLAG_APPROACH_TYPE.value ==
  #       constants.ApproachTypeChoices.cached_realm):
  #   raise NotImplementedError("This part needs to be tested anew.")
    # config_path = FLAG_RETRIEVER_CONFIG_PATH.value
    # realm_save = tf_utils.REALMSave(**utils.from_json_file(config_path))
    #
    # # Approx 15 min when not in dev mode, on CPU
    # with utils.log_duration(LOGGER, "main",
    #                         "whole of FullyCachedRetriever.__init__",
    #                         logging.INFO):
    #
    #   retriever = retrievers.FullyCachedRetriever(
    #       db_path=FLAG_FULLYCACHED_H5_PATH.value,
    #       block_records_path=realm_save.text_records,
    #       num_block_records=realm_save.num_block_records,
    #       )

  ##############################################################################
  # Distributed training task
  ##############################################################################
  if FLAG_TASK.value == constants.TaskChoices.train:
    with utils.log_duration(LOGGER, "main", "Load model"):
      utils.print_mem("before loading model", LOGGER)
      model_specific = task_specific.load_model(FLAG_MODEL_LOAD_PATH.value,
                                                FLAG_MODEL_KEY.value,
                                                FLAG_DISTRIBUTE_MODE.value,
                                                tpu_setup,
                                                FLAG_NUM_REPLICAS.value)
      utils.print_mem("after loading model", LOGGER)
      model_or_replicas = model_specific.model
      if isinstance(model_or_replicas, list):
        model_or_replicas: List[transformers.TFGPT2LMHeadModel]
      else:
        model_or_replicas: transformers.TFGPT2LMHeadModel

      tokenizer = model_specific.tokenizer

      def make_optimizer():
        return tensor2tensor.utils.adafactor.AdafactorOptimizer(
            learning_rate=FLAG_LEARNING_RATE.value)

      if model_specific.strategy:
        with model_specific.strategy.scope():
          optimizer = make_optimizer()
      else:
        optimizer = make_optimizer()

    ############################################################################
    # Prepare the dataset functions
    ############################################################################
    rg = np.random.default_rng(FLAG_RANDOM_SEED.value)

    def call_lm_preproc(
        repeat,
        split,
        random_seed
    ):
      """Using functools.partial prevents the linter from doing its job."""
      if FLAG_DATASET_NAME.value == constants.DatasetNameChoices.kilt_eli5:
        return task_specific.create_lm_ds_kilt_eli5(
            tokenizer=tokenizer,
            context_window_size=(
                model_or_replicas[0].config.n_positions
                if isinstance(model_or_replicas, list)
                else model_or_replicas.config.n_positions
            ),
            dataset_name=FLAG_DATASET_NAME.value,
            # Batches are split over the replicas:
            batch_size=FLAG_BATCH_SIZE.value * actual_num_replicas,
            db_path=FLAG_DB_PATH.value,
            random_seed=random_seed,
            use_subset=FLAG_USE_SUBSET.value,
            subset_size=FLAG_SUBSET_SIZE.value,
            use_helper_words=FLAG_USE_HELPER_WORDS.value,
            approach_type=FLAG_APPROACH_TYPE.value,
            num_retrievals=FLAG_NUM_RETRIEVALS.value,
            retrieval_temperature=FLAG_RETRIEVAL_TEMPERATURE.value,
            retriever=retriever,
            repeat=repeat,
            split=split,
            enable_debug_checks=FLAG_DATASET_DEBUG.value,
            retrieval_bank_size=FLAG_RETRIEVAL_BANK_SIZE.value,
            dataset_type=FLAG_DATASET_TYPE.value,
            qty_shuffle=FLAG_QTY_SHUFFLE.value,
            tfr_prefix=FLAG_TFR_PREFIX.value,
            max_length_generation=FLAG_MAX_LENGTH_GENERATION.value,
        )
      else:
        raise NotImplementedError(
            f"FLAG_DATASET_NAME.value unsupported: `{FLAG_DATASET_NAME.value}`"
        )

    make_training_dataset: Callable[Ellipsis, tf.data.Dataset] = functools.partial(
        call_lm_preproc,
        split="train",
        repeat=False,
    )
    make_eval_dataset: Callable[Ellipsis, tf.data.Dataset] = functools.partial(
        call_lm_preproc,
        split="eval",
        repeat=True,
    )

    ############################################################################
    # Prepare the step functions
    ############################################################################
    utils.check_contained(
        FLAG_DISTRIBUTE_MODE.value, constants.DistributeModeChoices.choices()
    )
    tf_function_flags = dict(
        experimental_compile=FLAG_EXPERIMENTAL_COMPILE.value,
        reduce_retracing=not FLAG_INPUT_FIXED_SIZE.value
    )

    if (FLAG_DISTRIBUTE_MODE.value ==
        constants.DistributeModeChoices.split_and_data_parallel):
      if not isinstance(model_or_replicas, list):
        raise RuntimeError(type(model_or_replicas))
      training_step = build_manual_data_parallel_training_step(
          model_or_replicas, optimizer, tf_function_flags
      )

    else:
      training_step = build_regular_training_step(
          model_or_replicas,
          optimizer,
          strategy=model_specific.strategy,
          tf_function_kwargs=tf_function_flags
      )

    evaluation_step = build_evaluation_step(
        model_or_replicas, tf_function_flags
    )

    secs_since_last_ckpt = time.time()
    # Model checkpoints are saved to the tmp_directory and then rsynced to GCS
    ##########################################################################
    # Prepare the different logging facilities
    ##########################################################################
    train_log_dir = os.path.join(instance_output_dir, "tensorboard", "train")
    eval_log_dir = os.path.join(instance_output_dir, "tensorboard", "eval")
    flags_log_dir = os.path.join(instance_output_dir, "tensorboard", "params")
    writers = dict(
        train=tf.summary.create_file_writer(train_log_dir),
        eval=tf.summary.create_file_writer(eval_log_dir),
        flags=tf.summary.create_file_writer(flags_log_dir)
    )
    with writers["flags"].as_default():
      tf.summary.text(
          "Flags",
          # Tensorboard takes Markdown:
          json.dumps(flags_dict, indent=4).replace("\n", "\n\n"),
          step=0
          )

    ma_loss = dict(
        train=utils.MovingAverage(0.9),
        eval=utils.MovingAverage(0.9)
        )
    step_counters = dict(train=0, eval=0)
    batch_counters = dict(train=0, eval=0)
    prev_batch_end = time.time()

    # The eval ds has no real concept of epoch, repeats forever, shuffling
    # each time it reaches its end
    with utils.log_duration(LOGGER, "main", "All of make_eval_dataset"):
      eval_ds_instance = make_eval_dataset(
          random_seed=rg.integers(-2**63, 2**63 - 1),
      )
    LOGGER.debug("Distributing the eval dataset to the replicas.")
    if FLAG_DATASET_TYPE.value == "tfr":
      eval_ds_instance = (
          model_specific.strategy.experimental_distribute_dataset(
              eval_ds_instance
          )
      )

    LOGGER.debug("Done distributing the eval dataset to the replcias.")
    eval_ds_instance = iter(eval_ds_instance)

    ##########################################################################
    # Training Loop
    ##########################################################################
    for epoch in itertools.count():
      ####################################################################
      # Epoch Setup
      ####################################################################
      LOGGER.debug("EPOCH %d START", epoch)
      # Shuffle differently every epoch
      with utils.log_duration(
          LOGGER, "main", "All of make_training_dataset"
      ):
        train_ds_instance = make_training_dataset(
            random_seed=rg.integers(-2**63, 2**63 - 1),
        )
      LOGGER.debug(
          "Attempting to distribute the training dataset to the replicas."
      )
      if FLAG_DATASET_TYPE.value == "tfr":
        train_ds_instance = (
            model_specific.strategy.experimental_distribute_dataset(
                train_ds_instance
            )
        )

      LOGGER.debug(
          "Done distributing the training dataset to the replicas."
      )
      train_ds_instance = iter(train_ds_instance)

      # This allows us to see if we reached the end of the training iterator,
      # in which case "did_at_least_one_training_batch == False".
      # We could also test that it did all the batches, to similar results.
      did_at_least_one_training_batch = True
      split = "eval"
      while did_at_least_one_training_batch:
        # Invert split
        if split == "train":
          split = "eval"
        else:
          split = "train"

        # Prepare to test if we did at least one training batch
        if split == "train":
          did_at_least_one_training_batch = False

        if split == "train":
          dataset_iterator = itertools.islice(
              train_ds_instance, FLAG_BATCHES_BETWEEN_EVALS.value
          )
        else:
          # The evaluation DS is tiny, so we reshuffle and take a random
          dataset_iterator = itertools.islice(
              eval_ds_instance, FLAG_NUMBER_EVAL_BATCHES.value
          )

        LOGGER.debug("Batching")
        for batch in dataset_iterator:
          # LOGGER.debug("Input sentence:\n\"%s\"",
          #              tokenizer.decode([x for x in batch["input_ids"][0]
          #                                if x != tokenizer.eos_token_id]))
          # LOGGER.debug("Label:\n\"%s\"",
          #              tokenizer.decode([(x if x != -100 else 0)
          #                                for x in batch["label_ids"][0]]))

          if FLAG_DATASET_TYPE.value != "tfr":
            batch = (
                model_specific.strategy
                .experimental_distribute_values_from_function(
                    tf_utils.make_dict_distribute_fn(batch)
                ))

          # We only care about training epochs as, obviously, we don't train
          # over eval samples; the number of  eval samples seen only
          # contributes to lowering the variance in the evaluation of when to
          # do early stopping.
          if split == "train":
            did_at_least_one_training_batch = True

          input_ids = batch["input_ids"]
          label_ids = batch["label_ids"]

          ####################################################################
          # Training Step
          ####################################################################
          step_counters[split] += (
              FLAG_BATCH_SIZE.value * actual_num_replicas
          )

          if split == "train":
            batch_counters[split] += 1
            training_kwargs = dict(
                input_ids=input_ids,
                label_ids=label_ids,
            )

            if model_specific.strategy:
              utils.print_mem("before running", LOGGER)

              LOGGER.debug("Training, Calling strategy.run")
              loss = model_specific.strategy.run(
                  training_step,
                  kwargs=training_kwargs
              )
              LOGGER.debug("Training, Done with strategy.run")
              utils.print_mem("after running", LOGGER)

            else:
              loss = training_step(**training_kwargs)  # pytype: disable=wrong-arg-count
              # If we are in the strategy-free data parallel mode, we need
              # to change the weights of all replicas to those of the model at
              # index 0
              if (
                  FLAG_DISTRIBUTE_MODE.value ==
                  constants.DistributeModeChoices.split_and_data_parallel
              ):
                for replica in model_or_replicas[1:]:
                  replica.set_weights(model_or_replicas[0].get_weights())

          ####################################################################
          # Evaluation Step
          ####################################################################
          elif split == "eval":
            evaluation_kwargs = dict(
                input_ids=input_ids,
                label_ids=label_ids,
            )

            if model_specific.strategy:
              loss = model_specific.strategy.run(
                  evaluation_step,
                  kwargs=evaluation_kwargs
              )
            else:
              loss = evaluation_step(**evaluation_kwargs)
          else:
            raise ValueError(f"Unexpected value for split: {split}")

          ####################################################################
          # Logging
          ####################################################################
          if (FLAG_DISTRIBUTE_MODE.value in
              constants.PURE_DATA_PARALLEL_STRATEGIES):
            utils.check_equal(len(loss.values), actual_num_replicas)
            LOGGER.debug("Split: %s", split)
            LOGGER.debug("Real num replicas: %s", actual_num_replicas)
            LOGGER.debug("Loss: %s", loss)
            LOGGER.debug("Loss values: %s", loss.values)

            average_loss = float(tf.math.reduce_mean(loss.values).numpy())
          else:
            average_loss = float(loss.numpy())

          # tf.debugging.check_numerics(loss)
          now = time.time()
          batch_duration = now - prev_batch_end
          prev_batch_end = now
          ma_loss[split].update(average_loss)

          # Actual logging
          LOGGER.info("Epoch: # %d", epoch)
          LOGGER.info("Tensorboard_dir: %s", instance_output_dir)
          LOGGER.info("Batch: %s # %d", split, batch_counters[split])
          LOGGER.info("Step: %s # %d", split, step_counters[split])
          if FLAG_USE_SUBSET.value:
            LOGGER.warning(">> USING A SUBSET OF THE DATASET <<")
          LOGGER.info(
              "%(split)s Batch loss:           %(metric)f",
              dict(split=split, metric=average_loss)
          )
          LOGGER.info(
              "%(split)s Moving average loss:  %(metric)f",
              dict(split=split, metric=ma_loss[split].average)
          )
          LOGGER.info(
              "%(split)s Moving average ppl:   %(metric)f",
              dict(split=split, metric=np.exp(ma_loss[split].average))
          )
          LOGGER.info(
              "%(split)s Batch duration:       %(duration)s",
              dict(
                  split=split,
                  duration=utils.TimeStamp.from_seconds(
                      batch_duration).format()
              )
          )
          if FLAG_DISTRIBUTE_MODE.value in constants.DATA_PARALLEL_DMC:
            LOGGER.info(
                "%(split)s Duration per sample:  %(duration)s",
                dict(
                    split=split,
                    duration=utils.TimeStamp.from_seconds(
                        batch_duration / (
                            FLAG_BATCH_SIZE.value * actual_num_replicas
                        )
                    )
                )
            )

          # Write to Tensorboard
          with writers[split].as_default():
            tf.summary.scalar(
                f"Loss/{split}", average_loss, step_counters[split]
            )
            tf.summary.scalar(
                f"PPL/{split}", np.exp(average_loss), step_counters[split]
            )
          writers[split].flush()

          # Save every 5 min
          if (time.time() - secs_since_last_ckpt) / (60 * 20) >= 1:
            secs_since_last_ckpt = time.time()
            save_model(
                train_steps=step_counters["train"],
                model_or_replicas=model_or_replicas,
                instance_output_dir=instance_output_dir
            )

        secs_since_last_ckpt = time.time()
        save_model(
            train_steps=step_counters["train"],
            model_or_replicas=model_or_replicas,
            instance_output_dir=instance_output_dir
        )
    #############################################################
    # Post Training Cleanup
    #######################################################################
    for writer in writers.values():
      writer.close()

if __name__ == "__main__":
  app.run(main)


text:
namespace CompositePattern
{
    public abstract class PersonComponent
    {
        protected PersonComponent(string name)
        {
            this.Name = name;
        }

        protected string Name { get; private set; }

        public abstract void Display(int depth);
    }
}

redpajama_set_name: RedPajamaGithub


text:
<!doctype html>

	<meta charset="utf-8">
	<title>jQuery UI Datepicker Test Suite</title>

	<link rel="stylesheet" href="../../../external/qunit.css">

jQuery UI Datepicker Test Suite

<input type="text" id="inp"><input type="text" id="alt">

<input type="text" id="inp2">


text:
I/O Docs - Open Source in Node.js
=================================
Copyright 2011 Mashery, Inc.

[http://www.mashery.com](http://www.mashery.com)

[http://developer.mashery.com](http://developer.mashery.com)

SYNOPSIS
--------
I/O Docs is a live interactive documentation system for RESTful web APIs. By defining APIs at the resource, method and parameter
levels in a JSON schema, I/O Docs will generate a JavaScript client interface. API calls can be executed from this interface, which are then proxied through the I/O Docs server with payload data cleanly formatted (pretty-printed if JSON or XML).

You can find the latest version here: [https://github.com/mashery/iodocs](https://github.com/mashery/iodocs)
You can find a demo of it running here: [http://iodocs.demo.mashery.com](http://iodocs.demo.mashery.com)

However, we recommend that you install I/O Docs with *npm*, the Node package manager. See instructions below.

BUILD/RUNTIME DEPENDENCIES
--------------------------
1. Node.js - server-side JS engine
2. npm - node package manager
3. Redis - key+value storage engine

Note: Node and some of the modules require compiler (like gcc). If you are on a Mac, you will need to install XCode. If you're on Linux, you'll need to install build-essentials, or something equivalent.

INSTALLATION INSTRUCTIONS FOR NODE, NPM & REDIS
-----------------------------------------------
1. Node.js - [https://github.com/joyent/node/wiki/Installation](https://github.com/joyent/node/wiki/Installation)
2. npm (Node package manager) - [https://github.com/isaacs/npm](https://github.com/isaacs/npm)
3. Redis - [http://redis.io/download](http://redis.io/download)

INSTALLATION INSTRUCTIONS FOR I/O DOCS
--------------------------------------
From the command line type in:
<pre>  git clone http://github.com/mashery/iodocs.git
  cd iodocs
  npm install
</pre>

### Node Module Dependencies
These will be automatically installed when you use any of the above *npm* installation methods above.

1. [express](http://expressjs.com/) - framework
2. [oauth](https://github.com/unscene/node-oauth) - oauth library
3. [redis](https://github.com/mranney/node_redis) - connector to Redis
3. [connect-redis](https://github.com/visionmedia/connect-redis) - Redis session store
4. [hashlib](https://github.com/brainfucker/hashlib) - used for signatures
5. [querystring](https://github.com/visionmedia/node-querystring) - used to parse query string
6. [jade](http://jade-lang.com/) - the view engine

RUNNING I/O DOCS
----------------
1. You will need to copy *config.json.sample* to *config.json*. The defaults will work, but feel free to change them.
2. node ./app.js
3. Point your browser to: [http://localhost:3000](http://localhost:3000)

QUICK API CONFIGURATION EXAMPLE
-------------------------------
Adding an API to the I/O Docs configuration is relatively simple.

First, append the new top-level service information to the `./public/data/apiconfig.json` file.

Example:

```js
"lowercaseapi": {
    "name": "Lower Case API",
    "protocol": "http",
    "baseURL": "api.lowercase.sample.com",
    "publicPath": "/v1",
    "auth": "key",
    "keyParam": "api_key_var_name"
}
```

Add the file `./public/data/lowercaseapi.json` to define the API.

Example:

```js
{
    "endpoints": [
        {
            "name": "Resource Group A",
            "methods": [
                {
                    "MethodName": "Method A1",
                    "Synopsis": "Grabs information from the A1 data set",
                    "HTTPMethod": "GET",
                    "URI": "/a1/grab",
                    "RequiresOAuth": "N",
                    "parameters": [
                        {
                            "Name": "param_1_name",
                            "Required": "Y",
                            "Default": "",
                            "Type": "string",
                            "Description": "Description of the first parameter."
                        }
                    ]
                }
            ]
        }
    ]
}
```

TOP-LEVEL SERVICE CONFIG DETAILS - apiconfig.json
-------------------------------------------------
The *apiconfig.json* file contains high-level information about an API.

### Example #1 - Explanation of each field in an example API config that uses basic key authentication:

```js
"lower": {
   "name": "My API",
   "protocol": "http",
   "baseURL": "api.lowercase.sample.com",
   "publicPath": "/v1",
   "auth": "key",
   "keyParam": "api_key_var_name"
}
```

Line:

1. Handle of the API. It is used to pull up the client
   interface in the URL:

    Ex: http://127.0.0.1:3000/lower

2. "name" key value is a string that holds the name
    of the API that is used in the Jade template output.

3. "protocol" key value is either *http* or *https*

4. "baseURL" key value is the host name of
    the API calls (should not include protocol)

5. "publicPath" key value is the full path prefix prepended
    to all method URIs. This value often includes the version
    in RESTful APIs.

    Ex: "/v1"

    In the Example #2 below, there is also "privatePath"
    which is used for endpoints behind protected resources.

6. "auth" key value is the auth method. Valid values can be:

         "key" - simple API key in the URI
         "oauth1" - OAuth 1.0/1.0a
         "" - no authentication

7. "keyParam" key value is name of the query parameter that
    is added to an API request when the "auth" key value from
    (5) is set to "key"

8. Closing curly-bracket ;)

---

### Example #2 - Twitter API config that uses 3-legged OAuth

```js
"twitter": {
    "name": "Twitter API",
    "protocol": "http",
    "baseURL": "api.twitter.com",
    "publicPath": "/1",
    "privatePath": "/1",
    "booleanTrueVal": "true",
    "booleanFalseVal": "false",
    "auth": "oauth",
    "oauth" : {
       "type": "three-legged",
       "requestURL": "https://api.twitter.com/oauth/request_token",
       "signinURL": "https://api.twitter.com/oauth/authorize?oauth_token="
       "accessURL": "https://api.twitter.com/oauth/access_token",
       "version": "1.0",
       "crypt": "HMAC-SHA1",
   }
   "keyParam": "",
}
```

Line:

1. Handle of the API. It is used to pull up the client
    interface in the URL:

    Ex: http://127.0.0.1:3000/twitter

2. "name" key value is a string that holds the name
    of the API that is used in the Jade template output.

3. "protocol" key value contains either *http* or *https*,
    but you're welcome to try other protocols.

4. "baseURL" key value is the base URL that accepts
    the API calls (should not include protocol)

5. "publicPath" key value is the path prefix prepended
    to all method URIs for non-protected method resources.
    This value often includes the version in RESTful APIs.

    Ex: "/v1", "/1", etc.

6. "privatePath" key value is the path prefix prepended
    to all method URIs for OAuth protected method resources.
    This value is most often the version in RESTful APIs.

    Ex: "/v1", "/1", etc.

7. "booleanTrueVal" key value is the default value for
    true Boolean values that are sent in API requests.
    Some APIs are designed to accept a wide variety
    of true derivatives, but some are very strict about
    this value.

    Ex: "true", "TRUE", "True", "t", "T", "1", etc.
    Default: "true"

8. "booleanFalseVal" key value is the default value for
    false Boolean values that are sent in API requests.
    Some APIs are designed to accept a wide variety
    of false derivatives, but some are very strict about
    this value.

    Ex: "false", "FALSE", "False", "f", "F", "0", etc.
    Default: "false"

9. "auth" key value is set to "oauth" when OAuth is the
    authentication mechanism. Field is required.

10. "oauth" key value is a JSON object that contains the
    OAuth implementation details for this API. Field is
    required when "auth" value is "oauth".

11. "type" key value is the OAuth is the authorization flow
     used for this API. Valid values are "three-legged" (normal
     authorization flow) and "two-legged" (no authorization flow).

12. "requestURL" key value is the Request Token URL used in
    the OAuth dance (used in *three-legged* scenario).

13. "signinURL" key value is the User Authorization URL used
    in the OAuth dance (where the user is redirected to provide
    their credentials -- used in *three-legged* scenario).

14. "accessURL" key value is the Access Token URL used in the
    OAuth dance (used in *three-legged* scenario).

15. "version" key value is the OAuth version. As of I/O Docs v1.1,
    "1.0" is the only supported version. Note: use "1.0" for both
    1.0 and 1.0A implementations.

16. "crypt" key value is the OAuth signature method. As of I/O Docs
    v1.1 "HMAC-SHA1" is the only supported signing method.

17. Closing curly bracket for "oauth" JSON object.

18. "keyParam" key value is blank when OAuth is the authentication
    method.

19. Closing curly bracket for main object.

API-LEVEL CONFIG DETAILS
========================
For every API that is configured in *apiconfig.json* a JSON config file must exist.
You should look at the *./public/data/* directory for examples.

### Example #1 - Explanation of each field in an example API-level configuration

```js
{
   "name":"User Resources",
   "methods":[
      {
        "MethodName":"users/show",
         "Synopsis":"Returns extended user information",
         "HTTPMethod":"GET",
         "URI":"/users/show.json",
         "RequiresOAuth":"N",
         "parameters":[
             {
                "Name":"user_id",
                "Required":"Y",
                "Default":"",
                "Type":"string",
                "Description":"The ID of the user",
             },
             {
                "Name":"cereal",
                "Required":"Y",
                "Default":"fruitscoops",
                "Type":"enumerated",
                "EnumeratedList": [
                    "fruitscoops",
                    "sugarbombs",
                    "frostedteeth"
                   ],
                "Description":"The type of cereal desired"
             },
             {
                "Name":"skip_status",
                "Required":"N",
                "Default":"",
                "Type":"boolean",
                "Description":"If true, status not included"
             }
        ]
    }]
}
```

Line:

3. "name" key holds the value of the Resource name. Methods are grouped into Resources.

4. "methods" key value is an array of JSON objects (each one being a method)

6. "MethodName" key value is a string that is displayed via the view template.

7. "Synopsis" key value is a short description of the method.

8. "HTTPMethod" key value can be either GET, POST, DELETE or PUT (all caps)

9. "URI" key value is the path to the method that is appended to the *baseURL* and the public/private path.

10. "RequiresOAuth" key value is either Y or N. If Y, the *privatePath* is used from the top-level config. If N, the *publicPath* is used from the top-level config.

11. "parameters" key value is an array of JSON objects (each one being a parameter)

13. "Name" key value is a string that contains the name of the parameter.

14. "Required" key value is either Y or N. If Y, the parameter will be output as bold.

15. "Default" key value is a string, containing a default value that will be automatically populated onto the form.

16. "Type" key value can be an arbitrary string that describes the variable type; however, the value is *boolean* or *enumerated* a drop-down (select) box will appear.

17. "Description" key value is a string, containing the description of the parameter.

23. "Type" key value is set to *enumerated* for this parameter.

24. "EnumeratedList" key value is an array of enumerated values that will render a drop-down (select box) on the form.

25. Each value in the list is a string.

35. "Type" key value is *boolean* that will render a drop-down (select box) on the form for *true* and *false*.

SUPPORT
=======
If you need any help with I/O Docs, you can reach out to us via the GitHub Issues page at:
<code>[http://github.com/mashery/iodocs/issues](http://github.com/mashery/iodocs/issues)</code>


text:
<?xml version="1.0" encoding="utf-8"?>
<selector xmlns:android="http://schemas.android.com/apk/res/android" >
    <item android:state_pressed="true">
        <shape android:shape="rectangle">
            <solid android:color="#000000"/>
            <padding
                android:left="2dp"
                android:top="2dp"
                android:right="2dp"
                android:bottom="2dp"/>
            <stroke
                android:width="2dp"
                android:color="#8E8E8E"/>
        </shape>
    </item>
</selector>


text:
package org.omg.PortableInterceptor;

/**
* org/omg/PortableInterceptor/ServerIdHelper.java .
* Generated by the IDL-to-Java compiler (portable), version "3.2"
* from c:/re/workspace/8-2-build-windows-amd64-cygwin/jdk8u72/5732/corba/src/share/classes/org/omg/PortableInterceptor/Interceptors.idl
* Tuesday, December 22, 2015 7:17:38 PM PST
*/

// Should be type string
abstract public class ServerIdHelper
{
  private static String  _id = "IDL:omg.org/PortableInterceptor/ServerId:1.0";

  public static void insert (org.omg.CORBA.Any a, String that)
  {
    org.omg.CORBA.portable.OutputStream out = a.create_output_stream ();
    a.type (type ());
    write (out, that);
    a.read_value (out.create_input_stream (), type ());
  }

  public static String extract (org.omg.CORBA.Any a)
  {
    return read (a.create_input_stream ());
  }

  private static org.omg.CORBA.TypeCode __typeCode = null;
  synchronized public static org.omg.CORBA.TypeCode type ()
  {
    if (__typeCode == null)
    {
      __typeCode = org.omg.CORBA.ORB.init ().create_string_tc (0);
      __typeCode = org.omg.CORBA.ORB.init ().create_alias_tc (org.omg.PortableInterceptor.ServerIdHelper.id (), "ServerId", __typeCode);
    }
    return __typeCode;
  }

  public static String id ()
  {
    return _id;
  }

  public static String read (org.omg.CORBA.portable.InputStream istream)
  {
    String value = null;
    value = istream.read_string ();
    return value;
  }

  public static void write (org.omg.CORBA.portable.OutputStream ostream, String value)
  {
    ostream.write_string (value);
  }

}


text:
/***************************************************************************\
    Tpost.h
    Scott Randolph
    August 21, 1995

    Most basic element of the terrain map.  Posts contain all the
    information about a point on the ground.  There may in the future be
    multiple versions of posts (ie: textured vs. not textured).  For now,
    only one.

 This is version that is used at runtime in memory.
\***************************************************************************/
#ifndef _TPOST_H_
#define _TPOST_H_

#include "TerrTex.h"
#include "FarTex.h"
#include "Ttypes.h"

typedef struct Tpost
{

    // X can be obtained from:   x = LEVEL_POST_TO_WORLD( levelRow, LOD );
    // Y can be obtained from:   y = LEVEL_POST_TO_WORLD( levelCol, LOD );
    float z; // Units of floating point feet -- positive z axis points DOWN

    // Color information
    int colorIndex; // (could shrink to a byte)

    // Texture information
    float u; // u is East/West
    float v; // v is North/South
    float d;
    TextureID texID; // 16 bit value (see TerrTex.h)

    // normal information
    float theta;
    float phi;

} Tpost;

#endif // _TPOST_H_


text:
#include "pebble.h"
#include <stdint.h>
#include <string.h>
#include "config.h"
#ifdef ENABLE_LOCALIZE
  #include "localize.h"
#endif
#include "pebblebike.h"
#include "communication.h"
#ifdef PBL_HEALTH
  #include "health.h"
#endif
#include "heartrate.h"
#include "buttons.h"
#include "menu.h"
#include "screens.h"
#include "screen_data.h"
#ifdef ENABLE_FUNCTION_LIVE
  #include "screen_live.h"
#endif
#include "screen_map.h"
#include "screen_config.h"
#include "graph.h"

#ifdef PBL_PLATFORM_CHALK
GFont font_roboto_bold_16;
#endif
GFont font_roboto_bold_62;

AppData s_data;
GPSData s_gpsdata;
#ifdef ENABLE_FUNCTION_LIVE
LiveData s_live;
#endif
bool title_instead_of_units = true;
GColor bg_color_data_main;
int nbchange_state=0;

void change_units(uint8_t units, bool first_time) {
  if ((units == s_gpsdata.units) && !first_time) {
    return;
  }

  if (first_time) {
    if (persist_exists(PERSIST_UNITS_KEY)) {
      units = persist_read_int(PERSIST_UNITS_KEY);
    } else {
      // default value
      units = UNITS_METRIC;
    }
  } else {
    // save new value
    persist_write_int(PERSIST_UNITS_KEY, units);
  }

  s_gpsdata.units = units;
  switch(s_gpsdata.units) {
    case UNITS_IMPERIAL:
      strncpy(s_data.unitsSpeed, SPEED_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsDistance, DISTANCE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsAltitude, ALTITUDE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsAscentRate, ASCENT_RATE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsTemperature, TEMPERATURE_UNIT_IMPERIAL, 8);
      break;
    case UNITS_METRIC:
      strncpy(s_data.unitsSpeed, SPEED_UNIT_METRIC, 8);
      strncpy(s_data.unitsDistance, DISTANCE_UNIT_METRIC, 8);
      strncpy(s_data.unitsAltitude, ALTITUDE_UNIT_METRIC, 8);
      strncpy(s_data.unitsAscentRate, ASCENT_RATE_UNIT_METRIC, 8);
      strncpy(s_data.unitsTemperature, TEMPERATURE_UNIT_METRIC, 8);
      break;
    case UNITS_NAUTICAL_IMPERIAL:
      strncpy(s_data.unitsSpeed, SPEED_UNIT_NAUTICAL, 8);
      strncpy(s_data.unitsDistance, DISTANCE_UNIT_NAUTICAL, 8);
      strncpy(s_data.unitsAltitude, ALTITUDE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsAscentRate, ASCENT_RATE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsTemperature, TEMPERATURE_UNIT_IMPERIAL, 8);
      break;
    case UNITS_NAUTICAL_METRIC:
      strncpy(s_data.unitsSpeed, SPEED_UNIT_NAUTICAL, 8);
      strncpy(s_data.unitsDistance, DISTANCE_UNIT_NAUTICAL, 8);
      strncpy(s_data.unitsAltitude, ALTITUDE_UNIT_METRIC, 8);
      strncpy(s_data.unitsAscentRate, ASCENT_RATE_UNIT_METRIC, 8);
      strncpy(s_data.unitsTemperature, TEMPERATURE_UNIT_METRIC, 8);
      break;
    case UNITS_RUNNING_IMPERIAL:
      strncpy(s_data.unitsSpeed, SPEED_UNIT_RUNNING_IMPERIAL, 8);
      strncpy(s_data.unitsDistance, DISTANCE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsAltitude, ALTITUDE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsAscentRate, ASCENT_RATE_UNIT_IMPERIAL, 8);
      strncpy(s_data.unitsTemperature, TEMPERATURE_UNIT_IMPERIAL, 8);
      break;
    case UNITS_RUNNING_METRIC:
      strncpy(s_data.unitsSpeed, SPEED_UNIT_RUNNING_METRIC, 8);
      strncpy(s_data.unitsDistance, DISTANCE_UNIT_METRIC, 8);
      strncpy(s_data.unitsAltitude, ALTITUDE_UNIT_METRIC, 8);
      strncpy(s_data.unitsAscentRate, ASCENT_RATE_UNIT_METRIC, 8);
      strncpy(s_data.unitsTemperature, TEMPERATURE_UNIT_METRIC, 8);
      break;
  }

  if (!first_time) {
    //todo(custom) screen_data_dirty
    layer_mark_dirty(text_layer_get_layer(s_data.screenData_layer.field_top.unit_layer));
    layer_mark_dirty(text_layer_get_layer(s_data.screenData_layer.field_top2.unit_layer));
    layer_mark_dirty(text_layer_get_layer(s_data.screenData_layer.field_bottom_left.unit_layer));
    layer_mark_dirty(text_layer_get_layer(s_data.screenData_layer.field_bottom_right.unit_layer));
  }
  if (s_data.data_subpage != SUBPAGE_UNDEF) {
    strncpy(s_data.unitsSpeedOrHeartRate, s_data.unitsSpeed, 8);
  }
}

void change_state(uint8_t state) {
  if (state == s_data.state) {
    return;
  }
  s_data.state = state;
  if (s_data.state == STATE_STOP) {
    screen_reset_instant_data();
  }
  layer_set_hidden(text_layer_get_layer(s_data.topbar_layer.accuracy_layer), s_data.state == STATE_STOP);
  buttons_update();

  nbchange_state++;
}

static void handle_tick(struct tm *tick_time, TimeUnits units_changed) {

  char *time_format;
  if (clock_is_24h_style()) {
    time_format = "%R";
  } else {
    time_format = "%I:%M";
  }

  strftime(s_data.time, sizeof(s_data.time), time_format, tick_time);

  // Kludge to handle lack of non-padded hour format string
  // for twelve hour clock.
  if (!clock_is_24h_style() && (s_data.time[0] == '0')) {
    memmove(s_data.time, &s_data.time[1], sizeof(s_data.time) - 1);
  }
  //layer_mark_dirty(s_data.topbar_layer.layer);
  update_screens();
}

void bt_callback(bool connected) {
  topbar_toggle_bluetooth_icon(connected);
}

static void init(void) {
  config_load();

#ifdef ENABLE_LOCALIZE
  locale_init();
#endif
  heartrate_init();
  s_gpsdata.heartrate = 255; // no data at startup
  bg_color_data_main = BG_COLOR_DATA_MAIN;

#ifdef ENABLE_NAVIGATION_FULL
  for (uint16_t i = 0; i < NAV_NB_POINTS_STORAGE - 1; i++) {
    s_gpsdata.nav_xpos[i] = s_gpsdata.nav_ypos[i] = INT16_MAX;
  }
#endif

#ifdef ENABLE_DEBUG_FIELDS_SIZE
  strcpy(s_data.speed, "188.8");
  strcpy(s_data.distance, "88.8");
  strcpy(s_data.avgspeed, "888.8");
  strcpy(s_data.altitude, "888.8");
  strcpy(s_data.ascent, "1342");
  strcpy(s_data.ascentrate, "548");
  strcpy(s_data.slope, "5");
  strcpy(s_data.accuracy, "9");
  strcpy(s_data.bearing, "270");
  strcpy(s_data.elapsedtime, "1:05:28");
  strcpy(s_data.maxspeed, "25.3");
  strcpy(s_data.heartrate, "128");
  strcpy(s_data.cadence, "90");
#endif
#ifdef ENABLE_DEMO
  strcpy(s_data.maxspeed, "26.1");
  strcpy(s_data.distance, "2.0");
  strcpy(s_data.avgspeed, "14.0");
  strcpy(s_data.altitude, "1139");
  strcpy(s_data.accuracy, "4");
  strcpy(s_data.steps, "7548");
  strcpy(s_data.elapsedtime, "1:15:28");
  strcpy(s_data.heartrate, "154");
  s_gpsdata.heartrate = 154;
  heartrate_new_data(s_gpsdata.heartrate);
  s_data.live = 1;
  s_data.state = STATE_START;

  s_gpsdata.nav_distance_to_destination100 = 12100;
  s_gpsdata.nav_next_distance1000 = 1345;
  s_gpsdata.nav_error1000 = 55;
  snprintf(s_data.nav_next_distance,   sizeof(s_data.nav_next_distance),   "%d",   s_gpsdata.nav_next_distance1000);
#else
  strcpy(s_data.speed, "0.0");
  strcpy(s_data.distance, "-");
  strcpy(s_data.avgspeed, "-");
  strcpy(s_data.altitude, "-");
  strcpy(s_data.ascent, "-");
  strcpy(s_data.ascentrate, "-");
  strcpy(s_data.slope, "-");
  strcpy(s_data.accuracy, "-");
  strcpy(s_data.bearing, "-");
  strcpy(s_data.elapsedtime, "00:00:00");
  strcpy(s_data.maxspeed, "-");
  strcpy(s_data.heartrate, "-");
  strcpy(s_data.cadence, "-");
  strcpy(s_data.nav_next_distance, "-");
  strcpy(s_data.nav_distance_to_destination, "-");
  strcpy(s_data.nav_ttd, "-");
  strcpy(s_data.nav_eta, "-");
#endif
  //strcpy(s_data.lat, "-");
  //strcpy(s_data.lon, "-");
  //strcpy(s_data.nbascent, "-");

  s_data.phone_battery_level = -1;

#ifdef PBL_PLATFORM_CHALK
  font_roboto_bold_16 = fonts_load_custom_font(resource_get_handle(RESOURCE_ID_FONT_ROBOTO_BOLD_16));
#endif
  font_roboto_bold_62 = fonts_load_custom_font(resource_get_handle(RESOURCE_ID_FONT_ROBOTO_BOLD_62));

  // set default unit of measure
  change_units(UNITS_IMPERIAL, true);

  buttons_init();

  s_data.window = window_create();
  window_set_background_color(s_data.window, BG_COLOR_WINDOW);
#ifdef PBL_SDK_2
  window_set_fullscreen(s_data.window, true);
#endif
  topbar_layer_init(s_data.window);

  screen_data_layer_init(s_data.window);
  //screen_altitude_layer_init(s_data.window);
#ifdef ENABLE_FUNCTION_LIVE
  screen_live_layer_init(s_data.window);
#endif
  screen_map_layer_init(s_data.window);

  #ifdef PRODUCTION
    #ifndef ENABLE_DEMO
      screen_reset_instant_data();
    #endif
  #endif

  action_bar_init(s_data.window);
  menu_init();

  // Reduce the sniff interval for more responsive messaging at the expense of
  // increased energy consumption by the Bluetooth module
  // The sniff interval will be restored by the system after the app has been
  // unloaded
  //app_comm_set_sniff_interval(SNIFF_INTERVAL_REDUCED);

  communication_init();

  screen_data_update_config(true);
  //screen_altitude_update_config();
  graph_init();

  window_stack_push(s_data.window, true /* Animated */);

  tick_timer_service_subscribe(MINUTE_UNIT, handle_tick);
  bluetooth_connection_service_subscribe(bt_callback);

  send_version(true);
}

#ifdef ENABLE_GLANCE
static void prv_update_app_glance(AppGlanceReloadSession *session, size_t limit, void *context) {
  if (limit < 1) {
    return;
  }

  char message[50];
  snprintf(message, sizeof(message), "%s%s - %s%s", s_data.distance, s_data.unitsDistance, s_data.ascent, s_data.unitsAltitude);
  const AppGlanceSlice entry = (AppGlanceSlice ) {
    .layout = {
        //.icon = RESOURCE_ID_MENU_ICON,
        .subtitle_template_string = message
    },
    .expiration_time = APP_GLANCE_SLICE_NO_EXPIRATION
  };

  const AppGlanceResult result = app_glance_add_slice(session, entry);
  if (result != APP_GLANCE_RESULT_SUCCESS) {
    LOG_INFO("AppGlance Error: %d", result);
  }
}
#endif
static void deinit(void) {

#ifdef ENABLE_GLANCE
  if (s_gpsdata.distance100 > 0) {
    app_glance_reload(prv_update_app_glance, NULL);
  }
#endif
#ifdef PBL_HEALTH
  health_deinit();
#endif
  communication_deinit();
  tick_timer_service_unsubscribe();
  bluetooth_connection_service_unsubscribe();

#ifdef ENABLE_APP_DEINIT
  topbar_layer_deinit();
  screen_data_deinit();
  graph_deinit();
  //screen_altitude_layer_deinit();
#ifdef ENABLE_FUNCTION_LIVE
  screen_live_layer_deinit();
#endif
  screen_map_layer_deinit();

  action_bar_deinit();
  menu_deinit();

  buttons_deinit();

#ifdef PBL_PLATFORM_CHALK
  fonts_unload_custom_font(font_roboto_bold_16);
#endif
  fonts_unload_custom_font(font_roboto_bold_62);
  window_destroy(s_data.window);
#endif
}

int main(void) {
  init();
  app_event_loop();
  deinit();
}


text:
id: V-38691
status: implemented
tag: services
---

The Ansible tasks will disable the ``bluetooth`` service and stop it if it is
running on the system.

To opt-out of this change, adjust the following Ansible variable to ``no``:

.. code-block:: yaml

    security_disable_bluetooth: no


text:
var express = require("express");
var os = require('os');
const path = require('path');
var bodyParser = require('body-parser');

var app = express();

var port = process.env.PORT || 3000;
// var database = process.env.DATABASE || db;
// console.log("Database: " + database);
/*
 To set a port other than 3000:
 in Unix:

 $ PORT=1234 node server.js

 in Windows:

 set PORT=1234
 node server.js
 */

var ifaces = os.networkInterfaces();

Object.keys(ifaces).forEach(function (ifname) {
    var alias = 0;

    ifaces[ifname].forEach(function (iface) {
        if ('IPv4' !== iface.family || iface.internal !== false) {
            // skip over internal (i.e. 127.0.0.1) and non-ipv4 addresses
            return;
        }

        if (alias >= 1) {
            // this single interface has multiple ipv4 addresses
            console.log(ifname + ':' + alias, iface.address);
        } else {
            // this interface has only one ipv4 adress
            console.log(ifname, iface.address);
        }
        ++alias;
    });
});

// parse application/x-www-form-urlencoded
app.use(bodyParser.urlencoded({ extended: false }));

// parse application/json
app.use(bodyParser.json());

// app.use("/server.js",function (req, res, next) {
//     res.redirect("/")
// });

app.use(express.static(path.join(__dirname)));

app.all('*', function (req, res) {
    res.redirect("/");
});

app.listen(port, function () {
    console.log('listening on port ' + port);
    // console.log('press Ctrl + C to shut down server');
});


text:
package adasim.model.internal;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.io.StringReader;
import java.util.List;

import org.jdom.Document;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.junit.Before;
import org.junit.Test;

import adasim.agent.AbstractAdasimAgent;
import adasim.algorithm.delay.LinearTrafficDelayFunction;
import adasim.algorithm.delay.QuadraticTrafficDelayFunction;
import adasim.algorithm.routing.AlwaysRecomputeRoutingAlgorithm;
import adasim.algorithm.routing.ShortestPathRoutingAlgorithm;
import adasim.filter.AdasimFilter;
import adasim.filter.FakeFilter;
import adasim.filter.IdentityFilter;
import adasim.model.AdasimMap;
import adasim.model.ConfigurationException;
import adasim.model.RoadSegment;
import adasim.model.Vehicle;
import adasim.util.ReflectionException;
import adasim.util.ReflectionUtils;

/**
 * @author Jochen Wuttke - wuttkej@gmail.com
 *
 */
public class SimulationXMLBuilderTest {

	private SAXBuilder parser;  //we use this builder to set up our input elements
	private SimulationXMLBuilder builder;

	@Before
	public void setUp() {
		parser = new SAXBuilder( false );
		builder = new SimulationXMLBuilder();
	}

	@Test
	public void nodeNoOptionals() throws JDOMException, IOException, NoSuchMethodException, ReflectionException {
		Document doc = parser.build( new StringReader( "<node id=\"27\" neighbors=\"1 2 3\" delay=\"1\"/>" ) );
		RoadSegment node = builder.buildNode( doc.getRootElement(), new FilterMap() );
		assertEquals( 27, node.getID() );
		assertTrue( node.getNeighbors().isEmpty() );
		assertNull( node.getSpeedStrategy() );
		assertEquals(-1, ReflectionUtils.getProperty(node, "getCapacity") );
		assertEquals(1, ReflectionUtils.getProperty(node, "getDelay") ); //1 is the default delay
	}

	@Test
	public void nodeNoAllOptionals() throws JDOMException, IOException, NoSuchMethodException, ReflectionException {
		Document doc = parser.build( new StringReader( "<node id=\"27\" neighbors=\"1 2 3\" delay=\"2\" capacity=\"5\" strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\" uncertainty_filter=\"adasim.filter.IdentityFilter\"/>" ) );
		RoadSegment node = builder.buildNode( doc.getRootElement(), new FilterMap() );
		assertEquals( 27, node.getID() );
		assertTrue( node.getNeighbors().isEmpty() );
		assertTrue( node.getSpeedStrategy() instanceof LinearTrafficDelayFunction );
		assertEquals(5, ReflectionUtils.getProperty(node, "getCapacity") );
		assertEquals(2, ReflectionUtils.getProperty(node, "getDelay") );
		assertNotNull( "No uncertainty filter assigned", node.getUncertaintyFilter() );
		assertTrue( "Uncertainty filter has wrong type", node.getUncertaintyFilter() instanceof IdentityFilter );
	}

	@Test
	public void graphWithDefaults() throws JDOMException, IOException, ConfigurationException, NoSuchMethodException, ReflectionException {
		Document doc = parser.build( new StringReader( "<graph default_strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\" default_capacity=\"0\">" +
				"<node id=\"1\" neighbors=\"1 2 3 4\" delay=\"2\" capacity=\"5\"/>" +
				"<node id=\"2\" neighbors=\"3\" delay=\"2\" capacity=\"5\" strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\"/>" +
				"<node id=\"4\" neighbors=\"2 4\" delay=\"2\" strategy=\"adasim.algorithm.delay.QuadraticTrafficDelayFunction\"/>" +
				"</graph>" ) );
		AdasimMap graph = builder.buildGraph( doc.getRootElement(), new FilterMap() );
		assertEquals( 3, graph.getRoadSegments().size() );
		RoadSegment node = graph.getRoadSegment( 1 );
		assertEquals( 3, node.getNeighbors().size() );
		assertNotNull( "No default speed strategy assigned", node.getSpeedStrategy() );
		assertTrue( "Default speed strategy has wrong type", node.getSpeedStrategy() instanceof LinearTrafficDelayFunction );
		assertNotNull( "No uncertainty filter assigned", node.getUncertaintyFilter() );
		assertTrue( "Uncertainty filter has wrong type", node.getUncertaintyFilter() instanceof IdentityFilter );
		node = graph.getRoadSegment(4);
		assertEquals(0, ReflectionUtils.getProperty(node, "getCapacity") );
		assertTrue( node.getSpeedStrategy() instanceof QuadraticTrafficDelayFunction );
	}

	@Test
	public void graphWithUncertaintyFilter() throws JDOMException, IOException, ConfigurationException, NoSuchMethodException, ReflectionException {
		Document doc = parser.build( new StringReader( "<graph default_strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\" default_capacity=\"0\" >" +
				"<filters>" +
				"<filter type=\"uncertainty\" filter=\"adasim.filter.FakeFilter\"/>" +
				"</filters>" +
				"<node id=\"1\" neighbors=\"1 2 3 4\" delay=\"2\" capacity=\"5\"/>" +
				"<node id=\"2\" neighbors=\"3\" delay=\"2\" capacity=\"5\" strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\"/>" +
				"<node id=\"4\" neighbors=\"2 4\" delay=\"2\" strategy=\"adasim.algorithm.delay.QuadraticTrafficDelayFunction\">" +
				"<filters>" +
				"<filter type=\"uncertainty\" filter=\"adasim.filter.IdentityFilter\" />" +
				"</filters>" +
				"</node>" +
				"</graph>" ) );
		AdasimMap graph = builder.buildGraph( doc.getRootElement(), new FilterMap() );
		assertEquals( 3, graph.getRoadSegments().size() );
		RoadSegment node = graph.getRoadSegment( 1 );
		assertEquals( 3, node.getNeighbors().size() );
		assertNotNull( "No default speed strategy assigned", node.getSpeedStrategy() );
		assertTrue( "Default speed strategy has wrong type", node.getSpeedStrategy() instanceof LinearTrafficDelayFunction );
		assertNotNull( "No uncertainty filter assigned", node.getUncertaintyFilter() );
		assertTrue( "Uncertainty filter has wrong type " + node.getUncertaintyFilter().getClass(), node.getUncertaintyFilter() instanceof FakeFilter );
		node = graph.getRoadSegment(4);
		assertEquals(0, ReflectionUtils.getProperty(node, "getCapacity") );
		assertTrue( node.getSpeedStrategy() instanceof QuadraticTrafficDelayFunction );
		assertNotNull( "No uncertainty filter assigned", node.getUncertaintyFilter() );
		assertTrue( "Uncertainty filter has wrong type", node.getUncertaintyFilter() instanceof IdentityFilter );
	}

	@Test
	public void graphWithUncertaintyFilterHookup() throws JDOMException, IOException, ConfigurationException, NoSuchMethodException, ReflectionException {
		//test that the uncertainty filter gets called correctly.
		Document doc = parser.build( new StringReader( "<graph default_strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\" default_capacity=\"0\" >" +
				"<filters>" +
				"<filter type=\"uncertainty\" filter=\"adasim.filter.FakeFilter\"/>" +
				"</filters>" +
				"<node id=\"1\" neighbors=\"1 2 3 4\" delay=\"2\" capacity=\"5\"/>" +
				"<node id=\"2\" neighbors=\"3\" delay=\"2\" capacity=\"5\" strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\"/>" +
				"<node id=\"4\" neighbors=\"2 4\" delay=\"2\" strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\" />" +
				"</graph>" ) );
		AdasimMap graph = builder.buildGraph( doc.getRootElement(), new FilterMap() );
		assertEquals( 3, graph.getRoadSegments().size() );
		RoadSegment node = graph.getRoadSegment(4);
		assertNotNull( "No uncertainty filter assigned", node.getUncertaintyFilter() );
		assertTrue( "Uncertainty filter has wrong type", node.getUncertaintyFilter() instanceof FakeFilter );
		//this "fake" node has a capacity of -1, because it has not passed validation yet
		assertEquals( 0, ReflectionUtils.getProperty(node, "getCapacity") );
		assertEquals( 2, ReflectionUtils.getProperty(node, "getDelay") );	//pulls the unfiltered delay
		assertEquals( 3, node.getCurrentDelay( this.getClass() ) );			//expected: delay + 1 + 0 (configured delay + FakeFilter + IdentityFilter)
		assertEquals( 1, node.numVehiclesAtNode() );
		assertEquals( 4, node.getID() );
		assertFalse( node.isClosed() );
	}

	@Test
	public void graphWithPrivacyFilterElement() throws JDOMException, IOException, ConfigurationException, NoSuchMethodException, ReflectionException {
		Document doc = parser.build( new StringReader( "<graph default_strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\" default_capacity=\"0\">" +
				"<filters>" +
				"<filter type=\"privacy\" filter=\"adasim.filter.FakeFilter\" criterion=\"adasim.model.internal.SimulationXMLBuilderTest\"/>" +
				"</filters>" +
				"<node id=\"1\" neighbors=\"1 2 3 4\" delay=\"2\" capacity=\"5\"/>" +
				"<node id=\"2\" neighbors=\"3\" delay=\"2\" capacity=\"5\" strategy=\"adasim.algorithm.delay.LinearTrafficDelayFunction\"/>" +
				"<node id=\"4\" neighbors=\"2 4\" delay=\"2\" strategy=\"adasim.algorithm.delay.QuadraticTrafficDelayFunction\" >" +
				"<filters>" +
				"<filter type=\"privacy\" filter=\"adasim.filter.IdentityFilter\" criterion=\"adasim.model.internal.SimulationXMLBuilderTest\"/>" +
				"</filters>" +
				"</node>" +
				"</graph>" ) );
		AdasimMap graph = builder.buildGraph( doc.getRootElement(), new FilterMap() );
		assertEquals( 3, graph.getRoadSegments().size() );
		RoadSegment node = graph.getRoadSegment( 1 );
		assertEquals( 3, node.getNeighbors().size() );
		assertNotNull( "No default speed strategy assigned", node.getSpeedStrategy() );
		assertTrue( "Default speed strategy has wrong type", node.getSpeedStrategy() instanceof LinearTrafficDelayFunction );
		assertNotNull( "No privacy filter assigned", node.getPrivacyFilter(this.getClass()) );
		assertTrue( "Privacy filter has wrong type " + node.getPrivacyFilter(this.getClass()).getClass().getCanonicalName() , node.getPrivacyFilter(this.getClass()) instanceof FakeFilter );
		node = graph.getRoadSegment(4);
		assertEquals(0, ReflectionUtils.getProperty(node, "getCapacity") );
		assertTrue( node.getSpeedStrategy() instanceof QuadraticTrafficDelayFunction );
		assertNotNull( "No privacy filter assigned", node.getPrivacyFilter(this.getClass()) );
		assertTrue( "Privacy filter has wrong type", node.getPrivacyFilter(this.getClass()) instanceof IdentityFilter );
	}

	@Test
	public void carNoOptionals() throws JDOMException, IOException, NoSuchMethodException, ReflectionException {
		Document doc = parser.build( new StringReader( "<car id=\"27\" start=\"1\" end=\"1\"/>" ) );
		Vehicle car = builder.buildVehicle( doc.getRootElement() );
		assertEquals( 27, car.getID() );
		assertNull( ReflectionUtils.getProperty( car, "getStartNode") );
		assertNull( ReflectionUtils.getProperty( car, "getEndNode") );
		assertNull( car.getStrategy() );
	}

	@Test
	public void carAllOptionals() throws JDOMException, IOException, NoSuchMethodException, ReflectionException {
		Document doc = parser.build( new StringReader( "<car id=\"27\" start=\"1\" end=\"1\" strategy=\"adasim.algorithm.routing.ShortestPathRoutingAlgorithm\" />" ) );
		Vehicle car = builder.buildVehicle( doc.getRootElement() );
		assertEquals( 27, car.getID() );
		assertNull( ReflectionUtils.getProperty( car, "getStartNode") );
		assertNull( ReflectionUtils.getProperty( car, "getEndNode") );
		assertTrue( car.getStrategy() instanceof ShortestPathRoutingAlgorithm );
	}

	@Test
	public void carListWithDefaults() throws JDOMException, IOException, ConfigurationException {
		Document doc = parser.build( new StringReader( "<cars default_strategy=\"adasim.algorithm.routing.AlwaysRecomputeRoutingAlgorithm\">" +
				"<car id=\"1\" start=\"1\" end=\"1\" />" +
				"<car id=\"2\" start=\"1\" end=\"1\" strategy=\"adasim.algorithm.routing.ShortestPathRoutingAlgorithm\" />" +
				"<car id=\"3\" start=\"1\" end=\"1\" />" +
				"</cars>" ) );
		List<Vehicle> cars = builder.buildVehicles( doc.getRootElement() );
		assertEquals( 3, cars.size() );
		Vehicle car = cars.get(0);	//this should be the car with id 1
		assertEquals( 1, car.getID() );
		assertTrue( car.getStrategy() instanceof AlwaysRecomputeRoutingAlgorithm );
		car = cars.get(1);
		assertEquals( 2, car.getID() );
		assertTrue( car.getStrategy() instanceof ShortestPathRoutingAlgorithm );
		assertFalse( "Routing strategies are ==", cars.get(0).getStrategy() == cars.get(2).getStrategy() );
	}

	@Test
	public void agentNoOptionals() throws JDOMException, IOException, ConfigurationException {
		Document doc = parser.build( new StringReader( "<agent id=\"27\" class=\"adasim.model.internal.FakeAgent\" />" ) );
		builder.buildAgent( doc.getRootElement(), new FilterMap() );
	}

	@Test
	public void agentAllOptionals() throws JDOMException, IOException, ConfigurationException {
		Document doc = parser.build( new StringReader( "<agent id=\"27\" class=\"adasim.model.internal.FakeAgent\" parameters=\"blabab\">" +
				"<filters>" +
				"<filter type=\"privacy\" filter=\"adasim.filter.FakeFilter\" criterion=\"adasim.model.internal.SimulationXMLBuilderTest\"/>" +
				"<filter type=\"uncertainty\" filter=\"adasim.filter.FakeFilter\" criterion=\"adasim.model.internal.SimulationXMLBuilderTest\"/>" +
				"</filters>" +
				"</agent>"
		));
		AbstractAdasimAgent agent = (AbstractAdasimAgent)builder.buildAgent( doc.getRootElement(), new FilterMap() );
		assertNotNull( "No default privacy filter assigned", agent.getPrivacyFilter(Object.class) );
		assertTrue( "Privacy filter has wrong type", agent.getPrivacyFilter(Object.class) instanceof IdentityFilter );
		assertNotNull( "No privacy filter assigned for THIS", agent.getPrivacyFilter(this.getClass()) );
		assertTrue( "Privacy filter has wrong type " + agent.getPrivacyFilter(this.getClass()).getClass(), agent.getPrivacyFilter(this.getClass()) instanceof FakeFilter );
		assertNotNull( "No uncertainty filter assigned", agent.getUncertaintyFilter() );
		assertTrue( "Uncertaitny filter has wrong type", agent.getUncertaintyFilter() instanceof FakeFilter );
	}

	@Test
	public void filtersWithoutAgentDeclaration() throws JDOMException, IOException {
		Document doc = parser.build( new StringReader( "<filters>" +
				"<filter type=\"privacy\" filter=\"adasim.filter.FakeFilter\" criterion=\"adasim.model.internal.SimulationXMLBuilderTest\"/>" +
				"<filter type=\"uncertainty\" filter=\"adasim.filter.FakeFilter\" criterion=\"adasim.model.internal.SimulationXMLBuilderTest\"/>" +
				"</filters>"
		));
		FilterMap fm = builder.buildFilters(doc.getRootElement(), new FilterMap(), Object.class );
		assertNotNull( "Default agent has no uncertainty filter", fm.get( Object.class ) );
		assertEquals( "Default agent has wrong uncertainty filter type", FakeFilter.class, fm.get( Object.class).uncertaintyFilter.getClass() );
	}

}

class FakeAgent extends AbstractAdasimAgent {

	public FakeAgent(String s) {}

	/* (non-Javadoc)
	 * @see adasim.model.AdasimAgent#takeSimulationStep()
	 */
	@Override
	public void takeSimulationStep( long cycle) {}

	/* (non-Javadoc)
	 * @see adasim.agent.AdasimAgent#getID()
	 */
}

class FakeFilter2 implements AdasimFilter {

	public FakeFilter2() {
	}

	@Override
	public byte filter(byte b) {
		return 0;
	}

	@Override
	public char filter(char b) {
		return 0;
	}

	@Override
	public short filter(short b) {
		return 0;
	}

	@Override
	public int filter(int b) {
		return 0;
	}

	@Override
	public long filter(long b) {
		return 0;
	}

	@Override
	public float filter(float b) {
		return 0;
	}

	@Override
	public double filter(double b) {
		return 0;
	}

	@Override
	public boolean filter(boolean b) {
		return false;
	}

	@Override
	public <T> T filter(T b) {
		return null;
	}
}

class FakeFilter3 implements AdasimFilter {

	public FakeFilter3() {
	}

	@Override
	public byte filter(byte b) {
		return 0;
	}

	@Override
	public char filter(char b) {
		return 0;
	}

	@Override
	public short filter(short b) {
		return 0;
	}

	@Override
	public int filter(int b) {
		return 0;
	}

	@Override
	public long filter(long b) {
		return 0;
	}

	@Override
	public float filter(float b) {
		return 0;
	}

	@Override
	public double filter(double b) {
		return 0;
	}

	@Override
	public boolean filter(boolean b) {
		return false;
	}

	@Override
	public <T> T filter(T b) {
		return null;
	}
}

class FakeFilter4 implements AdasimFilter {

	public FakeFilter4() {
	}

	@Override
	public byte filter(byte b) {
		return 0;
	}

	@Override
	public char filter(char b) {
		return 0;
	}

	@Override
	public short filter(short b) {
		return 0;
	}

	@Override
	public int filter(int b) {
		return 0;
	}

	@Override
	public long filter(long b) {
		return 0;
	}

	@Override
	public float filter(float b) {
		return 0;
	}

	@Override
	public double filter(double b) {
		return 0;
	}

	@Override
	public boolean filter(boolean b) {
		return false;
	}

	@Override
	public <T> T filter(T b) {
		return null;
	}
}


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.11"/>
<title>GrPPI: Farm pattern</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>

<link href="search/search.css" rel="stylesheet" type="text/css"/>

<link href="doxygen.css" rel="stylesheet" type="text/css" />

GrPPI
    0.3.1

Generic and Reusable Parallel Pattern Interface

Main Page

Related Pages

Modules

Namespaces

Classes

Files

          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)"
               onblur="searchBox.OnSearchFieldFocus(false)"
               onkeyup="searchBox.OnSearchFieldChange(event)"/>

Farm pattern

The farm pattern (or task-farm) is a streaming pattern that applies an operation to every element in a stream. This streaming pattern can only be used inside another pattern and consequently does not take an execution policy itself, but uses the execution policy of its enclosing pattern.

The interface to the farm pattern is provided by function grppi::farm().

grppi::pipeline(exec,

  stage1,

  grppi::farm(arguments...),

  stage3,

  stage4);

Farm variants

There is a single variant:

Composable farm: Defines a farm that can be used as a building block by another pattern (e.g. a pipeline).

Key elements in a farm

The key elements of a farm are the cardinality and the Transformer.

The cardinality is the number of replicas of the farm that can be concurrently executed.

The central element in a farm is the Transformer. The operation may be any C++ callable entity. This operation, is a unary operation taking a data item and returning its transformation. Thus, a transformer op is any operation that, given an input value x of type T makes valid the following:

U res{transformer(x)};

Details on farm variants

Composable farm

A composable farm applies a Transformer to each data item in a stream. The farm can be inserted into another upper level pattern (which will be responsible for generation and consumption)

 Example: Use a farm as a stage of a composed pipeline.

grppi::pipeline(exec,

  stageA,

  stageB,

  grppi::farm(4, [](auto x) {

    return x.length();

  }),

  stageC

);

 Note: For brevity we do not show here the details of other stages.

For composing complex patterns, the farm() function may be used to create an object that may be used later in the composition.

 Example: Build a farm as a composable pattern and use later as a stage of a pipeline.

auto print_long_words = grppi::farm(3, [](auto x) {

  if (x.length() > 4) std::cout << x << std::endl;

});

grppi::pipeline(exec,

  stageA,

  stageB,

  print_long_words,

  stageC

);

 Note: For brevity we do not show here the details of other stages.

<address class="footer">
Generated by

 1.8.11
</address>


text:
var app = angular.module("myapp")

app.controller("loginControl", function($scope, loginService) {

  $scope.login = function(user) {
    loginService.login(user, $scope); // will call login service

  }

});


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<meta name="viewport" content="width=device-width" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>Billing e.g. invoices and receipts</title>
<link href="styles.css" media="all" rel="stylesheet" type="text/css" />

Amount Paid Successfully

Thanks for using PetsAr.us. You saved a innocent life today !

												User
Invoice #12345
22 September 2017

										View in browser

										PetsAr.Us Inc. 123 Van Ness, San Francisco 94102

							Questions? Email support@PetsAr.US


text:
<readable><title>200771289_31902164a7</title><content>
Men wearing sunglasses with their backs turned to the camera .
The backs of two men wearing backpacks near some tents .
Two men in black bulletproof vests watch something taking place to the right of the picture .
Two men in black outfits have backpacks .
Two men in bullet proof vests watch over a crowd by some white tents .
</content></readable>


text:
<?php
/**
 * Created by IntelliJ IDEA.
 * User: Besitzer
 * Date: 12.09.2016
 * Time: 08:00
 */

namespace FbBasic\GraphNodes;

use Facebook\GraphNodes\GraphLocation;

class CoverPhoto extends GraphLocation implements GraphInterface
{
    protected static $graphObjectFields = [
        "id",
        "city",
        "source",
        "offset_y",
        "offset_x"
    ];

    /**
     * Getter for $graphObjectFields.
     *
     * @return array
     */
    public static function getObjectFields()
    {
        return static::$graphObjectFields;
    }
    /**
     * Getter for $graphObjectEdgesMap.
     *
     * @return array
     */
    public static function getObjectEdges()
    {
        return static::$graphObjectEdgesMap;
    }

}


text:
Collection of projects for the ESP8266. Have a look at the projects for more detailed instructions or go to http://blog.squix.ch

## ESP8266 Starter Kit

If you are just starting with ESP8266 and need a nice kit to get started have a look at my starter kit.
https://blog.squix.org/product/weatherstation-kit-w-white-oled
By buying the kit from me you are supporting future development. Thank you!

[![Squix ESP8266 WeatherStation Classic Kit](https://blog.squix.org/wp-content/uploads/2016/12/Components4-300x300.jpg)](https://blog.squix.org/product/weatherstation-kit-w-white-oled)


text:
import click

from extensions.apigee.terraformBugWorkaround import runTerraform,\
                                                     restoreOldRoleToExistingFunctionWithCLI,\
                                                     featureName,\
                                                     gatewayFunctionName
from utils.api_config import update_config
from utils.jenkins import startJob
from utils.helper import terraformStateSanityCheck

@click.command()
@click.option(
    "--region",
    "-r",
    help='Specify the region where your Jazz installation lives',
    type=click.Choice(['us-east-1', 'us-west-2']),
    default='us-east-1'
)
@click.option(
    "--stackprefix",
    "-p",
    help='Specify the stackprefix of your existing Jazz installation (e.g. myjazz), \
          your existing config will be imported',
    prompt=True
)
@click.option(
    '--jazz_userpass',
    nargs=2,
    required=True,
    help='Provide the username and password \
    of the jazz application separated by a space',
    prompt=True)
@click.option(
    "--jazz_apiendpoint",
    help='Specify the Jazz Endpoint',
    prompt=True
)
@click.option(
    "--jenkins_url",
    help='Specify the Jenkins url',
    prompt=True
)
@click.option(
    '--jenkins_userpass',
    nargs=2,
    required=True,
    help='Provide the username and password \
    of the jenkins separated by a space',
    prompt=True)
def uninstall(region, stackprefix, jazz_userpass, jazz_apiendpoint, jenkins_url, jenkins_userpass):
    click.secho('\n\nThis will remove {0} functionality from your Jazz deployment'.format(featureName), fg='blue')
    terraformStateSanityCheck("extensions/apigee")

    # TODO remove this entire module when the terraform bug is fixed
    click.secho('\nRestoring old role to gateway function', fg='blue')

    # Restore old role first, before we destroy the Terraform resources
    restoreOldRoleToExistingFunctionWithCLI(stackprefix + "_" + gatewayFunctionName)
    jazz_userpass_list = ''.join(list(jazz_userpass)).split()
    jazz_username, jazz_password = jazz_userpass_list[0], jazz_userpass_list[1]
    jenkins_userpass_list = ''.join(list(jenkins_userpass)).split()
    jenkins_username, jenkins_password = jenkins_userpass_list[0], jenkins_userpass_list[1]
    runTerraform(region, stackprefix, False)
    update_config(
        "APIGEE.ENABLE_APIGEE",
        'false',
        jazz_username,
        jazz_password,
        jazz_apiendpoint
    )
    # Trigger metrics api
    metricJobUrl = "job/build-pack-api/buildWithParameters?token=jazz-101-job&service_name=metrics&domain" \
                   "=jazz&scm_branch=master"
    startJob(jenkins_url, jenkins_username, jenkins_password, metricJobUrl)
    # Trigger jazz ui
    startJob(jenkins_url, jenkins_username, jenkins_password, "job/jazz_ui/buildWithParameters?token=jazz-101-job")


text:
<?xml version="1.0" encoding="UTF-8"?>

<recipeml version="0.5">
  <recipe>
    <head>
      <title>Artichoke and Cherry Tomato Tarts</title>
      <categories>
        <cat>None</cat></categories>
      <yield>1</yield></head>
    <ingredients>
      <ing>
        <amt>
          <qty>1</qty>
          <unit>large</unit></amt>
        <item>Egg yolk; beaten lightly</item></ing>
      <ing>
        <amt>
          <qty>1/4</qty>
          <unit>cups</unit></amt>
        <item>Sour cream</item></ing>
      <ing>
        <amt>
          <qty>1</qty>
          <unit/></amt>
        <item>Sheet; (1/2 pound) thawed frozen puff pastry, cut into two</item></ing>
      <ing>
        <amt>
          <qty>4</qty>
          <unit/></amt>
        <item>Inch squares and reserving the rest for another use</item></ing>
      <ing>
        <amt>
          <qty>1</qty>
          <unit>cup</unit></amt>
        <item>Quartered drained marinated artichoke hearts</item></ing>
      <ing>
        <amt>
          <qty>10</qty>
          <unit/></amt>
        <item>Vine-ripened cherry tomatoes; halved</item></ing>
      <ing>
        <amt>
          <qty>1</qty>
          <unit>teaspoon</unit></amt>
        <item>Fresh thyme leaves</item></ing></ingredients>
    <directions>
      <step>  Can be prepared in 45 minutes or less.

  Preheat oven to 400 deg. F. and lightly flour a baking sheet.

  In a bowl whisk together yolk, sour cream, and salt and pepper to taste.
  Arrange pastry squares on baking sheet and spoon one fourth of mixture onto
  center of each. Top mixture with artichokes and tomatoes. Spoon remaining
  sour cream mixture over vegetables and sprinkle with thyme.

  Bake tarts in middle of oven until pastry is golden, about
       12    minutes.
  Serves 2 as a light main course

  Gourmet September 1996

  Posted to EAT-L Digest  by Jennie Craig <jecraig@LAN-INC.COM> on Apr 3,
  1998

</step></directions></recipe></recipeml>


text:
package xworker.eclipse.zest.zestmodel;

import org.eclipse.swt.SWT;
import org.eclipse.swt.graphics.Color;
import org.eclipse.swt.graphics.Font;
import org.eclipse.swt.graphics.Image;
import org.eclipse.zest.core.widgets.Graph;
import org.eclipse.zest.core.widgets.GraphNode;
import org.xmeta.ActionContext;
import org.xmeta.Thing;

import xworker.swt.style.StyleSetStyleCreator;

public class Node {
	/**
	 * 改方法应该由节点的实例调用。
	 *
	 * @param actionContext
	 * @return
	 */
	public static GraphNode create(ActionContext actionContext){
		Thing self = actionContext.getObject("self");
		Graph parent = actionContext.getObject("parent");

		GraphNode node = new GraphNode(parent, SWT.NONE);

		//背景顏色
		Color backgroundColor = (Color) StyleSetStyleCreator.createResource(self.getString("backgroundColor"),
	                "xworker.swt.graphics.Color", "rgb", actionContext);
        if(backgroundColor != null){
            node.setBackgroundColor(backgroundColor);
        }

        //边框颜色borderColor
		Color borderColor = (Color) StyleSetStyleCreator.createResource(self.getString("borderColor"),
                "xworker.swt.graphics.Color", "rgb", actionContext);
	    if(borderColor != null){
	        node.setBorderColor(borderColor);
	    }

	    //borderHighlightColor
		Color borderHighlightColor = (Color) StyleSetStyleCreator.createResource(self.getString("borderHighlightColor"),
                "xworker.swt.graphics.Color", "rgb", actionContext);
	    if(borderHighlightColor != null){
	        node.setBorderHighlightColor(borderHighlightColor);
	    }

	    //borderWidth
	    if(self.getStringBlankAsNull("borderWidth") != null){
	    	node.setBorderWidth(self.getInt("borderWidth"));
	    }

	    //cacheLabel
	    if(self.getStringBlankAsNull("cacheLabel") != null){
	    	node.setCacheLabel(self.getBoolean("cacheLabel"));
	    }

	    //font
        Font font = (Font) StyleSetStyleCreator.createResource(self.getString("font"),
                "xworker.swt.graphics.Font", "fontData", actionContext);
        if(font != null){
            node.setFont(font);
        }

        //foregroundColor
		Color foregroundColor = (Color) StyleSetStyleCreator.createResource(self.getString("foregroundColor"),
                "xworker.swt.graphics.Color", "rgb", actionContext);
	    if(foregroundColor != null){
	        node.setForegroundColor(foregroundColor);
	    }

	    //highlightColor
		Color highlightColor = (Color) StyleSetStyleCreator.createResource(self.getString("highlightColor"),
                "xworker.swt.graphics.Color", "rgb", actionContext);
	    if(highlightColor != null){
	        node.setHighlightColor(highlightColor);
	    }

	    //image
        Image image = (Image) StyleSetStyleCreator.createResource(self.getString("image"),
                "xworker.swt.graphics.Image", "imageFile", actionContext);
        if(image != null){
            node.setImage(image);
        }

        //x,y
        if(self.getStringBlankAsNull("x") != null && self.getStringBlankAsNull("y") != null){
        	double x = self.getDouble("x");
        	double y = self.getDouble("y");
        	node.setLocation(x, y);
        }

        //width,height
        if(self.getStringBlankAsNull("width") != null && self.getStringBlankAsNull("height") != null){
        	double width = self.getDouble("width");
        	double height = self.getDouble("height");
        	node.setSize(width, height);
        }

        node.setVisible(self.getBoolean("visible"));

        String text = self.getStringBlankAsNull("text");
        if(text != null){
        	node.setText(text);
        }

        //数据是事物本身
        node.setData(self);

        //保存变量
        actionContext.g().put(self.getMetadata().getName(), node);

		return node;
	}
}


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.devtools.j2objc.ast;

import com.google.devtools.j2objc.types.ExecutablePair;
import java.util.List;
import javax.lang.model.element.ExecutableElement;
import javax.lang.model.type.ExecutableType;
import javax.lang.model.type.TypeMirror;

/**
 * Node for a super constructor invocation. (i.e. "super(...);")
 */
public class SuperConstructorInvocation extends Statement {

  private ExecutablePair method = ExecutablePair.NULL;
  private TypeMirror varargsType = null;
  private final ChildLink<Expression> expression = ChildLink.create(Expression.class, this);
  private final ChildList<Expression> arguments = ChildList.create(Expression.class, this);

  public SuperConstructorInvocation() {}

  public SuperConstructorInvocation(SuperConstructorInvocation other) {
    super(other);
    method = other.getExecutablePair();
    varargsType = other.getVarargsType();
    expression.copyFrom(other.getExpression());
    arguments.copyFrom(other.getArguments());
  }

  public SuperConstructorInvocation(ExecutablePair method) {
    this.method = method;
  }

  @Override
  public Kind getKind() {
    return Kind.SUPER_CONSTRUCTOR_INVOCATION;
  }

  public ExecutablePair getExecutablePair() {
    return method;
  }

  public SuperConstructorInvocation setExecutablePair(ExecutablePair newMethod) {
    method = newMethod;
    return this;
  }

  public ExecutableElement getExecutableElement() {
    return method.element();
  }

  public ExecutableType getExecutableType() {
    return method.type();
  }

  public TypeMirror getVarargsType() {
    return varargsType;
  }

  public SuperConstructorInvocation setVarargsType(TypeMirror type) {
    varargsType = type;
    return this;
  }

  public Expression getExpression() {
    return expression.get();
  }

  public SuperConstructorInvocation setExpression(Expression newExpression) {
    expression.set(newExpression);
    return this;
  }

  public List<Expression> getArguments() {
    return arguments;
  }

  public SuperConstructorInvocation addArgument(Expression arg) {
    arguments.add(arg);
    return this;
  }

  public SuperConstructorInvocation addArgument(int index, Expression arg) {
    arguments.add(index, arg);
    return this;
  }

  @Override
  protected void acceptInner(TreeVisitor visitor) {
    if (visitor.visit(this)) {
      expression.accept(visitor);
      arguments.accept(visitor);
    }
    visitor.endVisit(this);
  }

  @Override
  public SuperConstructorInvocation copy() {
    return new SuperConstructorInvocation(this);
  }
}