text:
"""BERT-joint baseline for NQ v1.0."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import gzip
import json
import os
import random
import re

import enum
from bert import modeling
from bert import optimization
from bert import tokenization
import numpy as np
import tensorflow.compat.v1 as tf
from tensorflow.compat.v1 import estimator as tf_estimator
from tensorflow.contrib import cluster_resolver as contrib_cluster_resolver
from tensorflow.contrib import tpu as contrib_tpu

flags = tf.flags
FLAGS = flags.FLAGS

flags.DEFINE_string(
    "bert_config_file", None,
    "The config json file corresponding to the pre-trained BERT model. "
    "This specifies the model architecture.")

flags.DEFINE_string("vocab_file", None,
                    "The vocabulary file that the BERT model was trained on.")

flags.DEFINE_string(
    "output_dir", None,
    "The output directory where the model checkpoints will be written.")

flags.DEFINE_string("train_precomputed_file", None,
                    "Precomputed tf records for training.")

flags.DEFINE_integer("train_num_precomputed", None,
                     "Number of precomputed tf records for training.")

flags.DEFINE_string(
    "predict_file", None,
    "NQ json for predictions. E.g., dev-v1.1.jsonl.gz or test-v1.1.jsonl.gz")

flags.DEFINE_string(
    "output_prediction_file", None,
    "Where to print predictions in NQ prediction format, to be passed to"
    "natural_questions.nq_eval.")

flags.DEFINE_string(
    "init_checkpoint", None,
    "Initial checkpoint (usually from a pre-trained BERT model).")

flags.DEFINE_bool(
    "do_lower_case", True,
    "Whether to lower case the input text. Should be True for uncased "
    "models and False for cased models.")

flags.DEFINE_integer(
    "max_seq_length", 384,
    "The maximum total input sequence length after WordPiece tokenization. "
    "Sequences longer than this will be truncated, and sequences shorter "
    "than this will be padded.")

flags.DEFINE_integer(
    "doc_stride", 128,
    "When splitting up a long document into chunks, how much stride to "
    "take between chunks.")

flags.DEFINE_integer(
    "max_query_length", 64,
    "The maximum number of tokens for the question. Questions longer than "
    "this will be truncated to this length.")

flags.DEFINE_bool("do_train", False, "Whether to run training.")

flags.DEFINE_bool("do_predict", False, "Whether to run eval on the dev set.")

flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.")

flags.DEFINE_integer("predict_batch_size", 8,
                     "Total batch size for predictions.")

flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.")

flags.DEFINE_float("num_train_epochs", 3.0,
                   "Total number of training epochs to perform.")

flags.DEFINE_float(
    "warmup_proportion", 0.1,
    "Proportion of training to perform linear learning rate warmup for. "
    "E.g., 0.1 = 10% of training.")

flags.DEFINE_integer("save_checkpoints_steps", 1000,
                     "How often to save the model checkpoint.")

flags.DEFINE_integer("iterations_per_loop", 1000,
                     "How many steps to make in each estimator call.")

flags.DEFINE_integer(
    "n_best_size", 20,
    "The total number of n-best predictions to generate in the "
    "nbest_predictions.json output file.")

flags.DEFINE_integer(
    "max_answer_length", 30,
    "The maximum length of an answer that can be generated. This is needed "
    "because the start and end predictions are not conditioned on one another.")

flags.DEFINE_float(
    "include_unknowns", -1.0,
    "If positive, probability of including answers of type `UNKNOWN`.")

flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.")

tf.flags.DEFINE_string(
    "tpu_name", None,
    "The Cloud TPU to use for training. This should be either the name "
    "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 "
    "url.")

tf.flags.DEFINE_string(
    "tpu_zone", None,
    "[Optional] GCE zone where the Cloud TPU is located in. If not "
    "specified, we will attempt to automatically detect the GCE project from "
    "metadata.")

tf.flags.DEFINE_string(
    "gcp_project", None,
    "[Optional] Project name for the Cloud TPU-enabled project. If not "
    "specified, we will attempt to automatically detect the GCE project from "
    "metadata.")

tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.")

flags.DEFINE_integer(
    "num_tpu_cores", 8,
    "Only used if `use_tpu` is True. Total number of TPU cores to use.")

flags.DEFINE_bool(
    "verbose_logging", False,
    "If true, all of the warnings related to data processing will be printed. "
    "A number of warnings are expected for a normal NQ evaluation.")

flags.DEFINE_boolean(
    "skip_nested_contexts", True,
    "Completely ignore context that are not top level nodes in the page.")

flags.DEFINE_integer("task_id", 0,
                     "Train and dev shard to read from and write to.")

flags.DEFINE_integer("max_contexts", 48,
                     "Maximum number of contexts to output for an example.")

flags.DEFINE_integer(
    "max_position", 50,
    "Maximum context position for which to generate special tokens.")

TextSpan = collections.namedtuple("TextSpan", "token_positions text")

class AnswerType(enum.IntEnum):
  """Type of NQ answer."""
  UNKNOWN = 0
  YES = 1
  NO = 2
  SHORT = 3
  LONG = 4

class Answer(collections.namedtuple("Answer", ["type", "text", "offset"])):
  """Answer record.

  An Answer contains the type of the answer and possibly the text (for
  long) as well as the offset (for extractive).
  """

  def __new__(cls, type_, text=None, offset=None):
    return super(Answer, cls).__new__(cls, type_, text, offset)

class NqExample(object):
  """A single training/test example."""

  def __init__(self,
               example_id,
               qas_id,
               questions,
               doc_tokens,
               doc_tokens_map=None,
               answer=None,
               start_position=None,
               end_position=None):
    self.example_id = example_id
    self.qas_id = qas_id
    self.questions = questions
    self.doc_tokens = doc_tokens
    self.doc_tokens_map = doc_tokens_map
    self.answer = answer
    self.start_position = start_position
    self.end_position = end_position

def has_long_answer(a):
  return (a["long_answer"]["start_token"] >= 0 and
          a["long_answer"]["end_token"] >= 0)

def should_skip_context(e, idx):
  if (FLAGS.skip_nested_contexts and
      not e["long_answer_candidates"][idx]["top_level"]):
    return True
  elif not get_candidate_text(e, idx).text.strip():
    # Skip empty contexts.
    return True
  else:
    return False

def get_first_annotation(e):
  """Returns the first short or long answer in the example.

  Args:
    e: (dict) annotated example.

  Returns:
    annotation: (dict) selected annotation
    annotated_idx: (int) index of the first annotated candidate.
    annotated_sa: (tuple) char offset of the start and end token
        of the short answer. The end token is exclusive.
  """
  positive_annotations = sorted(
      [a for a in e["annotations"] if has_long_answer(a)],
      key=lambda a: a["long_answer"]["candidate_index"])

  for a in positive_annotations:
    if a["short_answers"]:
      idx = a["long_answer"]["candidate_index"]
      start_token = a["short_answers"][0]["start_token"]
      end_token = a["short_answers"][-1]["end_token"]
      return a, idx, (token_to_char_offset(e, idx, start_token),
                      token_to_char_offset(e, idx, end_token) - 1)

  for a in positive_annotations:
    idx = a["long_answer"]["candidate_index"]
    return a, idx, (-1, -1)

  return None, -1, (-1, -1)

def get_text_span(example, span):
  """Returns the text in the example's document in the given token span."""
  token_positions = []
  tokens = []
  for i in range(span["start_token"], span["end_token"]):
    t = example["document_tokens"][i]
    if not t["html_token"]:
      token_positions.append(i)
      token = t["token"].replace(" ", "")
      tokens.append(token)
  return TextSpan(token_positions, " ".join(tokens))

def token_to_char_offset(e, candidate_idx, token_idx):
  """Converts a token index to the char offset within the candidate."""
  c = e["long_answer_candidates"][candidate_idx]
  char_offset = 0
  for i in range(c["start_token"], token_idx):
    t = e["document_tokens"][i]
    if not t["html_token"]:
      token = t["token"].replace(" ", "")
      char_offset += len(token) + 1
  return char_offset

def get_candidate_type(e, idx):
  """Returns the candidate's type: Table, Paragraph, List or Other."""
  c = e["long_answer_candidates"][idx]
  first_token = e["document_tokens"][c["start_token"]]["token"]
  if first_token == "<Table>":
    return "Table"
  elif first_token == "<P>":
    return "Paragraph"
  elif first_token in ("<Ul>", "<Dl>", "<Ol>"):
    return "List"
  elif first_token in ("<Tr>", "<Li>", "<Dd>", "<Dt>"):
    return "Other"
  else:
    tf.logging.warning("Unknoww candidate type found: %s", first_token)
    return "Other"

def add_candidate_types_and_positions(e):
  """Adds type and position info to each candidate in the document."""
  counts = collections.defaultdict(int)
  for idx, c in candidates_iter(e):
    context_type = get_candidate_type(e, idx)
    if counts[context_type] < FLAGS.max_position:
      counts[context_type] += 1
    c["type_and_position"] = "[%s=%d]" % (context_type, counts[context_type])

def get_candidate_type_and_position(e, idx):
  """Returns type and position info for the candidate at the given index."""
  if idx == -1:
    return "[NoLongAnswer]"
  else:
    return e["long_answer_candidates"][idx]["type_and_position"]

def get_candidate_text(e, idx):
  """Returns a text representation of the candidate at the given index."""
  # No candidate at this index.
  if idx < 0 or idx >= len(e["long_answer_candidates"]):
    return TextSpan([], "")

  # This returns an actual candidate.
  return get_text_span(e, e["long_answer_candidates"][idx])

def candidates_iter(e):
  """Yield's the candidates that should not be skipped in an example."""
  for idx, c in enumerate(e["long_answer_candidates"]):
    if should_skip_context(e, idx):
      continue
    yield idx, c

def create_example_from_jsonl(line):
  """Creates an NQ example from a given line of JSON."""
  e = json.loads(line, object_pairs_hook=collections.OrderedDict)
  add_candidate_types_and_positions(e)
  annotation, annotated_idx, annotated_sa = get_first_annotation(e)

  # annotated_idx: index of the first annotated context, -1 if null.
  # annotated_sa: short answer start and end char offsets, (-1, -1) if null.
  question = {"input_text": e["question_text"]}
  answer = {
      "candidate_id": annotated_idx,
      "span_text": "",
      "span_start": -1,
      "span_end": -1,
      "input_text": "long",
  }

  # Yes/no answers are added in the input text.
  if annotation is not None:
    assert annotation["yes_no_answer"] in ("YES", "NO", "NONE")
    if annotation["yes_no_answer"] in ("YES", "NO"):
      answer["input_text"] = annotation["yes_no_answer"].lower()

  # Add a short answer if one was found.
  if annotated_sa != (-1, -1):
    answer["input_text"] = "short"
    span_text = get_candidate_text(e, annotated_idx).text
    answer["span_text"] = span_text[annotated_sa[0]:annotated_sa[1]]
    answer["span_start"] = annotated_sa[0]
    answer["span_end"] = annotated_sa[1]
    expected_answer_text = get_text_span(
        e, {
            "start_token": annotation["short_answers"][0]["start_token"],
            "end_token": annotation["short_answers"][-1]["end_token"],
        }).text
    assert expected_answer_text == answer["span_text"], (expected_answer_text,
                                                         answer["span_text"])

  # Add a long answer if one was found.
  elif annotation and annotation["long_answer"]["candidate_index"] >= 0:
    answer["span_text"] = get_candidate_text(e, annotated_idx).text
    answer["span_start"] = 0
    answer["span_end"] = len(answer["span_text"])

  context_idxs = [-1]
  context_list = [{"id": -1, "type": get_candidate_type_and_position(e, -1)}]
  context_list[-1]["text_map"], context_list[-1]["text"] = (
      get_candidate_text(e, -1))
  for idx, _ in candidates_iter(e):
    context = {"id": idx, "type": get_candidate_type_and_position(e, idx)}
    context["text_map"], context["text"] = get_candidate_text(e, idx)
    context_idxs.append(idx)
    context_list.append(context)
    if len(context_list) >= FLAGS.max_contexts:
      break

  # Assemble example.
  example = {
      "name": e["document_title"],
      "id": str(e["example_id"]),
      "questions": [question],
      "answers": [answer],
      "has_correct_context": annotated_idx in context_idxs
  }

  single_map = []
  single_context = []
  offset = 0
  for context in context_list:
    single_map.extend([-1, -1])
    single_context.append("[ContextId=%d] %s" %
                          (context["id"], context["type"]))
    offset += len(single_context[-1]) + 1
    if context["id"] == annotated_idx:
      answer["span_start"] += offset
      answer["span_end"] += offset

    # Many contexts are empty once the HTML tags have been stripped, so we
    # want to skip those.
    if context["text"]:
      single_map.extend(context["text_map"])
      single_context.append(context["text"])
      offset += len(single_context[-1]) + 1

  example["contexts"] = " ".join(single_context)
  example["contexts_map"] = single_map
  if annotated_idx in context_idxs:
    expected = example["contexts"][answer["span_start"]:answer["span_end"]]

    # This is a sanity check to ensure that the calculated start and end
    # indices match the reported span text. If this assert fails, it is likely
    # a bug in the data preparation code above.
    assert expected == answer["span_text"], (expected, answer["span_text"])

  return example

def make_nq_answer(contexts, answer):
  """Makes an Answer object following NQ conventions.

  Args:
    contexts: string containing the context
    answer: dictionary with `span_start` and `input_text` fields

  Returns:
    an Answer object. If the Answer type is YES or NO or LONG, the text
    of the answer is the long answer. If the answer type is UNKNOWN, the text of
    the answer is empty.
  """
  start = answer["span_start"]
  end = answer["span_end"]
  input_text = answer["input_text"]

  if (answer["candidate_id"] == -1 or start >= len(contexts) or
      end > len(contexts)):
    answer_type = AnswerType.UNKNOWN
    start = 0
    end = 1
  elif input_text.lower() == "yes":
    answer_type = AnswerType.YES
  elif input_text.lower() == "no":
    answer_type = AnswerType.NO
  elif input_text.lower() == "long":
    answer_type = AnswerType.LONG
  else:
    answer_type = AnswerType.SHORT

  return Answer(answer_type, text=contexts[start:end], offset=start)

def read_nq_entry(entry, is_training):
  """Converts a NQ entry into a list of NqExamples."""

  def is_whitespace(c):
    return c in " \t\r\n" or ord(c) == 0x202F

  examples = []
  contexts_id = entry["id"]
  contexts = entry["contexts"]
  doc_tokens = []
  char_to_word_offset = []
  prev_is_whitespace = True
  for c in contexts:
    if is_whitespace(c):
      prev_is_whitespace = True
    else:
      if prev_is_whitespace:
        doc_tokens.append(c)
      else:
        doc_tokens[-1] += c
      prev_is_whitespace = False
    char_to_word_offset.append(len(doc_tokens) - 1)

  questions = []
  for i, question in enumerate(entry["questions"]):
    qas_id = "{}".format(contexts_id)
    question_text = question["input_text"]
    start_position = None
    end_position = None
    answer = None
    if is_training:
      answer_dict = entry["answers"][i]
      answer = make_nq_answer(contexts, answer_dict)

      # For now, only handle extractive, yes, and no.
      if answer is None or answer.offset is None:
        continue
      start_position = char_to_word_offset[answer.offset]
      end_position = char_to_word_offset[answer.offset + len(answer.text) - 1]

      # Only add answers where the text can be exactly recovered from the
      # document. If this CAN'T happen it's likely due to weird Unicode
      # stuff so we will just skip the example.
      #
      # Note that this means for training mode, every example is NOT
      # guaranteed to be preserved.
      actual_text = " ".join(doc_tokens[start_position:(end_position + 1)])
      cleaned_answer_text = " ".join(
          tokenization.whitespace_tokenize(answer.text))
      if actual_text.find(cleaned_answer_text) == -1:
        tf.logging.warning("Could not find answer: '%s' vs. '%s'", actual_text,
                           cleaned_answer_text)
        continue

    questions.append(question_text)
    example = NqExample(
        example_id=int(contexts_id),
        qas_id=qas_id,
        questions=questions[:],
        doc_tokens=doc_tokens,
        doc_tokens_map=entry.get("contexts_map", None),
        answer=answer,
        start_position=start_position,
        end_position=end_position)
    examples.append(example)
  return examples

def convert_examples_to_features(examples, tokenizer, is_training, output_fn):
  """Converts a list of NqExamples into InputFeatures."""
  num_spans_to_ids = collections.defaultdict(list)

  for example in examples:
    example_index = example.example_id
    features = convert_single_example(example, tokenizer, is_training)
    num_spans_to_ids[len(features)].append(example.qas_id)

    for feature in features:
      feature.example_index = example_index
      feature.unique_id = feature.example_index + feature.doc_span_index
      output_fn(feature)

  return num_spans_to_ids

def convert_single_example(example, tokenizer, is_training):
  """Converts a single NqExample into a list of InputFeatures."""
  tok_to_orig_index = []
  orig_to_tok_index = []
  all_doc_tokens = []
  features = []
  for (i, token) in enumerate(example.doc_tokens):
    orig_to_tok_index.append(len(all_doc_tokens))
    sub_tokens = tokenize(tokenizer, token)
    tok_to_orig_index.extend([i] * len(sub_tokens))
    all_doc_tokens.extend(sub_tokens)

  # `tok_to_orig_index` maps wordpiece indices to indices of whitespace
  # tokenized word tokens in the contexts. The word tokens might themselves
  # correspond to word tokens in a larger document, with the mapping given
  # by `doc_tokens_map`.
  if example.doc_tokens_map:
    tok_to_orig_index = [
        example.doc_tokens_map[index] for index in tok_to_orig_index
    ]

  # QUERY
  query_tokens = []
  query_tokens.append("[Q]")
  query_tokens.extend(tokenize(tokenizer, example.questions[-1]))
  if len(query_tokens) > FLAGS.max_query_length:
    query_tokens = query_tokens[-FLAGS.max_query_length:]

  # ANSWER
  tok_start_position = 0
  tok_end_position = 0
  if is_training:
    tok_start_position = orig_to_tok_index[example.start_position]
    if example.end_position < len(example.doc_tokens) - 1:
      tok_end_position = orig_to_tok_index[example.end_position + 1] - 1
    else:
      tok_end_position = len(all_doc_tokens) - 1

  # The -3 accounts for [CLS], [SEP] and [SEP]
  max_tokens_for_doc = FLAGS.max_seq_length - len(query_tokens) - 3

  # We can have documents that are longer than the maximum sequence length.
  # To deal with this we do a sliding window approach, where we take chunks
  # of up to our max length with a stride of `doc_stride`.
  _DocSpan = collections.namedtuple(  # pylint: disable=invalid-name
      "DocSpan", ["start", "length"])
  doc_spans = []
  start_offset = 0
  while start_offset < len(all_doc_tokens):
    length = len(all_doc_tokens) - start_offset
    length = min(length, max_tokens_for_doc)
    doc_spans.append(_DocSpan(start=start_offset, length=length))
    if start_offset + length == len(all_doc_tokens):
      break
    start_offset += min(length, FLAGS.doc_stride)

  for (doc_span_index, doc_span) in enumerate(doc_spans):
    tokens = []
    token_to_orig_map = {}
    segment_ids = []
    tokens.append("[CLS]")
    segment_ids.append(0)
    tokens.extend(query_tokens)
    segment_ids.extend([0] * len(query_tokens))
    tokens.append("[SEP]")
    segment_ids.append(0)

    for i in range(doc_span.length):
      split_token_index = doc_span.start + i
      token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index]
      tokens.append(all_doc_tokens[split_token_index])
      segment_ids.append(1)
    tokens.append("[SEP]")
    segment_ids.append(1)
    assert len(tokens) == len(segment_ids)

    input_ids = tokenizer.convert_tokens_to_ids(tokens)

    # The mask has 1 for real tokens and 0 for padding tokens. Only real
    # tokens are attended to.
    input_mask = [1] * len(input_ids)

    # Zero-pad up to the sequence length.
    padding = [0] * (FLAGS.max_seq_length - len(input_ids))
    input_ids.extend(padding)
    input_mask.extend(padding)
    segment_ids.extend(padding)

    assert len(input_ids) == FLAGS.max_seq_length
    assert len(input_mask) == FLAGS.max_seq_length
    assert len(segment_ids) == FLAGS.max_seq_length

    start_position = None
    end_position = None
    answer_type = None
    answer_text = ""
    if is_training:
      doc_start = doc_span.start
      doc_end = doc_span.start + doc_span.length - 1
      # For training, if our document chunk does not contain an annotation
      # we throw it out, since there is nothing to predict.
      contains_an_annotation = (
          tok_start_position >= doc_start and tok_end_position <= doc_end)
      if ((not contains_an_annotation) or
          example.answer.type == AnswerType.UNKNOWN):
        # If an example has unknown answer type or does not contain the answer
        # span, then we only include it with probability --include_unknowns.
        # When we include an example with unknown answer type, we set the first
        # token of the passage to be the annotated short span.
        if (FLAGS.include_unknowns < 0 or
            random.random() > FLAGS.include_unknowns):
          continue
        start_position = 0
        end_position = 0
        answer_type = AnswerType.UNKNOWN
      else:
        doc_offset = len(query_tokens) + 2
        start_position = tok_start_position - doc_start + doc_offset
        end_position = tok_end_position - doc_start + doc_offset
        answer_type = example.answer.type

      answer_text = " ".join(tokens[start_position:(end_position + 1)])

    feature = InputFeatures(
        unique_id=-1,
        example_index=-1,
        doc_span_index=doc_span_index,
        token_to_orig_map=token_to_orig_map,
        input_ids=input_ids,
        input_mask=input_mask,
        segment_ids=segment_ids,
        start_position=start_position,
        end_position=end_position,
        answer_text=answer_text,
        answer_type=answer_type)

    features.append(feature)

  return features

# A special token in NQ is made of non-space chars enclosed in square brackets.
_SPECIAL_TOKENS_RE = re.compile(r"^\[[^ ]*\]$", re.UNICODE)

def tokenize(tokenizer, text, apply_basic_tokenization=False):
  """Tokenizes text, optionally looking up special tokens separately.

  Args:
    tokenizer: a tokenizer from bert.tokenization.FullTokenizer
    text: text to tokenize
    apply_basic_tokenization: If True, apply the basic tokenization. If False,
      apply the full tokenization (basic + wordpiece).

  Returns:
    tokenized text.

  A special token is any text with no spaces enclosed in square brackets with no
  space, so we separate those out and look them up in the dictionary before
  doing actual tokenization.
  """
  tokenize_fn = tokenizer.tokenize
  if apply_basic_tokenization:
    tokenize_fn = tokenizer.basic_tokenizer.tokenize
  tokens = []
  for token in text.split(" "):
    if _SPECIAL_TOKENS_RE.match(token):
      if token in tokenizer.vocab:
        tokens.append(token)
      else:
        tokens.append(tokenizer.wordpiece_tokenizer.unk_token)
    else:
      tokens.extend(tokenize_fn(token))
  return tokens

class CreateTFExampleFn(object):
  """Functor for creating NQ tf.Examples."""

  def __init__(self, is_training):
    self.is_training = is_training
    self.tokenizer = tokenization.FullTokenizer(
        vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)

  def process(self, example):
    """Coverts an NQ example in a list of serialized tf examples."""
    nq_examples = read_nq_entry(example, self.is_training)
    input_features = []
    for nq_example in nq_examples:
      input_features.extend(
          convert_single_example(nq_example, self.tokenizer, self.is_training))

    for input_feature in input_features:
      input_feature.example_index = int(example["id"])
      input_feature.unique_id = (
          input_feature.example_index + input_feature.doc_span_index)

      def create_int_feature(values):
        return tf.train.Feature(
            int64_list=tf.train.Int64List(value=list(values)))

      features = collections.OrderedDict()
      features["unique_ids"] = create_int_feature([input_feature.unique_id])
      features["input_ids"] = create_int_feature(input_feature.input_ids)
      features["input_mask"] = create_int_feature(input_feature.input_mask)
      features["segment_ids"] = create_int_feature(input_feature.segment_ids)

      if self.is_training:
        features["start_positions"] = create_int_feature(
            [input_feature.start_position])
        features["end_positions"] = create_int_feature(
            [input_feature.end_position])
        features["answer_types"] = create_int_feature(
            [input_feature.answer_type])
      else:
        token_map = [-1] * len(input_feature.input_ids)
        for k, v in input_feature.token_to_orig_map.items():
          token_map[k] = v
        features["token_map"] = create_int_feature(token_map)

      yield tf.train.Example(features=tf.train.Features(
          feature=features)).SerializeToString()

class InputFeatures(object):
  """A single set of features of data."""

  def __init__(self,
               unique_id,
               example_index,
               doc_span_index,
               token_to_orig_map,
               input_ids,
               input_mask,
               segment_ids,
               start_position=None,
               end_position=None,
               answer_text="",
               answer_type=AnswerType.SHORT):
    self.unique_id = unique_id
    self.example_index = example_index
    self.doc_span_index = doc_span_index
    self.token_to_orig_map = token_to_orig_map
    self.input_ids = input_ids
    self.input_mask = input_mask
    self.segment_ids = segment_ids
    self.start_position = start_position
    self.end_position = end_position
    self.answer_text = answer_text
    self.answer_type = answer_type

def read_nq_examples(input_file, is_training):
  """Read a NQ json file into a list of NqExample."""
  input_paths = tf.gfile.Glob(input_file)
  input_data = []

  def _open(path):
    if path.endswith(".gz"):
      return gzip.GzipFile(fileobj=tf.gfile.Open(path, "rb"))
    else:
      return tf.gfile.Open(path, "r")

  for path in input_paths:
    tf.logging.info("Reading: %s", path)
    with _open(path) as input_file:
      for line in input_file:
        input_data.append(create_example_from_jsonl(line))

  examples = []
  for entry in input_data:
    examples.extend(read_nq_entry(entry, is_training))
  return examples

def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
                 use_one_hot_embeddings):
  """Creates a classification model."""
  model = modeling.BertModel(
      config=bert_config,
      is_training=is_training,
      input_ids=input_ids,
      input_mask=input_mask,
      token_type_ids=segment_ids,
      use_one_hot_embeddings=use_one_hot_embeddings)

  # Get the logits for the start and end predictions.
  final_hidden = model.get_sequence_output()

  final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3)
  batch_size = final_hidden_shape[0]
  seq_length = final_hidden_shape[1]
  hidden_size = final_hidden_shape[2]

  output_weights = tf.get_variable(
      "cls/nq/output_weights", [2, hidden_size],
      initializer=tf.truncated_normal_initializer(stddev=0.02))

  output_bias = tf.get_variable(
      "cls/nq/output_bias", [2], initializer=tf.zeros_initializer())

  final_hidden_matrix = tf.reshape(final_hidden,
                                   [batch_size * seq_length, hidden_size])
  logits = tf.matmul(final_hidden_matrix, output_weights, transpose_b=True)
  logits = tf.nn.bias_add(logits, output_bias)

  logits = tf.reshape(logits, [batch_size, seq_length, 2])
  logits = tf.transpose(logits, [2, 0, 1])

  unstacked_logits = tf.unstack(logits, axis=0)

  (start_logits, end_logits) = (unstacked_logits[0], unstacked_logits[1])

  # Get the logits for the answer type prediction.
  answer_type_output_layer = model.get_pooled_output()
  answer_type_hidden_size = answer_type_output_layer.shape[-1].value

  num_answer_types = 5  # YES, NO, UNKNOWN, SHORT, LONG
  answer_type_output_weights = tf.get_variable(
      "answer_type_output_weights", [num_answer_types, answer_type_hidden_size],
      initializer=tf.truncated_normal_initializer(stddev=0.02))

  answer_type_output_bias = tf.get_variable(
      "answer_type_output_bias", [num_answer_types],
      initializer=tf.zeros_initializer())

  answer_type_logits = tf.matmul(
      answer_type_output_layer, answer_type_output_weights, transpose_b=True)
  answer_type_logits = tf.nn.bias_add(answer_type_logits,
                                      answer_type_output_bias)

  return (start_logits, end_logits, answer_type_logits)

def model_fn_builder(bert_config, init_checkpoint, learning_rate,
                     num_train_steps, num_warmup_steps, use_tpu,
                     use_one_hot_embeddings):
  """Returns `model_fn` closure for TPUEstimator."""

  def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument
    """The `model_fn` for TPUEstimator."""

    tf.logging.info("*** Features ***")
    for name in sorted(features.keys()):
      tf.logging.info("  name = %s, shape = %s" % (name, features[name].shape))

    unique_ids = features["unique_ids"]
    input_ids = features["input_ids"]
    input_mask = features["input_mask"]
    segment_ids = features["segment_ids"]

    is_training = (mode == tf_estimator.ModeKeys.TRAIN)

    (start_logits, end_logits, answer_type_logits) = create_model(
        bert_config=bert_config,
        is_training=is_training,
        input_ids=input_ids,
        input_mask=input_mask,
        segment_ids=segment_ids,
        use_one_hot_embeddings=use_one_hot_embeddings)

    tvars = tf.trainable_variables()

    initialized_variable_names = {}
    scaffold_fn = None
    if init_checkpoint:
      (assignment_map, initialized_variable_names
      ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)
      if use_tpu:

        def tpu_scaffold():
          tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
          return tf.train.Scaffold()

        scaffold_fn = tpu_scaffold
      else:
        tf.train.init_from_checkpoint(init_checkpoint, assignment_map)

    tf.logging.info("**** Trainable Variables ****")
    for var in tvars:
      init_string = ""
      if var.name in initialized_variable_names:
        init_string = ", *INIT_FROM_CKPT*"
      tf.logging.info("  name = %s, shape = %s%s", var.name, var.shape,
                      init_string)

    output_spec = None
    if mode == tf_estimator.ModeKeys.TRAIN:
      seq_length = modeling.get_shape_list(input_ids)[1]

      # Computes the loss for positions.
      def compute_loss(logits, positions):
        one_hot_positions = tf.one_hot(
            positions, depth=seq_length, dtype=tf.float32)
        log_probs = tf.nn.log_softmax(logits, axis=-1)
        loss = -tf.reduce_mean(
            tf.reduce_sum(one_hot_positions * log_probs, axis=-1))
        return loss

      # Computes the loss for labels.
      def compute_label_loss(logits, labels):
        one_hot_labels = tf.one_hot(
            labels, depth=len(AnswerType), dtype=tf.float32)
        log_probs = tf.nn.log_softmax(logits, axis=-1)
        loss = -tf.reduce_mean(
            tf.reduce_sum(one_hot_labels * log_probs, axis=-1))
        return loss

      start_positions = features["start_positions"]
      end_positions = features["end_positions"]
      answer_types = features["answer_types"]

      start_loss = compute_loss(start_logits, start_positions)
      end_loss = compute_loss(end_logits, end_positions)
      answer_type_loss = compute_label_loss(answer_type_logits, answer_types)

      total_loss = (start_loss + end_loss + answer_type_loss) / 3.0

      train_op = optimization.create_optimizer(total_loss, learning_rate,
                                               num_train_steps,
                                               num_warmup_steps, use_tpu)

      output_spec = contrib_tpu.TPUEstimatorSpec(
          mode=mode,
          loss=total_loss,
          train_op=train_op,
          scaffold_fn=scaffold_fn)
    elif mode == tf_estimator.ModeKeys.PREDICT:
      predictions = {
          "unique_ids": unique_ids,
          "start_logits": start_logits,
          "end_logits": end_logits,
          "answer_type_logits": answer_type_logits,
      }
      output_spec = contrib_tpu.TPUEstimatorSpec(
          mode=mode, predictions=predictions, scaffold_fn=scaffold_fn)
    else:
      raise ValueError("Only TRAIN and PREDICT modes are supported: %s" %
                       (mode))

    return output_spec

  return model_fn

def input_fn_builder(input_file, seq_length, is_training, drop_remainder):
  """Creates an `input_fn` closure to be passed to TPUEstimator."""

  name_to_features = {
      "unique_ids": tf.FixedLenFeature([], tf.int64),
      "input_ids": tf.FixedLenFeature([seq_length], tf.int64),
      "input_mask": tf.FixedLenFeature([seq_length], tf.int64),
      "segment_ids": tf.FixedLenFeature([seq_length], tf.int64),
  }

  if is_training:
    name_to_features["start_positions"] = tf.FixedLenFeature([], tf.int64)
    name_to_features["end_positions"] = tf.FixedLenFeature([], tf.int64)
    name_to_features["answer_types"] = tf.FixedLenFeature([], tf.int64)

  def _decode_record(record, name_to_features):
    """Decodes a record to a TensorFlow example."""
    example = tf.parse_single_example(record, name_to_features)

    # tf.Example only supports tf.int64, but the TPU only supports tf.int32.
    # So cast all int64 to int32.
    for name in list(example.keys()):
      t = example[name]
      if t.dtype == tf.int64:
        t = tf.to_int32(t)
      example[name] = t

    return example

  def input_fn(params):
    """The actual input function."""
    batch_size = params["batch_size"]

    # For training, we want a lot of parallel reading and shuffling.
    # For eval, we want no shuffling and parallel reading doesn't matter.
    d = tf.data.TFRecordDataset(input_file)
    if is_training:
      d = d.repeat()
      d = d.shuffle(buffer_size=100)

    d = d.apply(
        tf.data.experimental.map_and_batch(
            lambda record: _decode_record(record, name_to_features),
            batch_size=batch_size,
            drop_remainder=drop_remainder))

    return d

  return input_fn

RawResult = collections.namedtuple(
    "RawResult",
    ["unique_id", "start_logits", "end_logits", "answer_type_logits"])

class FeatureWriter(object):
  """Writes InputFeature to TF example file."""

  def __init__(self, filename, is_training):
    self.filename = filename
    self.is_training = is_training
    self.num_features = 0
    self._writer = tf.python_io.TFRecordWriter(filename)

  def process_feature(self, feature):
    """Write a InputFeature to the TFRecordWriter as a tf.train.Example."""
    self.num_features += 1

    def create_int_feature(values):
      feature = tf.train.Feature(
          int64_list=tf.train.Int64List(value=list(values)))
      return feature

    features = collections.OrderedDict()
    features["unique_ids"] = create_int_feature([feature.unique_id])
    features["input_ids"] = create_int_feature(feature.input_ids)
    features["input_mask"] = create_int_feature(feature.input_mask)
    features["segment_ids"] = create_int_feature(feature.segment_ids)

    if self.is_training:
      features["start_positions"] = create_int_feature([feature.start_position])
      features["end_positions"] = create_int_feature([feature.end_position])
      features["answer_types"] = create_int_feature([feature.answer_type])
    else:
      token_map = [-1] * len(feature.input_ids)
      for k, v in feature.token_to_orig_map.items():
        token_map[k] = v
      features["token_map"] = create_int_feature(token_map)

    tf_example = tf.train.Example(features=tf.train.Features(feature=features))
    self._writer.write(tf_example.SerializeToString())

  def close(self):
    self._writer.close()

Span = collections.namedtuple("Span", ["start_token_idx", "end_token_idx"])

class EvalExample(object):
  """Eval data available for a single example."""

  def __init__(self, example_id, candidates):
    self.example_id = example_id
    self.candidates = candidates
    self.results = {}
    self.features = {}

class ScoreSummary(object):

  def __init__(self):
    self.predicted_label = None
    self.short_span_score = None
    self.cls_token_score = None
    self.answer_type_logits = None

  def __lt__(self, other):
    return ((self.predicted_label, self.short_span_score, self.cls_token_score,
             self.answer_type_logits) <
            (other.predicted_label, other.short_span_score,
             other.cls_token_score, other.answer_type_logits))

def read_candidates_from_one_split(input_path):
  """Read candidates from a single jsonl file."""
  candidates_dict = {}
  with gzip.GzipFile(fileobj=tf.gfile.Open(input_path, "rb")) as input_file:
    tf.logging.info("Reading examples from: %s", input_path)
    for line in input_file:
      e = json.loads(line)
      candidates_dict[e["example_id"]] = e["long_answer_candidates"]
  return candidates_dict

def read_candidates(input_pattern):
  """Read candidates with real multiple processes."""
  input_paths = tf.gfile.Glob(input_pattern)
  final_dict = {}
  for input_path in input_paths:
    final_dict.update(read_candidates_from_one_split(input_path))
  return final_dict

def get_best_indexes(logits, n_best_size):
  """Get the n-best logits from a list."""
  index_and_score = sorted(
      enumerate(logits[1:], 1), key=lambda x: x[1], reverse=True)
  best_indexes = []
  for i in range(len(index_and_score)):
    if i >= n_best_size:
      break
    best_indexes.append(index_and_score[i][0])
  return best_indexes

def compute_predictions(example):
  """Converts an example into an NQEval object for evaluation."""
  predictions = []
  n_best_size = 10
  max_answer_length = 30

  for unique_id, result in example.results.items():
    if unique_id not in example.features:
      raise ValueError("No feature found with unique_id:", unique_id)
    token_map = example.features[unique_id]["token_map"].int64_list.value
    start_indexes = get_best_indexes(result["start_logits"], n_best_size)
    end_indexes = get_best_indexes(result["end_logits"], n_best_size)
    for start_index in start_indexes:
      for end_index in end_indexes:
        if end_index < start_index:
          continue
        if token_map[start_index] == -1:
          continue
        if token_map[end_index] == -1:
          continue
        length = end_index - start_index + 1
        if length > max_answer_length:
          continue
        summary = ScoreSummary()
        summary.short_span_score = (
            result["start_logits"][start_index] +
            result["end_logits"][end_index])
        summary.cls_token_score = (
            result["start_logits"][0] + result["end_logits"][0])
        summary.answer_type_logits = result["answer_type_logits"]
        start_span = token_map[start_index]
        end_span = token_map[end_index] + 1

        # Span logits minus the cls logits seems to be close to the best.
        score = summary.short_span_score - summary.cls_token_score
        predictions.append((score, summary, start_span, end_span))

  # Default empty prediction.
  score = -10000.0
  short_span = Span(-1, -1)
  long_span = Span(-1, -1)
  summary = ScoreSummary()

  if predictions:
    score, summary, start_span, end_span = sorted(predictions, reverse=True)[0]
    short_span = Span(start_span, end_span)
    for c in example.candidates:
      start = short_span.start_token_idx
      end = short_span.end_token_idx
      if c["top_level"] and c["start_token"] <= start and c["end_token"] >= end:
        long_span = Span(c["start_token"], c["end_token"])
        break

  summary.predicted_label = {
      "example_id": example.example_id,
      "long_answer": {
          "start_token": long_span.start_token_idx,
          "end_token": long_span.end_token_idx,
          "start_byte": -1,
          "end_byte": -1
      },
      "long_answer_score": score,
      "short_answers": [{
          "start_token": short_span.start_token_idx,
          "end_token": short_span.end_token_idx,
          "start_byte": -1,
          "end_byte": -1
      }],
      "short_answers_score": score,
      "yes_no_answer": "NONE"
  }

  return summary

def compute_pred_dict(candidates_dict, dev_features, raw_results):
  """Computes official answer key from raw logits."""
  raw_results_by_id = [(int(res["unique_id"] + 1), res) for res in raw_results]

  # Cast example id to int32 for each example, similarly to the raw results.
  sess = tf.Session()
  all_candidates = candidates_dict.items()
  example_ids = tf.to_int32(np.array([int(k) for k, _ in all_candidates
                                     ])).eval(session=sess)
  examples_by_id = zip(example_ids, all_candidates)

  # Cast unique_id also to int32 for features.
  feature_ids = []
  features = []
  for f in dev_features:
    feature_ids.append(f.features.feature["unique_ids"].int64_list.value[0] + 1)
    features.append(f.features.feature)
  feature_ids = tf.to_int32(np.array(feature_ids)).eval(session=sess)
  features_by_id = zip(feature_ids, features)

  # Join examples with features and raw results.
  examples = []
  merged = sorted(
      list(examples_by_id) + raw_results_by_id + list(features_by_id),
      key=lambda x: x[0])
  for idx, datum in merged:
    if isinstance(datum, tuple):
      examples.append(EvalExample(datum[0], datum[1]))
    elif "token_map" in datum:
      examples[-1].features[idx] = datum
    else:
      examples[-1].results[idx] = datum

  # Construct prediction objects.
  tf.logging.info("Computing predictions...")
  summary_dict = {}
  nq_pred_dict = {}
  for e in examples:
    summary = compute_predictions(e)
    summary_dict[e.example_id] = summary
    nq_pred_dict[e.example_id] = summary.predicted_label
    if len(nq_pred_dict) % 100 == 0:
      tf.logging.info("Examples processed: %d", len(nq_pred_dict))
  tf.logging.info("Done computing predictions.")

  return nq_pred_dict

def validate_flags_or_throw(bert_config):
  """Validate the input FLAGS or throw an exception."""
  if not FLAGS.do_train and not FLAGS.do_predict:
    raise ValueError("At least one of `{do_train,do_predict}` must be True.")

  if FLAGS.do_train:
    if not FLAGS.train_precomputed_file:
      raise ValueError("If `do_train` is True, then `train_precomputed_file` "
                       "must be specified.")
    if not FLAGS.train_num_precomputed:
      raise ValueError("If `do_train` is True, then `train_num_precomputed` "
                       "must be specified.")

  if FLAGS.do_predict:
    if not FLAGS.predict_file:
      raise ValueError(
          "If `do_predict` is True, then `predict_file` must be specified.")

  if FLAGS.max_seq_length > bert_config.max_position_embeddings:
    raise ValueError(
        "Cannot use sequence length %d because the BERT model "
        "was only trained up to sequence length %d" %
        (FLAGS.max_seq_length, bert_config.max_position_embeddings))

  if FLAGS.max_seq_length <= FLAGS.max_query_length + 3:
    raise ValueError(
        "The max_seq_length (%d) must be greater than max_query_length "
        "(%d) + 3" % (FLAGS.max_seq_length, FLAGS.max_query_length))

def main(_):
  tf.logging.set_verbosity(tf.logging.INFO)
  bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file)
  validate_flags_or_throw(bert_config)
  tf.gfile.MakeDirs(FLAGS.output_dir)

  tokenizer = tokenization.FullTokenizer(
      vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case)

  tpu_cluster_resolver = None
  if FLAGS.use_tpu and FLAGS.tpu_name:
    tpu_cluster_resolver = contrib_cluster_resolver.TPUClusterResolver(
        FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)

  is_per_host = contrib_tpu.InputPipelineConfig.PER_HOST_V2
  run_config = contrib_tpu.RunConfig(
      cluster=tpu_cluster_resolver,
      master=FLAGS.master,
      model_dir=FLAGS.output_dir,
      save_checkpoints_steps=FLAGS.save_checkpoints_steps,
      tpu_config=contrib_tpu.TPUConfig(
          iterations_per_loop=FLAGS.iterations_per_loop,
          num_shards=FLAGS.num_tpu_cores,
          per_host_input_for_training=is_per_host))

  num_train_steps = None
  num_warmup_steps = None
  if FLAGS.do_train:
    num_train_features = FLAGS.train_num_precomputed
    num_train_steps = int(num_train_features / FLAGS.train_batch_size *
                          FLAGS.num_train_epochs)

    num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion)

  model_fn = model_fn_builder(
      bert_config=bert_config,
      init_checkpoint=FLAGS.init_checkpoint,
      learning_rate=FLAGS.learning_rate,
      num_train_steps=num_train_steps,
      num_warmup_steps=num_warmup_steps,
      use_tpu=FLAGS.use_tpu,
      use_one_hot_embeddings=FLAGS.use_tpu)

  # If TPU is not available, this falls back to normal Estimator on CPU or GPU.
  estimator = contrib_tpu.TPUEstimator(
      use_tpu=FLAGS.use_tpu,
      model_fn=model_fn,
      config=run_config,
      train_batch_size=FLAGS.train_batch_size,
      predict_batch_size=FLAGS.predict_batch_size)

  if FLAGS.do_train:
    tf.logging.info("***** Running training on precomputed features *****")
    tf.logging.info("  Num split examples = %d", num_train_features)
    tf.logging.info("  Batch size = %d", FLAGS.train_batch_size)
    tf.logging.info("  Num steps = %d", num_train_steps)
    train_filenames = tf.gfile.Glob(FLAGS.train_precomputed_file)
    train_input_fn = input_fn_builder(
        input_file=train_filenames,
        seq_length=FLAGS.max_seq_length,
        is_training=True,
        drop_remainder=True)
    estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)

  if FLAGS.do_predict:
    if not FLAGS.output_prediction_file:
      raise ValueError(
          "--output_prediction_file must be defined in predict mode.")

    eval_examples = read_nq_examples(
        input_file=FLAGS.predict_file, is_training=False)
    eval_writer = FeatureWriter(
        filename=os.path.join(FLAGS.output_dir, "eval.tf_record"),
        is_training=False)
    eval_features = []

    def append_feature(feature):
      eval_features.append(feature)
      eval_writer.process_feature(feature)

    num_spans_to_ids = convert_examples_to_features(
        examples=eval_examples,
        tokenizer=tokenizer,
        is_training=False,
        output_fn=append_feature)
    eval_writer.close()
    eval_filename = eval_writer.filename

    tf.logging.info("***** Running predictions *****")
    tf.logging.info("  Num orig examples = %d", len(eval_examples))
    tf.logging.info("  Num split examples = %d", len(eval_features))
    tf.logging.info("  Batch size = %d", FLAGS.predict_batch_size)
    for spans, ids in num_spans_to_ids.items():
      tf.logging.info("  Num split into %d = %d", spans, len(ids))

    predict_input_fn = input_fn_builder(
        input_file=eval_filename,
        seq_length=FLAGS.max_seq_length,
        is_training=False,
        drop_remainder=False)

    # If running eval on the TPU, you will need to specify the number of steps.
    all_results = []
    for result in estimator.predict(
        predict_input_fn, yield_single_examples=True):
      if len(all_results) % 1000 == 0:
        tf.logging.info("Processing example: %d" % (len(all_results)))
      unique_id = int(result["unique_ids"])
      start_logits = [float(x) for x in result["start_logits"].flat]
      end_logits = [float(x) for x in result["end_logits"].flat]
      answer_type_logits = [float(x) for x in result["answer_type_logits"].flat]
      all_results.append(
          RawResult(
              unique_id=unique_id,
              start_logits=start_logits,
              end_logits=end_logits,
              answer_type_logits=answer_type_logits))

    candidates_dict = read_candidates(FLAGS.predict_file)
    eval_features = [
        tf.train.Example.FromString(r)
        for r in tf.python_io.tf_record_iterator(eval_filename)
    ]
    nq_pred_dict = compute_pred_dict(candidates_dict, eval_features,
                                     [r._asdict() for r in all_results])
    predictions_json = {"predictions": list(nq_pred_dict.values())}
    with tf.gfile.Open(FLAGS.output_prediction_file, "w") as f:
      json.dump(predictions_json, f, indent=4)

if __name__ == "__main__":
  flags.mark_flag_as_required("vocab_file")
  flags.mark_flag_as_required("bert_config_file")
  flags.mark_flag_as_required("output_dir")
  tf.app.run()


text:
import os, re, sys
try:
    SetType = set
except NameError:
    import sets
    SetType = sets.Set
    set = sets.Set

_defaults = None
def read_default(name=None):
    global _defaults
    from ConfigParser import SafeConfigParser, NoOptionError
    if not _defaults:
        if os.path.exists('/usr/share/python/debian_defaults'):
            config = SafeConfigParser()
            try:
                config.readfp(file('/usr/share/python/debian_defaults'))
            except IOError, msg:
                print msg
                sys.exit(1)
            _defaults = config
    if _defaults and name:
        try:
            value = _defaults.get('DEFAULT', name)
        except NoOptionError:
            raise ValueError
        return value
    return None

def parse_versions(vstring, add_exact=False):
    import operator
    operators = { None: operator.eq, '=': operator.eq,
                  '>=': operator.ge, '<=': operator.le,
                  '<<': operator.lt
                  }
    vinfo = {}
    exact_versions = set([])
    version_range = set(supported_versions(version_only=True)
                        + old_versions(version_only=True))
    relop_seen = False
    for field in vstring.split(','):
        field = field.strip()
        if field == 'all':
            vinfo['all'] = 'all'
            continue
        if field in ('current', 'current_ext'):
            vinfo['current'] = field
            continue
        vinfo.setdefault('versions', set())
        ve = re.compile('(>=|<=|<<|=)? *(\d\.\d)$')
        m = ve.match(field)
        try:
            if not m:
                raise ValueError('error parsing Python-Version attribute')
            op, v = m.group(1), m.group(2)
            vmaj, vmin = v.split('.')
            # Don't silently ignore Python 3 versions for Squeeze.
            #if int(vmaj) > 2:
            #    continue
            if op in (None, '='):
                exact_versions.add(v)
            else:
                relop_seen = True
                filtop = operators[op]
                version_range = [av for av in version_range if filtop(av ,v)]
        except Exception:
            raise ValueError, 'error parsing Python-Version attribute'
    if add_exact:
        if exact_versions:
            vinfo['vexact'] = exact_versions
        if 'versions' in vinfo:
            if relop_seen:
                vinfo['versions'] = set(version_range)
            else:
                del vinfo['versions']
    else:
        if 'versions' in vinfo:
            vinfo['versions'] = exact_versions
            if relop_seen:
                vinfo['versions'] = exact_versions.union(version_range)
    return vinfo

_old_versions = None
def old_versions(version_only=False):
    global _old_versions
    if not _old_versions:
        try:
            value = read_default('old-versions')
            _old_versions = [s.strip() for s in value.split(',')]
        except ValueError:
            _old_versions = []
    if version_only:
        return [v[6:] for v in _old_versions]
    else:
        return _old_versions

_unsupported_versions = None
def unsupported_versions(version_only=False):
    global _unsupported_versions
    if not _unsupported_versions:
        try:
            value = read_default('unsupported-versions')
            _unsupported_versions = [s.strip() for s in value.split(',')]
        except ValueError:
            _unsupported_versions = []
    if version_only:
        return [v[6:] for v in _unsupported_versions]
    else:
        return _unsupported_versions

_supported_versions = ["python%s" % ver.strip() for ver in
                       os.environ.get('DEBPYTHON_SUPPORTED', '').split(',')
                       if ver.strip()]
def supported_versions(version_only=False):
    global _supported_versions
    if not _supported_versions:
        try:
            value = read_default('supported-versions')
            _supported_versions = [s.strip() for s in value.split(',')]
        except ValueError:
            cmd = ['/usr/bin/apt-cache', '--no-all-versions',
                   'show', 'python-all']
            try:
                import subprocess
                p = subprocess.Popen(cmd, bufsize=1,
                                     shell=False, stdout=subprocess.PIPE)
                fd = p.stdout
            except ImportError:
                fd = os.popen(' '.join(cmd))
            depends = None
            for line in fd:
                if line.startswith('Depends:'):
                    depends = line.split(':', 1)[1].strip().split(',')
            fd.close()
            if depends:
                depends = [re.sub(r'\s*(\S+)[ (]?.*', r'\1', s) for s in depends]
                _supported_versions = depends
            if not _supported_versions:
                # last resort: python-minimal not installed, apt-cache
                # not available, hard code the value, #394084
                _supported_versions = ['python2.6', 'python2.7']
    if version_only:
        return [v[6:] for v in _supported_versions]
    else:
        return _supported_versions

_default_version = "python%s" % os.environ.get('DEBPYTHON_DEFAULT', '')
if _default_version == 'python':
    _default_version = None
def default_version(version_only=False):
    global _default_version
    if not _default_version:
        try:
            _default_version = link = os.readlink('/usr/bin/python')
        except OSError:
            _default_version = None
            try:
                cmd = ['/usr/bin/python', '-c', 'import sys; print sys.version[:3]']
                import subprocess
                p = subprocess.Popen(cmd, bufsize=1,
                                     shell=False, stdout=subprocess.PIPE)
                fd = p.stdout
            except ImportError:
                fd = os.popen("/usr/bin/python -c 'import sys; print sys.version[:3]'")
            line = fd.readline().strip()
            fd.close()
            if re.match(r'\d\.\d$', line):
                _default_version = 'python' + line
        # consistency check
        try:
            debian_default = read_default('default-version')
        except ValueError:
            debian_default = "python2.6"
        if not _default_version in (debian_default, os.path.join('/usr/bin', debian_default)):
            raise ValueError, "/usr/bin/python does not match the python default version. It must be reset to point to %s" % debian_default
        _default_version = debian_default
    if version_only:
        return _default_version[6:]
    else:
        return _default_version

def requested_versions(vstring, version_only=False):
    versions = None
    vinfo = parse_versions(vstring, add_exact=True)
    supported = supported_versions(version_only=True)
    if len(vinfo) == 1:
        if 'all' in vinfo:
            versions = supported
        elif 'current' in vinfo:
            versions = [default_version(version_only=True)]
        elif 'vexact' in vinfo:
            versions = vinfo['vexact']
        else:
            versions = vinfo['versions'].intersection(supported)
    elif 'all' in vinfo and 'current' in vinfo:
        raise ValueError, "both `current' and `all' in version string"
    elif 'all' in vinfo:
        if 'versions' in vinfo:
            versions = vinfo['versions'].intersection(supported)
        else:
            versions = set(supported)
        if 'vexact' in vinfo:
            versions.update(vinfo['vexact'])
    elif 'current' in vinfo:
        current = default_version(version_only=True)
        if not current in vinfo['versions']:
            raise ValueError, "`current' version not in supported versions"
        versions = [current]
    elif 'versions' in vinfo or 'vexact' in vinfo:
        versions = set()
        if 'versions' in vinfo:
            versions = vinfo['versions'].intersection(supported)
        if 'vexact' in vinfo:
            versions.update(vinfo['vexact'])
    else:
        raise ValueError, 'No Python versions in version string'
    if not versions:
        raise ValueError('computed set of supported versions is empty')
    if version_only:
        return versions
    else:
        return ['python%s' % v for v in versions]

def installed_versions(version_only=False):
    import glob
    supported = supported_versions()
    versions = [os.path.basename(s)
                for s in glob.glob('/usr/bin/python[0-9].[0-9]')
                if os.path.basename(s) in supported]
    versions.sort()
    if version_only:
        return [v[6:] for v in versions]
    else:
        return versions

class ControlFileValueError(ValueError):
    pass
class MissingVersionValueError(ValueError):
    pass

def extract_pyversion_attribute(fn, pkg):
    """read the debian/control file, extract the X-Python-Version or
    XS-Python-Version field; check that XB-Python-Version exists for the
    package."""

    version = None
    sversion = None
    section = None
    try:
        fp = file(fn, 'r')
    except IOError, msg:
        print "Cannot open %s: %s" % (fn, msg)
        sys.exit(2)
    for line in fp:
        line = line.strip()
        if line == '':
            if section == None:
                continue
            if pkg == 'Source':
                break
            section = None
        elif line.startswith('Source:'):
            section = 'Source'
        elif line.startswith('Package: ' + pkg):
            section = pkg
        elif line.lower().startswith(('xs-python-version:', 'x-python-version:')):
            if section != 'Source':
                raise ValueError, \
                      'attribute X(S)-Python-Version not in Source section'
            sversion = line.split(':', 1)[1].strip()
        elif line.lower().startswith('xb-python-version:'):
            if section == pkg:
                version = line.split(':', 1)[1].strip()
    if section == None:
        raise ControlFileValueError, 'not a control file'
    if pkg == 'Source':
        if sversion == None:
            raise MissingVersionValueError, \
                  'no X(S)-Python-Version in control file'
        return sversion
    if version == None:
        raise MissingVersionValueError, \
              'no XB-Python-Version for package `%s' % pkg
    return version

# compatibility functions to parse debian/pyversions

def version_cmp(ver1,ver2):
    v1=[int(i) for i in ver1.split('.')]
    v2=[int(i) for i in ver2.split('.')]
    return cmp(v1,v2)

def requested_versions_bis(vstring, version_only=False):
    versions = []
    py_supported_short = supported_versions(version_only=True)
    for item in vstring.split(','):
        v=item.split('-')
        if len(v)>1:
            if not v[0]:
                v[0] = py_supported_short[0]
            if not v[1]:
                v[1] = py_supported_short[-1]
            for ver in py_supported_short:
                try:
                    if version_cmp(ver,v[0]) >= 0 \
                           and version_cmp(ver,v[1]) <= 0:
                        versions.append(ver)
                except ValueError:
                    pass
        else:
            if v[0] in py_supported_short:
                versions.append(v[0])
    versions.sort(version_cmp)
    if not versions:
        raise ValueError, 'empty set of versions'
    if not version_only:
        versions=['python'+i for i in versions]
    return versions

def extract_pyversion_attribute_bis(fn):
    vstring = file(fn).readline().rstrip('\n')
    return vstring

def main():
    from optparse import OptionParser
    usage = '[-v] [-h] [-d|--default] [-s|--supported] [-i|--installed] [-r|--requested <version string>|<control file>]'
    parser = OptionParser(usage=usage)
    parser.add_option('-d', '--default',
                      help='print the default python version',
                      action='store_true', dest='default')
    parser.add_option('-s', '--supported',
                      help='print the supported python versions',
                      action='store_true', dest='supported')
    parser.add_option('-r', '--requested',
                      help='print the python versions requested by a build; the argument is either the name of a control file or the value of the X(S)-Python-Version attribute',
                      action='store_true', dest='requested')
    parser.add_option('-i', '--installed',
                      help='print the installed supported python versions',
                      action='store_true', dest='installed')
    parser.add_option('-v', '--version',
                      help='print just the version number(s)',
                      default=False, action='store_true', dest='version_only')
    opts, args = parser.parse_args()
    program = os.path.basename(sys.argv[0])

    if opts.default and len(args) == 0:
        try:
            print default_version(opts.version_only)
        except ValueError, msg:
            print "%s:" % program, msg
            sys.exit(1)
    elif opts.supported and len(args) == 0:
        print ' '.join(supported_versions(opts.version_only))
    elif opts.installed and len(args) == 0:
        print ' '.join(installed_versions(opts.version_only))
    elif opts.requested and len(args) <= 1:
        if len(args) == 0:
            versions = 'debian/control'
        else:
            versions = args[0]
        try:
            if os.path.isfile(versions):
                fn = versions
                try:
                    vstring = extract_pyversion_attribute(fn, 'Source')
                    vs = requested_versions(vstring, opts.version_only)
                except ControlFileValueError:
                    sys.stderr.write("%s: not a control file: %s, " \
                                     % (program, fn))
                    sys.exit(1)
                except MissingVersionValueError:
                    fn = os.path.join(os.path.dirname(fn), 'pyversions')
                    sys.stderr.write("%s: missing X(S)-Python-Version in control file, fall back to %s\n" \
                                     % (program, fn))
                    try:
                        vstring = extract_pyversion_attribute_bis(fn)
                        vs = requested_versions_bis(vstring, opts.version_only)
                    except IOError:
                        sys.stderr.write("%s: missing debian/pyversions file, fall back to supported versions\n" \
                                         % program)
                        vs = supported_versions(opts.version_only)
                except ValueError, e:
                    sys.stderr.write("%s: %s\n" % (program, e))
                    sys.exit(4)
            else:
                vs = requested_versions(versions, opts.version_only)
            print ' '.join(vs)
        except ValueError, msg:
            sys.stderr.write("%s: %s\n" % (program, msg))
            sys.exit(1)
    else:
        sys.stderr.write("usage: %s %s\n" % (program, usage))
        sys.exit(1)

if __name__ == '__main__':
    main()

redpajama_set_name: RedPajamaGithub


text:
package org.javalite.activejdbc.test_models;

import org.javalite.activejdbc.Model;
import org.javalite.activejdbc.annotations.UnrelatedTo;

/**
 * @author igor on 5/31/17.
 */

@UnrelatedTo({Fruit.class})
public class Box extends Model{
}


text:
package org.keycloak.dom.saml.v2.protocol;

import org.keycloak.dom.saml.common.CommonStatusDetailType;

/**
 *

 * Java class for StatusDetailType complex type.
 *
 *

 * The following schema fragment specifies the expected content contained within this class.
 *
 *

 * <complexType name="StatusDetailType">
 *   <complexContent>
 *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
 *       <sequence>
 *         <any/>
 *       </sequence>
 *     </restriction>
 *   </complexContent>
 * </complexType>
 *

 */
public class StatusDetailType extends CommonStatusDetailType {

}


text:
package com.amazonaws.services.inspector2.model;

import java.io.Serializable;
import javax.annotation.Generated;

/**
 *
 * @see AWS API
 *      Documentation
 */
@Generated("com.amazonaws:aws-java-sdk-code-generator")
public class GetConfigurationResult extends com.amazonaws.AmazonWebServiceResult<com.amazonaws.ResponseMetadata> implements Serializable, Cloneable {

    /**
     *

     * Specifies how the ECR automated re-scan duration is currently configured for your environment.
     *

     */
    private EcrConfigurationState ecrConfiguration;

    /**
     *

     * Specifies how the ECR automated re-scan duration is currently configured for your environment.
     *

     *
     * @param ecrConfiguration
     *        Specifies how the ECR automated re-scan duration is currently configured for your environment.
     */

    public void setEcrConfiguration(EcrConfigurationState ecrConfiguration) {
        this.ecrConfiguration = ecrConfiguration;
    }

    /**
     *

     * Specifies how the ECR automated re-scan duration is currently configured for your environment.
     *

     *
     * @return Specifies how the ECR automated re-scan duration is currently configured for your environment.
     */

    public EcrConfigurationState getEcrConfiguration() {
        return this.ecrConfiguration;
    }

    /**
     *

     * Specifies how the ECR automated re-scan duration is currently configured for your environment.
     *

     *
     * @param ecrConfiguration
     *        Specifies how the ECR automated re-scan duration is currently configured for your environment.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public GetConfigurationResult withEcrConfiguration(EcrConfigurationState ecrConfiguration) {
        setEcrConfiguration(ecrConfiguration);
        return this;
    }

    /**
     * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be
     * redacted from this string using a placeholder value.
     *
     * @return A string representation of this object.
     *
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append("{");
        if (getEcrConfiguration() != null)
            sb.append("EcrConfiguration: ").append(getEcrConfiguration());
        sb.append("}");
        return sb.toString();
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;

        if (obj instanceof GetConfigurationResult == false)
            return false;
        GetConfigurationResult other = (GetConfigurationResult) obj;
        if (other.getEcrConfiguration() == null ^ this.getEcrConfiguration() == null)
            return false;
        if (other.getEcrConfiguration() != null && other.getEcrConfiguration().equals(this.getEcrConfiguration()) == false)
            return false;
        return true;
    }

    @Override
    public int hashCode() {
        final int prime = 31;
        int hashCode = 1;

        hashCode = prime * hashCode + ((getEcrConfiguration() == null) ? 0 : getEcrConfiguration().hashCode());
        return hashCode;
    }

    @Override
    public GetConfigurationResult clone() {
        try {
            return (GetConfigurationResult) super.clone();
        } catch (CloneNotSupportedException e) {
            throw new IllegalStateException("Got a CloneNotSupportedException from Object.clone() " + "even though we're Cloneable!", e);
        }
    }

}


text:
from django.http import HttpResponse
import json
from pymongo import MongoClient
import time
from django.views.decorators.csrf import csrf_exempt
# Create your views here.
client = MongoClient()
db = client['rp_data_errors']
coll = db['report_retail_partner_data']

@csrf_exempt
def report_rp_data(request):
    rp_id = request.POST['retail_partner_id']
    reporter_id = request.POST['uin']
    error_reported = request.POST['error']
    curr_timestamp = time.time()
    rp_doc = coll.find_one({'rp_id': int(rp_id)})
    try:
        existing_reports = rp_doc['reports']
        num_reports = int(rp_doc['num_reports'])
        existing_reports.append({
            'id' : num_reports + 1,
            'error' : error_reported,
            'timestamp' : curr_timestamp,
            'reporter_id' : reporter_id
        })
        coll.update({'rp_id': int(rp_id)},{'reports': existing_reports, 'num_reports' : num_reports+1, 'rp_id': int(rp_id)})
    except Exception as e:
        rp_doc = {
            'rp_id' : int(rp_id),
            'reports' : [{
                'id' : 1,
                'error' : error_reported,
                'timestamp' : curr_timestamp,
                'reporter_id' : reporter_id
            }],
            'num_reports' : 1
        }
        coll.insert(rp_doc)
    return HttpResponse('Successfully Reported')


text:
#ifndef _MQTT_H_
#define _MQTT_H_

#include "netz.h"
#include "mqtt_pack.h"

/**
 * @brief	The mqtt_app_ctx_t struct
 *
 * @details	This structure encapsulates  all the components required
 *		to write a MQTT application: client context, network
 *		stuff and a semaphore. sem is very useful to protect all
 *		the elements of this structure when multiple threads are
 *		trying to access these variables.
 */
struct mqtt_app_ctx_t {
	struct mqtt_client_ctx_t *client;
	struct netz_ctx_t *netz_ctx;
	struct app_buf_t *tx_buf;
	struct app_buf_t *rx_buf;
	struct nano_sem sem;

	int (*cb_publish)(struct mqtt_msg_t *msg);

};

/**
 * @brief MQTT_APP_INIT		MQTT application structure initializer
 *
 * @details			This define initializes the MQTT
 *				application structure.
 *				However, the semaphore is not initialized.
 *				See the mqtt_init function.
 */
#define MQTT_APP_INIT(cl, net, tx, rx) {	.client = cl,		\
						.netz_ctx = net,	\
						.tx_buf = tx,		\
						.cb_publish = NULL,	\
						.rx_buf = rx}

/**
 * @brief mqtt_cb_publish	Installs a callback, processed when a
 *				PUBLISH message is received
 * @param app			MQTT Application Context structure
 * @return			0, always
 */
int mqtt_cb_publish(struct mqtt_app_ctx_t *app,
		    int (*cb_publish)(struct mqtt_msg_t *msg));

/*
 *	****************************************************************
 *	MQTT configuration functions. No network activity involved here.
 */

/**
 * @brief mqtt_init		Initializes runtime components
 * @details			So far, this function only initializes
 *				the semaphore.
 * @param app			MQTT application context.
 * @return			0, always
 */
int mqtt_init(struct mqtt_app_ctx_t *app);

/**
 * @brief mqtt_configure	MQTT connection parameters configuration
 * @details			Function arguments are passed to the
 *				MQTT client structure: app->client.
 * @param app			MQTT application context.
 * @param client_id		C-string containing the client name.
 * @param clean_session		So far, only clean_session = 1 is
 *				supported.
 * @param proto			MQTT protocol: MQTT_311.
 * @return			0, always.
 */
int mqtt_configure(struct mqtt_app_ctx_t *app, const char *client_id,
		   int clean_session, enum mqtt_protocol proto);

/**
 * @brief mqtt_auth		Authentication parameters configuration
 * @details			This function sets the username and pass
 *				parameters used in applications that
 *				authenticate against the MQTT server.
  * @param app			MQTT application context.
 * @param username		C-string
 * @param pass			C-string
 * @return			0, always
 */
int mqtt_auth(struct mqtt_app_ctx_t *app, const char *username,
	      const char *pass);

/**
 * @brief mqtt_will		Application will message configuration
 * @details			This functions sets the will message
 *				parameters. <b>The use of this function
 *				is optional</b>.
 *				For more information read the MQTT
 *				specification.
 * @param app			MQTT application context.
 * @param topic			Topic of the will message.
 * @param msg			Message that will be broadcasted.
 * @param qos			Desired QoS of the will message.
 * @param retained		Retained property.
 * @return			0, always.
 */
int mqtt_will(struct mqtt_app_ctx_t *app, const char *topic,
	      const char *msg, enum mqtt_qos qos, int retained);

/**
 * @brief mqtt_buffers		Sets the application buffers.
 * @details			This function takes two buffers that
 *				will be used in networking operations.
 * @param app			MQTT application context.
 * @param tx_buf		Transmission buffer.
 * @param rx_buf		Reception buffer
 * @return			0, always.
 */
int mqtt_buffers(struct mqtt_app_ctx_t *app,
		 struct app_buf_t *tx_buf, struct app_buf_t *rx_buf);

/**
 * @brief mqtt_network		Sets the network context.
 * @param app			MQTT application context.
 * @param netz_ctx		Network context.
 * @return			0, always.
 */
int mqtt_network(struct mqtt_app_ctx_t *app, struct netz_ctx_t *netz_ctx);

/*
 *	             End of MQTT configuration functions.
 *	****************************************************************
 */

/*
 *	****************************************************************
 *		MQTT routines that require network functionality.
 */

/**
 * @brief mqtt_connect		Sends the MQTT CONNECT message.
 * @details			This function packs and sends the MQTT
 *				CONNECT message to the server.
 * @param app			MQTT application context.
 * @return			0 on success.
 * @return			-EINVAL if an invalid parameter was
 *				passed as argument or received from the
 *				server.
 * @return			-EIO on network error.
 */
int mqtt_connect(struct mqtt_app_ctx_t *app);

/**
 * @brief mqtt_disconnect	Sends the MQTT DISCONNECT message.
 * @details			This function packs and sends the MQTT
 *				DISCONNECT message to the server.
 * @param app			MQTT application context.
 * @return			0 on success.
 * @return			-EINVAL if an invalid parameter was
 *				passed as argument or received from the
 *				server.
 * @return			-EIO on network error.
 */
int mqtt_disconnect(struct mqtt_app_ctx_t *app);

/**
 * @brief mqtt_publish		Sends the MQTT PUBLISH message.
 * @details			This function packs and sends the MQTT
 *				PUBLISH message to the server.
 *				See the mqtt_msg_topic and
 *				mqtt_msg_payload_str functions.
 * @param app			MQTT application context.
 * @param msg			MQTT Message structure.
 * @param qos			Message QoS.
 * @param retained		Retained property.
 * @return			0 on success.
 * @return			-EINVAL if an invalid parameter was
 *				passed as argument or received from the
 *				server.
 * @return			-EIO on network error.
 */
int mqtt_publish(struct mqtt_app_ctx_t *app, struct mqtt_msg_t *msg,
		 enum mqtt_qos qos, int retained);

/**
 * @brief mqtt_pingreq		Sends the MQTT PINGREQ message.
 * @details			This function packs and sends the MQTT
 *				PINGREQ message to the server.
 * @param app			MQTT application context.
 * @return			0 on success.
 * @return			-EINVAL if an invalid parameter was
 *				passed as argument or received from the
 *				server.
 * @return			-EIO on network error.
 */
int mqtt_pingreq(struct mqtt_app_ctx_t *app);

/**
 * @brief mqtt_subscribe	Sends the MQTT SUBSCRIBE message.
 * @details			This functions packs and sends the MQTT
 *				SUBSCRIBE message to the server.
 *				So far, only one topic is processed by
 *				message.
 *				TODO: add an array of topics/qos.
 * @param app			MQTT application context.
 * @param topic			Topic to subscribe to.
 * @param qos			The desired QoS.
 * @return			0 on success.
 * @return			-EINVAL if an invalid parameter was
 *				passed as argument or received from the
 *				server.
 * @return			-EIO on network error.
 */
int mqtt_subscribe(struct mqtt_app_ctx_t *app, char *topic, enum mqtt_qos qos);

/**
 * @brief mqtt_unsubscribe	Sends the MQTT UNSUBSCRIBE message
 * @details			This functions packs and sends the MQTT
 *				UNSUBSCRIBE message to the server.
 *				So far, only one topic is processed by
 *				message.
 *				TODO: add an array of topics.
 * @param app			MQTT application context.
 * @param topic			Topic to unsubscribe from.
 * @return			0 on success.
 * @return			-EINVAL if an invalid parameter was
 *				passed as argument or received from the
 *				server.
 * @return			-EIO on network error.
 */
int mqtt_unsubscribe(struct mqtt_app_ctx_t *app, char *topic);

/**
 * @brief mqtt_read		Read any received MQTT message
 * @details			If a MQTT PUBLISH message is received,
 *				this function handles in background the
 *				response (if any).
 *				TODO: implement more messages' handlers.
 * @param app			MQTT application context.
 * @return			0 on success.
 * @return			-EINVAL if an invalid parameter was
 *				passed as argument or received from the
 *				server.
 * @return			-EIO on network error.
 */
int mqtt_read(struct mqtt_app_ctx_t *app);

/*
 *	         End of MQTT functions with network support.
 *	****************************************************************
 */

#endif


text:
package alluxio.client.cli.fsadmin.command;

import alluxio.AlluxioURI;
import alluxio.ConfigurationRule;
import alluxio.SystemErrRule;
import alluxio.SystemOutRule;
import alluxio.cli.fsadmin.FileSystemAdminShell;
import alluxio.cli.fsadmin.journal.QuorumCommand;
import alluxio.cli.fsadmin.journal.QuorumElectCommand;
import alluxio.cli.fsadmin.journal.QuorumInfoCommand;
import alluxio.cli.fsadmin.journal.QuorumRemoveCommand;
import alluxio.conf.Configuration;
import alluxio.conf.PropertyKey;
import alluxio.exception.ExceptionMessage;
import alluxio.grpc.JournalDomain;
import alluxio.grpc.QuorumServerInfo;
import alluxio.grpc.QuorumServerState;
import alluxio.master.journal.JournalType;
import alluxio.multi.process.MasterNetAddress;
import alluxio.multi.process.MultiProcessCluster;
import alluxio.multi.process.PortCoordination;
import alluxio.testutils.BaseIntegrationTest;
import alluxio.util.CommonUtils;
import alluxio.util.WaitForOptions;

import org.junit.After;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

/**
 * Integration tests for the embedded journal.
 */
public final class QuorumCommandIntegrationTest extends BaseIntegrationTest {
  @Rule
  public ConfigurationRule mConf = new ConfigurationRule(
      PropertyKey.USER_METRICS_COLLECTION_ENABLED, false, Configuration.modifiableGlobal());

  public MultiProcessCluster mCluster;

  public ByteArrayOutputStream mOutput = new ByteArrayOutputStream();
  public ByteArrayOutputStream mErrOutput = new ByteArrayOutputStream();

  @Rule
  public ExpectedException mException = ExpectedException.none();

  @Rule
  public SystemOutRule mOutRule = new SystemOutRule(mOutput);

  @Rule
  public SystemErrRule mErrRule = new SystemErrRule(mErrOutput);

  @After
  public void after() throws Exception {
    if (mCluster != null) {
      mCluster.destroy();
    }
  }

  @Test
  public void quorumInfo() throws Exception {
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.QUORUM_SHELL_INFO)
        .setClusterName("QuorumShellInfo").setNumMasters(3).setNumWorkers(0)
        .addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED)
        .addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min")
        // To make the test run faster.
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms")
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms").build();
    mCluster.start();

    String output;

    try (FileSystemAdminShell shell = new FileSystemAdminShell(Configuration.global())) {
      // Validate quorum state is dumped as expected.
      mOutput.reset();
      shell.run("journal", "quorum", "info", "-domain", "MASTER");
      output = mOutput.toString().trim();
      Assert.assertTrue(output.contains(
          String.format(QuorumInfoCommand.OUTPUT_HEADER_DOMAIN, JournalDomain.MASTER.name())));
      Assert.assertTrue(
          output.contains(String.format(QuorumInfoCommand.OUTPUT_HEADER_QUORUM_SIZE, 3)));
      List<String> journalAddresses =
          Configuration.getList(PropertyKey.MASTER_EMBEDDED_JOURNAL_ADDRESSES);
      for (String address : journalAddresses) {
        String format = String.format(QuorumInfoCommand.OUTPUT_SERVER_INFO,
                QuorumServerState.AVAILABLE.name(), "0", address).trim();
        Assert.assertTrue(output.contains(format));
      }

      // Validate quorum state is updated as expected after a fail-over.
      mCluster.stopMaster(0);
      // Wait for up to 2 election timeouts until quorum notices the change.
      // Use shell "quorum info" for validation.
      CommonUtils.waitFor("Quorum noticing change.", () -> {
        mOutput.reset();
        shell.run("journal", "quorum", "info", "-domain", "MASTER");
        return mOutput.toString().trim().contains(QuorumServerState.UNAVAILABLE.name());
      }, WaitForOptions.defaults().setTimeoutMs(2 * (int) Configuration.getMs(
          PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT)));
    }
    mCluster.notifySuccess();
  }

  @Test
  public void quorumRemove() throws Exception {
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.QUORUM_SHELL_REMOVE)
        .setClusterName("QuorumShellRemove").setNumMasters(5).setNumWorkers(0)
        .addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED)
        .addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min")
        // To make the test run faster.
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms")
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms").build();
    mCluster.start();

    String output;

    try (FileSystemAdminShell shell = new FileSystemAdminShell(Configuration.global())) {
      AlluxioURI testDir = new AlluxioURI("/testDir");
      mCluster.getFileSystemClient().createDirectory(testDir);
      // Verify cluster is reachable.
      Assert.assertTrue(mCluster.getFileSystemClient().exists(testDir));

      // Validate quorum state is updated as expected after shutting down 2 masters.
      mCluster.stopMaster(0);
      mCluster.stopMaster(1);

      // Verify cluster is reachable.
      Assert.assertTrue(mCluster.getFileSystemClient().exists(testDir));

      // Wait for up to 2 election timeouts until quorum notices changes.
      CommonUtils.waitFor("Quorum noticing change.", () -> {
        try {
          return mCluster.getJournalMasterClientForMaster().getQuorumInfo().getServerInfoList()
              .stream().filter((info) -> info.getServerState() == QuorumServerState.UNAVAILABLE)
              .collect(Collectors.toList()).size() == 2;
        } catch (Exception e) {
          return false;
        }
      }, WaitForOptions.defaults().setTimeoutMs(6 * (int) Configuration.getMs(
          PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT)));

      // Remove unavailable masters using shell.
      for (QuorumServerInfo serverInfo : mCluster.getJournalMasterClientForMaster().getQuorumInfo()
          .getServerInfoList()) {
        if (serverInfo.getServerState() == QuorumServerState.UNAVAILABLE) {
          mOutput.reset();

          String serverAddress = String.format("%s:%d", serverInfo.getServerAddress().getHost(),
              serverInfo.getServerAddress().getRpcPort());
          shell.run("journal", "quorum", "remove", "-domain", "MASTER", "-address", serverAddress);

          output = mOutput.toString().trim();
          Assert.assertEquals(String.format(QuorumRemoveCommand.OUTPUT_RESULT, serverAddress,
              JournalDomain.MASTER.name()), lastLine(output));
        }
      }

      // Verify quorum size is resized down to 3 after removing 2 masters.
      Assert.assertEquals(3,
          mCluster.getJournalMasterClientForMaster().getQuorumInfo().getServerInfoList().size());

      mCluster.stopMaster(2);
      // Verify cluster is reachable.
      Assert.assertTrue(mCluster.getFileSystemClient().exists(testDir));
    }
    mCluster.notifySuccess();
  }

  @Test
  public void elect() throws Exception {
    final int MASTER_INDEX_WAIT_TIME = 5_000;
    int numMasters = 3;
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.QUORUM_SHELL_REMOVE)
            .setClusterName("QuorumShellElect").setNumMasters(numMasters).setNumWorkers(0)
            .addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED)
            .addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min")
            // To make the test run faster.
            .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms")
            .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms")
            .build();
    mCluster.start();

    try (FileSystemAdminShell shell = new FileSystemAdminShell(Configuration.global())) {
      int newLeaderIdx = (mCluster.getPrimaryMasterIndex(MASTER_INDEX_WAIT_TIME) + 1) % numMasters;
      // `getPrimaryMasterIndex` uses the same `mMasterAddresses` variable as getMasterAddresses
      // we can therefore access to the new leader's address this ways
      MasterNetAddress netAddress = mCluster.getMasterAddresses().get(newLeaderIdx);
      String newLeaderAddr = String.format("%s:%s", netAddress.getHostname(),
              netAddress.getEmbeddedJournalPort());

      mOutput.reset();
      shell.run("journal", "quorum", "elect", "-address" , newLeaderAddr);
      String output = mOutput.toString().trim();
      String expected = String.format("%s\n%s\n%s\n%s",
          String.format(QuorumElectCommand.TRANSFER_INIT, newLeaderAddr),
          String.format(QuorumElectCommand.TRANSFER_SUCCESS, newLeaderAddr),
          String.format(QuorumElectCommand.RESET_INIT, "successful"),
          QuorumElectCommand.RESET_SUCCESS);
      Assert.assertEquals(expected, output);
    }
    mCluster.notifySuccess();
  }

  @Test
  public void infoAfterElect() throws Exception {
    final int MASTER_INDEX_WAIT_TIME = 5_000;
    int numMasters = 3;
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.QUORUM_SHELL_REMOVE)
        .setClusterName("QuorumShellElect").setNumMasters(numMasters).setNumWorkers(0)
        .addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED)
        .addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min")
        // To make the test run faster.
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms")
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms")
        .build();
    mCluster.start();

    try (FileSystemAdminShell shell = new FileSystemAdminShell(Configuration.global())) {
      int newLeaderIdx = (mCluster.getPrimaryMasterIndex(MASTER_INDEX_WAIT_TIME) + 1) % numMasters;
      // `getPrimaryMasterIndex` uses the same `mMasterAddresses` variable as getMasterAddresses
      // we can therefore access to the new leader's address this ways
      MasterNetAddress netAddress = mCluster.getMasterAddresses().get(newLeaderIdx);
      String newLeaderAddr = String.format("%s:%s", netAddress.getHostname(),
          netAddress.getEmbeddedJournalPort());

      int success = shell.run("journal", "quorum", "elect", "-address" , newLeaderAddr);
      Assert.assertEquals("elect command failed", 0, success);
      mOutput.reset();
      shell.run("journal", "quorum", "info", "-domain", "MASTER");
      String output = mOutput.toString().trim();
      for (MasterNetAddress masterAddr : mCluster.getMasterAddresses()) {
        String expected = String.format(QuorumInfoCommand.OUTPUT_SERVER_INFO, "AVAILABLE", "1",
            String.format("%s:%d", masterAddr.getHostname(), masterAddr.getEmbeddedJournalPort()));
        Assert.assertTrue(output.contains(expected.trim()));
      }
    }
    mCluster.notifySuccess();
  }

  @Test
  public void quorumCommand() throws Exception {
    mCluster = MultiProcessCluster.newBuilder(PortCoordination.QUORUM_SHELL)
        .setClusterName("QuorumShell").setNumMasters(3).setNumWorkers(0)
        .addProperty(PropertyKey.MASTER_JOURNAL_TYPE, JournalType.EMBEDDED)
        .addProperty(PropertyKey.MASTER_JOURNAL_FLUSH_TIMEOUT_MS, "5min")
        // To make the test run faster.
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MIN_ELECTION_TIMEOUT, "750ms")
        .addProperty(PropertyKey.MASTER_EMBEDDED_JOURNAL_MAX_ELECTION_TIMEOUT, "1500ms").build();
    mCluster.start();
    try (FileSystemAdminShell shell = new FileSystemAdminShell(Configuration.global())) {
      String output;

      // Validate quorum sub-commands are validated.
      mOutput.reset();
      shell.run("journal", "quorum", "nonexistentCommand");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumCommand.description(), lastLine(output));

      // Validate option counts are validated for "quorum info"
      mOutput.reset();
      shell.run("journal", "quorum", "info");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumInfoCommand.description(), lastLine(output));

      mOutput.reset();
      shell.run("journal", "quorum", "info", "-op1", "val1", "-op2", "val2");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumInfoCommand.description(), lastLine(output));

      // Validate option counts are validated for "quorum", "remove"
      mOutput.reset();
      shell.run("journal", "quorum", "remove");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumRemoveCommand.description(), lastLine(output));
      mOutput.reset();
      shell.run("journal", "quorum", "remove", "-op1", "val1");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumRemoveCommand.description(), lastLine(output));
      mOutput.reset();
      shell.run("journal", "quorum", "remove", "-op1", "val1", "-op2", "val2", "-op3", "val3");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumRemoveCommand.description(), lastLine(output));

      // Validate option counts are validated for "quorum", "elect"
      mOutput.reset();
      shell.run("journal", "quorum", "elect");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumElectCommand.description(), lastLine(output));
      mOutput.reset();
      shell.run("journal", "quorum", "elect", "-op1", "val1");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumElectCommand.description(), lastLine(output));
      mOutput.reset();
      shell.run("journal", "quorum", "elect", "-op1", "val1", "-op2", "val2", "-op3",
              "val3");
      output = mOutput.toString().trim();
      Assert.assertEquals(QuorumElectCommand.description(), lastLine(output));

      // Validate option validation works for "quorum info".
      mOutput.reset();
      shell.run("journal", "quorum", "info", "-domain", "UNKNOWN");
      output = mOutput.toString().trim();
      Assert.assertEquals(ExceptionMessage.INVALID_OPTION_VALUE.getMessage(
          QuorumInfoCommand.DOMAIN_OPTION_NAME, Arrays.toString(JournalDomain.values())), output);

      // Validate option validation works for "journal quorum remove"
      // Validate -domain is validated.
      mOutput.reset();
      shell.run("journal", "quorum", "remove", "-domain", "UNKNOWN", "-address", "host:0");
      output = mOutput.toString().trim();
      Assert.assertEquals(ExceptionMessage.INVALID_OPTION_VALUE.getMessage(
          QuorumInfoCommand.DOMAIN_OPTION_NAME, Arrays.toString(JournalDomain.values())), output);

      // Validate -address is validated.
      mOutput.reset();
      shell.run("journal", "quorum", "remove", "-domain", "JOB_MASTER", "-address",
          "hostname:invalid_port");
      output = mOutput.toString().trim();
      Assert.assertEquals(ExceptionMessage.INVALID_ADDRESS_VALUE.getMessage(), output);

      // Validate -address is validated for transferLeader.
      mOutput.reset();
      shell.run("journal", "quorum", "elect", "-address", "hostname:invalid_port");
      output = mOutput.toString().trim();
      Assert.assertEquals(ExceptionMessage.INVALID_ADDRESS_VALUE.getMessage(), output);
    }
    mCluster.notifySuccess();
  }

  private String lastLine(String output) {
    String[] lines = output.split("\n");
    if (lines.length > 0) {
      return lines[lines.length - 1];
    }
    return "";
  }
}


text:
from django.conf.urls.defaults import patterns, include, url

urlpatterns = patterns('adelie.views',
    (r'^register$', 'register'),
    (r'^submit_registration$', 'submit_registration'),
)


text:
'use strict';

angular.module('lmApp')
  .config(function ($routeProvider) {
    $routeProvider
      .when('/', {
        templateUrl: 'app/main/main.html',
        controller: 'MainCtrl'
      });
  });


text:
  Marsanne est
      une commune localisée dans le département de Drôme en Rhône-Alpes. Elle comptait 1 213 habitants en 2008.

La ville compte quelques  aménagements, elle dispose, entre autres, de un terrain de tennis, un terrain de sport et une boucle de randonnée.

À proximité de Marsanne sont localisées les communes de
Saint-Gervais-sur-Roubion localisée à 6 km, 764 habitants,
La Roche-sur-Grane localisée à 6 km, 150 habitants,
Mirmande située à 6 km, 507 habitants,
Sauzet située à 6 km, 1 860 habitants,
Condillac située à 4 km, 145 habitants,
Bonlieu-sur-Roubion située à 5 km, 384 habitants,
 entre autres. De plus, Marsanne est située à seulement treize km de Montélimar.

Le parc d'habitations, à Marsanne, était réparti en 2011 en 73 appartements et 514 maisons soit
 un marché plutôt équilibré.

À Marsanne, la valeur moyenne à l'achat d'un appartement s'évalue à 1 577 € du m² en vente. La valeur moyenne d'une maison à l'achat se situe à 2 067 € du m². À la location la valorisation moyenne se situe à 0 € du m² par mois.