text:
//
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

#include "augmenter/augmenter.h"

#include <string>
#include <unordered_set>
#include <utility>
#include <vector>

#include "absl/container/flat_hash_set.h"
#include "absl/strings/ascii.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/str_split.h"
#include "augmenter/augmentations.h"
#include "augmenter/case_augmentation.h"
#include "augmenter/random_sampler.h"
#include "augmenter/shuffler.h"
#include "augmenter/token_range.h"
#include "protocol_buffer/document.pb.h"
#include "protocol_buffer/documents.pb.h"

namespace augmenter {

Augmenter::Augmenter(Augmentations augmentations,
                     bert_annotator::Documents* const documents,
                     RandomSampler* const address_sampler,
                     RandomSampler* const phone_sampler,
                     Shuffler* const shuffler, absl::BitGenRef bitgenref)
    : documents_(documents),
      address_sampler_(address_sampler),
      phone_sampler_(phone_sampler),
      shuffler_(shuffler),
      augmentations_(augmentations),
      bitgenref_(bitgenref) {
  // Assert valid probabilities.
  const std::vector<double> probabilities = {
      augmentations_.prob_lowercasing_complete_token,
      augmentations_.prob_lowercasing_first_letter,
      augmentations_.prob_uppercasing_complete_token,
      augmentations_.prob_uppercasing_first_letter,
      augmentations_.prob_address_replacement,
      augmentations_.prob_phone_replacement,
      augmentations_.prob_context_drop_between_labels,
      augmentations_.prob_context_drop_outside_one_label};
  if (absl::c_any_of(probabilities, [](double probability) {
        return probability < 0.0 || probability > 1.0;
      })) {
    std::cerr << "All probabilities must have values between zero and one."
              << std::endl;
    abort();
  }
  if (augmentations_.prob_lowercasing_complete_token +
          augmentations_.prob_lowercasing_first_letter +
          augmentations_.prob_uppercasing_complete_token +
          augmentations_.prob_uppercasing_first_letter >
      1) {
    std::cerr << "The probabilities for changing the case of tokens must sum "
                 "up to at most one."
              << std::endl;
    abort();
  }

  // Skip invalid sentences where the start/end value of tokens does not
  // match their length.
  for (int i = documents_->documents_size() - 1; i >= 0; --i) {
    bert_annotator::Document* document = documents_->mutable_documents(i);

    for (bert_annotator::Token token : document->token()) {
      if (static_cast<int>(token.word().size()) !=
          token.end() - token.start() + 1) {
        documents_->mutable_documents()->erase(
            documents_->mutable_documents()->begin() + i);
        break;
      }
    }
  }

  for (bert_annotator::Document& document : *documents_->mutable_documents()) {
    InitializeLabelList(&document);
    MergePhoneNumberTokens(&document);
    DropSeparatorTokens(&document);
    UnifyAndMergeAddressLabels(&document);

    if (augmentations_.modify_original) {
      AugmentAddress(&document);
      AugmentPhone(&document);
      AugmentContext(&document);
      AugmentCase(&document);
      AugmentPunctuation(&document);
    }
  }
}

void Augmenter::AugmentAddress(
    bert_annotator::Document* const augmented_document) {
  MaybeReplaceLabel(augmentations_.prob_address_replacement, address_sampler_,
                    Augmenter::kAddressReplacementLabel,
                    /*split_into_tokens=*/true, augmented_document);
}

void Augmenter::AugmentPhone(
    bert_annotator::Document* const augmented_document) {
  MaybeReplaceLabel(augmentations_.prob_phone_replacement, phone_sampler_,
                    Augmenter::kPhoneReplacementLabel,
                    /*split_into_tokens=*/false, augmented_document);
}

void Augmenter::AugmentCase(bert_annotator::Document* const document) {
  std::string* const text = document->mutable_text();
  std::string new_text;
  int text_index = 0;
  for (bert_annotator::Token& token : *document->mutable_token()) {
    std::string* const word = token.mutable_word();

    // Adds the non-tokens before the current token as they are.
    const int token_start = token.start();
    const int token_end = token.end();
    if (text_index < token_start) {
      new_text.append(text->begin() + text_index, text->begin() + token_start);
    }

    // Sample one of the five actions (lower/upper case complete token/first
    // letter, keep unmodified). Using a discrete distribution would be cleaner,
    // but cannot be mocked in tests.
    double random_value = absl::Uniform<double>(bitgenref_, 0.0, 1.0);
    double boundary_lowercase_complete_token =
        augmentations_.prob_lowercasing_complete_token;
    double boundary_lowercase_first_letter =
        boundary_lowercase_complete_token +
        augmentations_.prob_lowercasing_first_letter;
    double boundary_uppercase_complete_token =
        boundary_lowercase_first_letter +
        augmentations_.prob_uppercasing_complete_token;
    double boundary_uppercase_first_letter =
        boundary_uppercase_complete_token +
        augmentations_.prob_uppercasing_first_letter;

    if (random_value < boundary_lowercase_complete_token) {
      absl::AsciiStrToLower(word);
    } else if (random_value < boundary_lowercase_first_letter) {
      word->at(0) = std::tolower(word->at(0));
    } else if (random_value < boundary_uppercase_complete_token) {
      absl::AsciiStrToUpper(word);
    } else if (random_value < boundary_uppercase_first_letter) {
      word->at(0) = std::toupper(word->at(0));
    }

    new_text.append(*word);
    text_index = token_end + 1;
  }
  new_text.append(text->begin() + text_index, text->end());
  document->set_text(new_text);
}

bert_annotator::Document* Augmenter::AugmentContextless(
    const absl::string_view label, const bool split_into_tokens,
    RandomSampler* const sampler) {
  bert_annotator::Document* document = documents_->add_documents();

  const std::string sample = sampler->Sample();
  document->set_text(sample);
  std::vector<bert_annotator::Token> new_tokens;
  if (split_into_tokens) {
    new_tokens = SplitTextIntoTokens(0, sample);
  } else {
    bert_annotator::Token token;
    token.set_word(sample);
    token.set_start(0);
    token.set_end(sample.size() - 1);
    new_tokens.push_back(token);
  }
  InsertTokens(0, new_tokens, document);
  InitializeLabelList(document);
  InsertLabeledSpan(
      TokenRange{.start = 0, .end = static_cast<int>(new_tokens.size()) - 1},
      label, document);

  return document;
}

void Augmenter::Augment() {
  const int original_document_number = documents_->documents_size();

  for (int i = 0; i < augmentations_.num_total; ++i) {
    if (i % 100000 == 0) {
      std::cout << "Augmentation progress: "
                << std::to_string(i * 100 / augmentations_.num_total) << "%"
                << std::endl;
    }
    bert_annotator::Document* augmented_document;

    if (i < augmentations_.num_contextless_addresses) {
      augmented_document =
          AugmentContextless(kAddressReplacementLabel,
                             /*split_into_tokens=*/true, address_sampler_);
    } else if (i < augmentations_.num_contextless_addresses +
                       augmentations_.num_contextless_phones) {
      augmented_document = AugmentContextless(
          kPhoneReplacementLabel, /*split_into_tokens=*/false, phone_sampler_);
    } else {
      const int document_id = absl::Uniform(absl::IntervalClosed, bitgenref_, 0,
                                            original_document_number - 1);
      const bert_annotator::Document& original_document =
          documents_->documents(document_id);
      augmented_document = documents_->add_documents();
      augmented_document->CopyFrom(original_document);

      AugmentAddress(augmented_document);
      AugmentPhone(augmented_document);
      AugmentContext(augmented_document);
    }

    AugmentCase(augmented_document);
    AugmentPunctuation(augmented_document);
  }

  // If the documents are not shuffled, the unmodified sentences are all next to
  // each other. This is problematic when merging sentences in the next step.
  if (augmentations_.shuffle) {
    std::cout << "Shuffling..." << std::endl;
    shuffler_->Shuffle(documents_, bitgenref_);
  }

  std::cout << "Concatenating..." << std::endl;
  for (int i = documents_->documents_size() - 1; i > 0; --i) {
    if (absl::Bernoulli(bitgenref_,
                        augmentations_.prob_sentence_concatenation)) {
      AddConcatenatedDocument(documents_->documents(i - 1),
                              documents_->documents(i));
    }
  }

  // Shuffle again, now that concatenated sentences have been added.
  if (augmentations_.shuffle) {
    std::cout << "Shuffling..." << std::endl;
    shuffler_->Shuffle(documents_, bitgenref_);
  }

  if (augmentations_.mask_digits) {
    std::cout << "Maskinkg digits..." << std::endl;
    for (bert_annotator::Document& document :
         *documents_->mutable_documents()) {
      MaskDigits(&document);
    }
  }

  for (const bert_annotator::Document& document : documents_->documents()) {
    if (document.text().length() == 0) {
      std::cerr << "Empty document in output detected. This will break the "
                   "evaluation scripts, aborting."
                << std::endl;
      abort();
    }
  }
}

void Augmenter::AddConcatenatedDocument(
    const bert_annotator::Document& first_document,
    const bert_annotator::Document& second_document) {
  bert_annotator::Document* concatenated_document = documents_->add_documents();
  concatenated_document->CopyFrom(first_document);
  bert_annotator::Document tmp_copy_of_second_document;
  tmp_copy_of_second_document.CopyFrom(second_document);

  ShiftTokenBoundaries(0, first_document.text().size() + 1,
                       &tmp_copy_of_second_document);
  ShiftLabeledSpansForDroppedTokens(0, first_document.token_size(),
                                    &tmp_copy_of_second_document);
  concatenated_document->mutable_text()->append(
      " " + tmp_copy_of_second_document.text());
  InsertTokens(concatenated_document->token_size(),
               std::vector<bert_annotator::Token>(
                   tmp_copy_of_second_document.token().begin(),
                   tmp_copy_of_second_document.token().end()),
               concatenated_document);
  const google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>&
      labeled_spans = GetLabelList(tmp_copy_of_second_document);
  for (const bert_annotator::LabeledSpan& labeled_span : labeled_spans) {
    InsertLabeledSpan(TokenRange{.start = labeled_span.token_start(),
                                 .end = labeled_span.token_end()},
                      labeled_span.label(), concatenated_document);
  }
}

void Augmenter::AugmentContext(
    bert_annotator::Document* const augmented_document) {
  MaybeDropContextKeepLabels(augmentations_.prob_context_drop_between_labels,
                             augmented_document);
  MaybeDropContextDropLabels(augmented_document);
}

void Augmenter::AugmentPunctuation(bert_annotator::Document* const document) {
  for (int i = 0; i < document->token_size() - 1; ++i) {
    bert_annotator::Token* const token = document->mutable_token(i);
    const bool do_change_punctuation = absl::Bernoulli(
        bitgenref_, augmentations_.prob_punctuation_change_between_tokens);
    if (!do_change_punctuation) {
      continue;
    }
    const int punctuation_replacement_id = absl::Uniform(
        bitgenref_, 0,
        static_cast<int>(kPunctuationReplacementsWithinText.size()));
    const absl::string_view punctuation_replacement =
        kPunctuationReplacementsWithinText[punctuation_replacement_id];
    const int current_punctuation_length =
        document->token(i + 1).start() - token->end() - 1;
    document->mutable_text()->replace(token->end() + 1,
                                      current_punctuation_length,
                                      std::string(punctuation_replacement));
    ShiftTokenBoundaries(
        i + 1, punctuation_replacement.size() - current_punctuation_length,
        document);
  }

  const bool do_change_punctuation = absl::Bernoulli(
      bitgenref_, augmentations_.prob_punctuation_change_at_sentence_end);
  if (do_change_punctuation && document->token_size() > 0) {
    const int punctuation_replacement_id = absl::Uniform(
        bitgenref_, 0,
        static_cast<int>(kPunctuationReplacementsAtSentenceEnd.size()));
    const absl::string_view punctuation_replacement =
        kPunctuationReplacementsAtSentenceEnd[punctuation_replacement_id];
    bert_annotator::Token last_token =
        document->token(document->token_size() - 1);
    const int current_punctuation_length =
        document->text().size() - last_token.end() - 1;
    document->mutable_text()->replace(last_token.end() + 1,
                                      current_punctuation_length,
                                      std::string(punctuation_replacement));
  }
}

std::vector<TokenRange> Augmenter::GetUnlabeledRanges(
    const bert_annotator::Document& document) {
  std::vector<TokenRange> droppable_ranges;
  int start = 0;

  const google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>&
      labeled_spans = GetLabelList(document);
  for (const bert_annotator::LabeledSpan& labeled_span : labeled_spans) {
    if (labeled_span.label() == Augmenter::kAddressReplacementLabel ||
        labeled_span.label() == Augmenter::kPhoneReplacementLabel) {
      if (start != labeled_span.token_start()) {
        droppable_ranges.push_back(
            TokenRange{.start = start, .end = labeled_span.token_start() - 1});
      }
      start = labeled_span.token_end() + 1;
    }
  }
  if (start != document.token_size()) {
    droppable_ranges.push_back(
        TokenRange{.start = start, .end = document.token_size() - 1});
  }

  return droppable_ranges;
}

std::vector<TokenRange> Augmenter::GetLabeledRanges(
    const bert_annotator::Document& document,
    absl::flat_hash_set<absl::string_view> labels) {
  std::vector<TokenRange> labeled_ranges;
  const google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>&
      labeled_spans = GetLabelList(document);
  for (const bert_annotator::LabeledSpan& labeled_span : labeled_spans) {
    if (labels.contains(labeled_span.label())) {
      labeled_ranges.push_back(TokenRange{.start = labeled_span.token_start(),
                                          .end = labeled_span.token_end()});
    }
  }
  return labeled_ranges;
}

void Augmenter::MaybeDropContextKeepLabels(
    const double probability,
    bert_annotator::Document* const augmented_document) {
  std::vector<TokenRange> droppable_ranges =
      GetUnlabeledRanges(*augmented_document);
  // If there are no labels, we will get a single droppable range containing
  // the whole sentence. To drop from both its start and end, we duplicate it
  // here.
  bool no_labels = false;
  if (droppable_ranges.size() == 1 && droppable_ranges[0].start == 0 &&
      droppable_ranges[0].end == augmented_document->token_size() - 1) {
    no_labels = true;
    droppable_ranges.push_back(droppable_ranges[0]);
  }

  // Tokens will be dropped, so iterating backwards avoids the need to
  // update the indices in droppable_ranges.
  for (int i = droppable_ranges.size() - 1; i >= 0; --i) {
    const TokenRange& droppable_range = droppable_ranges[i];
    const bool do_drop = absl::Bernoulli(bitgenref_, probability);
    if (!do_drop) {
      continue;
    }

    // At least one token should be dropped, so the range needs to be at
    // least two tokens long.
    if (droppable_range.end == droppable_range.start) {
      continue;
    }

    int drop_tokens_start;
    int drop_tokens_end;
    // For context after the last label, drop a postfix of the sentence.
    if (i == static_cast<int>(droppable_ranges.size()) - 1 &&
        droppable_range.end == augmented_document->token_size() - 1) {
      drop_tokens_start =
          absl::Uniform(absl::IntervalClosed, bitgenref_,
                        droppable_range.start + 1, droppable_range.end);
      drop_tokens_end = droppable_range.end;
      // If no labels exist, the range was duplicated, so the index needs to
      // be updated.
      if (no_labels) {
        droppable_ranges[0].end = drop_tokens_start - 1;
      }
    }
    // For context before the first label, drop a prefix of the sentence. //
    else if (droppable_range.start == 0) {  // NOLINT
      drop_tokens_start = 0;
      drop_tokens_end = absl::Uniform(absl::IntervalClosed, bitgenref_, 0,
                                      droppable_range.end - 1);
    }
    // For context between two labels, drop a subsequence.
    else {  // NOLINT
      // At least the first and the last tokens must not be dropped.
      if (droppable_range.end - droppable_range.start == 1) {
        continue;
      }

      drop_tokens_start =
          absl::Uniform(absl::IntervalClosed, bitgenref_,
                        droppable_range.start + 1, droppable_range.end - 1);
      drop_tokens_end =
          absl::Uniform(absl::IntervalClosed, bitgenref_, drop_tokens_start,
                        droppable_range.end - 1);
    }

    const TokenRange boundaries =
        TokenRange{.start = drop_tokens_start, .end = drop_tokens_end};
    const int removed_characters = DropText(boundaries, augmented_document);
    DropTokens(boundaries, augmented_document);
    ShiftTokenBoundaries(boundaries.start, -removed_characters,
                         augmented_document);
    DropLabeledSpans(boundaries, augmented_document);
    ShiftLabeledSpansForDroppedTokens(
        drop_tokens_end + 1, -(drop_tokens_end - drop_tokens_start + 1),
        augmented_document);
  }
}

void Augmenter::MaybeDropContextDropLabels(
    bert_annotator::Document* const augmented_document) {
  if (augmentations_.prob_context_drop_outside_one_label == 0) {
    return;
  }

  const int token_count = augmented_document->token_size();
  std::vector<TokenRange> labeled_ranges = GetLabeledRanges(
      *augmented_document, {kAddressReplacementLabel, kPhoneReplacementLabel});
  // MaybeDropContextKeepLabels already implements dropping from sentences
  // without any labels.
  if (labeled_ranges.size() == 0) {
    return MaybeDropContextKeepLabels(
        augmentations_.prob_context_drop_outside_one_label, augmented_document);
  }

  const int label_id =
      absl::Uniform(bitgenref_, 0, static_cast<int>(labeled_ranges.size()));
  TokenRange labeled_range = labeled_ranges[label_id];
  if (labeled_range.end < token_count - 2) {
    TokenRange drop_range{.start = 0, .end = token_count - 1};
    const bool drop_right = absl::Bernoulli(
        bitgenref_, augmentations_.prob_context_drop_outside_one_label);
    if (drop_right) {
      drop_range.start = absl::Uniform(absl::IntervalClosed, bitgenref_,
                                       labeled_range.end + 2, token_count - 1);
      const int removed_characters = DropText(drop_range, augmented_document);
      DropTokens(drop_range, augmented_document);
      ShiftTokenBoundaries(drop_range.start, -removed_characters,
                           augmented_document);
      DropLabeledSpans(drop_range, augmented_document);
      ShiftLabeledSpansForDroppedTokens(
          drop_range.end + 1, -(drop_range.end - drop_range.start + 1),
          augmented_document);
    }
  }
  if (labeled_range.start > 1) {
    TokenRange drop_range{.start = 0, .end = 0};
    const bool drop_left = absl::Bernoulli(
        bitgenref_, augmentations_.prob_context_drop_outside_one_label);
    if (drop_left) {
      drop_range.end = absl::Uniform(absl::IntervalClosed, bitgenref_, 0,
                                     labeled_range.start - 2);
      const int removed_characters = DropText(drop_range, augmented_document);
      DropTokens(drop_range, augmented_document);
      ShiftTokenBoundaries(drop_range.start, -removed_characters,
                           augmented_document);
      DropLabeledSpans(drop_range, augmented_document);
      ShiftLabeledSpansForDroppedTokens(
          drop_range.end + 1, -(drop_range.end - drop_range.start + 1),
          augmented_document);
    }
  }
}

void Augmenter::MaybeReplaceLabel(const double probability,
                                  RandomSampler* const sampler,
                                  const absl::string_view label,
                                  const bool split_into_tokens,
                                  bert_annotator::Document* const document) {
  const std::vector<TokenRange>& labeled_ranges =
      GetLabeledRanges(*document, {label});

  // The replacement invalidates the boundaries of labeled ranges that occur to
  // the right, so it has to be done right to left.
  for (int i = labeled_ranges.size() - 1; i >= 0; --i) {
    TokenRange labeled_range = labeled_ranges[i];
    if (!absl::Bernoulli(bitgenref_, probability)) {
      continue;
    }
    const std::string replacement = sampler->Sample();
    ReplaceLabeledTokens(labeled_range, replacement, label, split_into_tokens,
                         document);
  }
}

void Augmenter::MergePhoneNumberTokens(
    bert_annotator::Document* const document) const {
  const google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>&
      labeled_spans = GetLabelList(*document);
  for (int label_index = labeled_spans.size() - 1; label_index >= 0;
       --label_index) {
    const bert_annotator::LabeledSpan& labeled_span =
        labeled_spans.at(label_index);
    const int token_start = labeled_span.token_start();
    const int token_end = labeled_span.token_end();
    if (labeled_span.label() == kPhoneReplacementLabel) {
      std::string merged_token_text;
      for (int token_index = token_start; token_index <= token_end;
           ++token_index) {
        if (token_index > token_start) {
          for (int intermediate_char_index =
                   document->token(token_index - 1).end() + 1;
               intermediate_char_index < document->token(token_index).start();
               ++intermediate_char_index) {
            absl::StrAppend(
                &merged_token_text,
                document->text().substr(intermediate_char_index, 1));
          }
        }
        absl::StrAppend(&merged_token_text,
                        document->token(token_index).word());
      }
      bert_annotator::Token merged_token;
      merged_token.set_start(document->token(token_start).start());
      merged_token.set_end(document->token(token_end).end());
      merged_token.set_word(merged_token_text);

      DropTokens(TokenRange{.start = token_start, .end = token_end}, document);
      InsertTokens(token_start, {merged_token}, document);
      ShiftLabeledSpansForDroppedTokens(token_start + 1,
                                        -(token_end - token_start), document);
    }
  }
}

void Augmenter::DropSeparatorTokens(
    bert_annotator::Document* const document) const {
  for (int i = document->token_size() - 1; i >= 0; --i) {
    const bert_annotator::Token& token = document->token(i);
    // Note: depends on the installed C locale, may need to be changed
    // for non-english languages.
    if (absl::c_none_of(token.word(), [](unsigned char c) {
          return std::isdigit(c) || std::isalpha(c);
        })) {
      const TokenRange removed_token = TokenRange{.start = i, .end = i};
      DropTokens(removed_token, document);

      // If this was the only token of a label (i.e., the whole label consisted
      // of non-alphanumeric values, such as emoticons), the label should be
      // dropped. If the label remains, its bounds have to be updated manually,
      // as ShiftLabeledSpansForDroppedTokens would update both start and end.
      int label_shift_start = removed_token.start;
      google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>* const
          labeled_spans = GetLabelList(document);
      for (bert_annotator::LabeledSpan& labeled_span : *labeled_spans) {
        if (labeled_span.token_start() == i && labeled_span.token_end() == i) {
          DropLabeledSpans(removed_token, document);
        } else if (labeled_span.token_start() <= i &&
                   labeled_span.token_end() >= i) {
          labeled_span.set_token_end(labeled_span.token_end() - 1);
          label_shift_start = labeled_span.token_end() + 1;
        }
      }

      ShiftLabeledSpansForDroppedTokens(label_shift_start, -1, document);
    }
  }
}

void Augmenter::UnifyAndMergeAddressLabels(
    bert_annotator::Document* const document) const {
  google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>* const
      labeled_spans = GetLabelList(document);
  for (int i = labeled_spans->size() - 1; i >= 0; --i) {
    bert_annotator::LabeledSpan& labeled_span = labeled_spans->at(i);
    if (kAddressLabels.contains(labeled_span.label())) {
      labeled_span.set_label(std::string(Augmenter::kAddressReplacementLabel));

      // Consecutive address labels should be merged.
      if (i <= labeled_spans->size() - 2) {
        bert_annotator::LabeledSpan& successor_span = labeled_spans->at(i + 1);
        if (successor_span.label() == Augmenter::kAddressReplacementLabel &&
            labeled_span.token_end() + 1 == successor_span.token_start()) {
          labeled_span.set_token_end(successor_span.token_end());
          labeled_spans->erase(labeled_spans->begin() + i + 1);
        }
      }
    }
  }
}

const int Augmenter::ReplaceText(
    const TokenRange& boundaries, const std::string& replacement,
    bert_annotator::Document* const document) const {
  const int string_start = document->token(boundaries.start).start();
  const int string_end = document->token(boundaries.end).end();
  std::string new_text;
  new_text.append(document->mutable_text()->begin(),
                  document->mutable_text()->begin() + string_start);
  new_text.append(replacement);
  if (static_cast<int>(document->text().size()) > string_end) {
    new_text.append(document->mutable_text()->begin() + string_end + 1,
                    document->mutable_text()->end());
  }
  document->set_text(new_text);

  return replacement.size() - (string_end - string_start + 1);
}

const int Augmenter::DropText(const TokenRange& boundaries,
                              bert_annotator::Document* const document) const {
  int text_start;
  int text_end;
  if (boundaries.start > 0) {
    text_start = document->token(boundaries.start - 1).end() + 1;
    text_end = document->token(boundaries.end).end();
  } else {
    text_start = 0;
    if (boundaries.end < document->token_size() - 1) {
      text_end = document->token(boundaries.end + 1).start() - 1;
    } else {
      text_end = document->text().size() - 1;
    }
  }

  std::string new_text;
  new_text.append(document->mutable_text()->begin(),
                  document->mutable_text()->begin() + text_start);
  if (static_cast<int>(document->text().size()) > text_end) {
    new_text.append(document->mutable_text()->begin() + text_end + 1,
                    document->mutable_text()->end());
  }
  document->set_text(new_text);

  return text_end - text_start + 1;
}

void Augmenter::InsertTokens(int index,
                             const std::vector<bert_annotator::Token> tokens,
                             bert_annotator::Document* const document) const {
  for (bert_annotator::Token token : tokens) {
    bert_annotator::Token* const new_token = document->add_token();
    new_token->CopyFrom(token);

    for (int i = document->token_size() - 1; i > index; --i) {
      document->mutable_token()->SwapElements(i, i - 1);
    }
    ++index;
  }
}

void Augmenter::DropTokens(const TokenRange boundaries,
                           bert_annotator::Document* const document) const {
  document->mutable_token()->erase(
      document->mutable_token()->begin() + boundaries.start,
      document->mutable_token()->begin() + boundaries.end +
          1);  // end is inclusive.
}

void Augmenter::ShiftTokenBoundaries(
    const int first_token, const int shift,
    bert_annotator::Document* const document) const {
  for (int i = first_token; i < document->token_size(); ++i) {
    bert_annotator::Token* token = document->mutable_token(i);
    token->set_start(token->start() + shift);
    token->set_end(token->end() + shift);
  }
}

void Augmenter::DropLabeledSpans(
    const TokenRange& removed_tokens,
    bert_annotator::Document* const document) const {
  google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>* const
      labeled_spans = GetLabelList(document);
  for (int i = labeled_spans->size() - 1; i >= 0; --i) {
    bert_annotator::LabeledSpan* const labeled_span = labeled_spans->Mutable(i);
    if (labeled_span->token_end() >= removed_tokens.start &&
        labeled_span->token_start() <= removed_tokens.end) {
      labeled_spans->erase(labeled_spans->begin() + i);
    }
  }
}

void Augmenter::InsertLabeledSpan(
    const TokenRange& range, const absl::string_view label,
    bert_annotator::Document* const document) const {
  google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>* const
      labeled_spans = GetLabelList(document);
  bert_annotator::LabeledSpan* const new_span = labeled_spans->Add();
  new_span->set_label(std::string(label));
  new_span->set_token_start(range.start);
  new_span->set_token_end(range.end);

  int i = labeled_spans->size() - 1;
  while (i > 0 && labeled_spans->at(i).token_start() <
                      labeled_spans->at(i - 1).token_end()) {
    labeled_spans->SwapElements(i, i - 1);
    --i;
  }
}

void Augmenter::ShiftLabeledSpansForDroppedTokens(
    const int start, const int shift,
    bert_annotator::Document* const document) const {
  google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>* const
      labeled_spans = GetLabelList(document);
  for (int i = labeled_spans->size() - 1; i >= 0; --i) {
    bert_annotator::LabeledSpan* const labeled_span = labeled_spans->Mutable(i);
    if (labeled_span->token_start() >= start) {
      labeled_span->set_token_start(labeled_span->token_start() + shift);
    }
    if (labeled_span->token_end() >= start) {
      labeled_span->set_token_end(labeled_span->token_end() + shift);
    }
  }
}

int Augmenter::RemovePrefixPunctuation(absl::string_view* const text) const {
  int removed_punctuation = 0;
  bool punctuation_removed = true;
  while (punctuation_removed && text->size() > 0) {
    punctuation_removed = false;
    const char first_char = text->at(0);
    if (!std::isalpha(first_char) && !std::isdigit(first_char)) {
      text->remove_prefix(1);
      ++removed_punctuation;
      punctuation_removed = true;
    }
  }
  return removed_punctuation;
}

int Augmenter::RemoveSuffixPunctuation(absl::string_view* const text) const {
  int removed_punctuation = 0;
  bool punctuation_removed = true;
  while (punctuation_removed && text->size() > 0) {
    punctuation_removed = false;
    const char last_char = text->at(text->size() - 1);
    if (!std::isalpha(last_char) && !std::isdigit(last_char)) {
      text->remove_suffix(1);
      ++removed_punctuation;
      punctuation_removed = true;
    }
  }
  return removed_punctuation;
}

std::vector<bert_annotator::Token> Augmenter::SplitTextIntoTokens(
    int text_start, const absl::string_view text) const {
  std::vector<bert_annotator::Token> new_tokens;
  int replacement_text_index = 0;
  for (absl::string_view potential_token_text : absl::StrSplit(text, " ")) {
    replacement_text_index += RemovePrefixPunctuation(&potential_token_text);
    int trailing_punctuation = RemoveSuffixPunctuation(&potential_token_text);

    if (potential_token_text.size() > 0) {
      bert_annotator::Token token;
      token.set_word(std::string(potential_token_text));
      token.set_start(text_start + replacement_text_index);
      token.set_end(text_start + replacement_text_index +
                    potential_token_text.size() - 1);
      new_tokens.push_back(token);
    }
    replacement_text_index +=
        potential_token_text.size() + 1 + trailing_punctuation;
  }

  return new_tokens;
}

void Augmenter::ReplaceLabeledTokens(
    const TokenRange& boundaries, const absl::string_view replacement,
    const absl::string_view replacement_label, const bool split_into_tokens,
    bert_annotator::Document* const document) const {
  const int text_start = document->token(boundaries.start).start();
  std::vector<bert_annotator::Token> new_tokens;
  if (split_into_tokens) {
    new_tokens = SplitTextIntoTokens(text_start, replacement);
  } else {
    bert_annotator::Token token;
    token.set_word(std::string(replacement));
    token.set_start(text_start);
    token.set_end(text_start + replacement.size() - 1);
    new_tokens.push_back(token);
  }

  const int text_shift =
      ReplaceText(boundaries, std::string(replacement), document);
  DropTokens(boundaries, document);
  InsertTokens(boundaries.start, new_tokens, document);
  ShiftTokenBoundaries(boundaries.start + new_tokens.size(), text_shift,
                       document);
  DropLabeledSpans(boundaries, document);
  const int token_number_increase =
      new_tokens.size() - (boundaries.end - boundaries.start + 1);
  // The remaining tokens should be shifted first, so they do not overlap with
  // the new ones.
  ShiftLabeledSpansForDroppedTokens(boundaries.end + 1, token_number_increase,
                                    document);
  InsertLabeledSpan(TokenRange{.start = boundaries.start,
                               .end = boundaries.start +
                                      static_cast<int>(new_tokens.size()) - 1},
                    replacement_label, document);
}

void Augmenter::InitializeLabelList(
    bert_annotator::Document* const document) const {
  if (document->labeled_spans().find(kLabelContainerName) ==
      document->labeled_spans().end()) {
    (*document->mutable_labeled_spans())[kLabelContainerName] = {};
  }
}

google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>
Augmenter::GetLabelList(const bert_annotator::Document& document) const {
  return document.labeled_spans().at(kLabelContainerName).labeled_span();
}

google::protobuf::RepeatedPtrField<bert_annotator::LabeledSpan>* const
Augmenter::GetLabelList(bert_annotator::Document* document) const {
  return document->mutable_labeled_spans()
      ->at(kLabelContainerName)
      .mutable_labeled_span();
}

void Augmenter::MaskDigits(std::string* text) const {
  absl::StrReplaceAll({{"1", "0"},
                       {"2", "0"},
                       {"3", "0"},
                       {"4", "0"},
                       {"5", "0"},
                       {"6", "0"},
                       {"7", "0"},
                       {"8", "0"},
                       {"9", "0"}},
                      text);
}

void Augmenter::MaskDigits(bert_annotator::Document* const document) const {
  MaskDigits(document->mutable_text());

  for (bert_annotator::Token& token : *document->mutable_token()) {
    MaskDigits(token.mutable_word());
  }
}

const absl::flat_hash_set<absl::string_view>& Augmenter::kAddressLabels =
    *new absl::flat_hash_set<absl::string_view>(
        {"LOCALITY", "COUNTRY", "ADMINISTRATIVE_AREA", "THOROUGHFARE",
         "THOROUGHFARE_NUMBER", "PREMISE", "POSTAL_CODE", "PREMISE_LEVEL"});

constexpr absl::string_view Augmenter::kAddressReplacementLabel;

const absl::flat_hash_set<absl::string_view>& Augmenter::kPhoneLabels = {
    "TELEPHONE"};

constexpr absl::string_view Augmenter::kPhoneReplacementLabel;

const std::string& Augmenter::kLabelContainerName = *new std::string("lucid");

const std::vector<absl::string_view>&
    Augmenter::kPunctuationReplacementsWithinText =
        *new std::vector<absl::string_view>({", ", "; ", ": ", " - "});

const std::vector<absl::string_view>&
    Augmenter::kPunctuationReplacementsAtSentenceEnd =
        *new std::vector<absl::string_view>({"?", "!", ".", ":", ";", " - "});

}  // namespace augmenter


text:
<!DOCTYPE html>

  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  <meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1"/>
  <title>Reliable Message Delivery - Gearpump 0.7.3 Documentation</title>

  <link rel="stylesheet" href="css/bootstrap-3.3.5.min.css">

  <link rel="stylesheet" href="css/main.css">
  <link rel="stylesheet" href="css/pygments-default.css">

        Gearpump
          0.7.3

Overview

            Introduction

Submit Your 1st Application

Client Command Line

Basic Concepts

Technical Highlights

Reliable Message Delivery

Performance

Gearpump Internals

            Deploying

Deployment

Local Mode

Standalone Mode

YARN Mode

Docker Mode

High Availability

Reliable Message Delivery

Configuration

Security

            Programming Guide

Write Your 1st App

Customized Message Passing

Scala API

Java API

RESTful API

Gearpump Connectors

Storm Compatibility

            More

How to Contribute

Coding Style

FAQ

About

Reliable Message Delivery

What is At Least Once Message Delivery?

Messages could be lost on delivery due to network partitions. At Least Once Message Delivery (at least once) means the lost messages are delivered one or more times such that at least one is processed and acked by the whole flow.

Gearpump guarantees at least once for any source that is able to replay message from a past timestamp. In Gearpump, each message is tagged with a timestamp, and the system tracks the minimum timestamp of all pending messages (the global minimum clock). On message loss, application will be restarted to the global minimum clock. Since the source is able to replay from the gloabl minimum clock, all pending messages before the restart will be replayed. Gearpump calls that kind of source TimeReplayableSource and already provides a built in
KafkaSource. With the KafkaSource to ingest data into Gearpump, users are guaranteed at least once message delievery.

What is Exactly Once Message Delivery?

At least once delivery doesn’t guarantee the correctness of the application result. For instance,  for a task keeping the count of received messages, there could be overcount with duplicated messages and the count is lost on task failure.
 In that case, Exactly Once Message Delivery (exactly once) is required, where state is updated by a message exactly once. This further requires that duplicated messages are filtered out and in-memory states are persisted.

Users are guaranteed exactly once in Gearpump if they use both a TimeReplayableSource to ingest data and the Persistent API to manage their in memory states. With the Persistent API, user state is periodically checkpointed by the system to a persistent store (e.g HDFS) along with its checkpointed time. Gearpump tracks the global minimum checkpoint timestamp of all pending states (global minimum checkpoint clock), which is persisted as well. On application restart, the system restores states at the global minimum checkpoint clock and source replays messages from that clock. This ensures that a message updates all states exactly once.

Persistent API

Persistent API consists of PersistentTask and PersistentState.

Here is an example of using them to keep count of incoming messages.

class CountProcessor(taskContext: TaskContext, conf: UserConfig)
  extends PersistentTask[Long](taskContext, conf) {

  override def persistentState: PersistentState[Long] = {
    import com.twitter.algebird.Monoid.longMonoid
    new NonWindowState[Long](new AlgebirdMonoid(longMonoid), new ChillSerializer[Long])
  }

  override def processMessage(state: PersistentState[Long], message: Message): Unit = {
    state.update(message.timestamp, 1L)
  }
}

The CountProcessor creates a customized PersistentState which will be managed by PersistentTask and overrides the processMessage method to define how the state is updated on a new message (each new message counts as 1, which is added to the existing value)

Gearpump has already offered two types of states

NonWindowState - state with no time or other boundary

WindowState - each state is bounded by a time window

They are intended for states that satisfy monoid laws.

has binary associative operation, like +

has an identity element, like 0

In the above example, we make use of the longMonoid from Twitter’s Algebird library which provides a bunch of useful monoids.

redpajama_set_name: RedPajamaGithub


text:
package org.zella.warandpeace.detail;

import android.content.Context;
import android.support.v4.view.PagerAdapter;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.ProgressBar;
import android.widget.Toast;

import com.squareup.picasso.Callback;
import com.squareup.picasso.Picasso;

import org.zella.warandpeace.R;

import java.util.List;

import uk.co.senab.photoview.PhotoView;

/**
 * Created by dru on 24.08.15.
 */
public class ViewPagerAdapter extends PagerAdapter {

   private final List<String> mUrls;

   public ViewPagerAdapter(List<String> imageUrls) {
      this.mUrls = imageUrls;
   }

   @Override
   public int getCount() {
      return mUrls.size();
   }

   @Override
   public View instantiateItem(ViewGroup container, int position) {

      final LayoutInflater inflater = (LayoutInflater) container.getContext()
           .getSystemService(Context.LAYOUT_INFLATER_SERVICE);

      final View v = inflater.inflate(R.layout.item_viewpager, null);

      final PhotoView photoView = (PhotoView) v.findViewById(R.id.photo_view_image);

      final View progressBar =  v.findViewById(R.id.photo_view_progress);

      Picasso.with(container.getContext())
           .load(mUrls.get(position))
                //  .placeholder(R.drawable.user_placeholder)
                //  .error(R.drawable.user_placeholder_error)
           .into(photoView, new Callback() {
              @Override
              public void onSuccess() {
                 progressBar.setVisibility(View.INVISIBLE);
                 photoView.setVisibility(View.VISIBLE);
              }

              @Override
              public void onError() {
                 progressBar.setVisibility(View.INVISIBLE);
                 Toast.makeText(container.getContext(), "Error loading image", Toast.LENGTH_SHORT).show();
              }
           });

      container.addView(v);

      return v;
   }

   @Override
   public void destroyItem(ViewGroup container, int position, Object object) {
      container.removeView((View) object);
   }

   @Override
   public boolean isViewFromObject(View view, Object object) {
      return view == object;
   }

}


text:
Gubernator is a frontend for displaying Kubernetes test results stored in GCS.

It runs on Google App Engine, and parses JSON and junit.xml results for display.

https://gubernator.k8s.io/

# Adding a repository to the PR Dashboard

To make Gubernator's [PR Dashboard](https://gubernator.k8s.io/pr) work
on another repository, it needs to receive webhook events.

Go to Settings -> Webhooks on the repository (or organization) you want to add.

Add a new webhook with these options:

Payload URL: https://github-dot-k8s-gubernator.appspot.com/webhook
Secret: Ask test-infra oncall.
Select: "Send me everything"

Gubernator will use the events it receives to build information about PRs, so
only updates after the webhook is added will be shown on the dashboard.

# Development

- Install the Google Cloud SDK: https://cloud.google.com/sdk/
- Run locally using `dev_appserver.py app.yaml` and visit http://localhost:8080
- Test and lint using `./test-gubernator.sh`
- Deploy with `make deploy` followed by `make migrate`

# Deployment

- Visit /config on the new deployment to configure Github [OAuth logins](https://github.com/settings/applications)
  and webhook secrets.

# GCS Layout

In order to correctly interpret jobs results, in GCS, Gubernator expects that
any one job directory is laid out in a specific manner, and that job directories
are laid out in a specific manner relative to each other.

## Job Artifact GCS Layout

Every run should upload `started.json`, `finished.json`, and `build-log.txt`, and
can optionally upload jUnit XML and/or other files to the `artifacts/` directory.
For a single build of a job, Gubernator expects the following layout in GCS:

```
.
├── artifacts         # all artifacts must be placed under this directory
│   └── junit_00.xml  # jUnit XML reports from the build
├── build-log.txt     # std{out,err} from the build
├── finished.json     # metadata uploaded once the build finishes
└── started.json      # metadata uploaded once the build starts
```

The following fields in `started.json` are honored:

```json
{
    "timestamp": "seconds after UNIX epoch that the build started",
    "pull": "$PULL_REFS from the run",
    "repos": {
        "org/repo": "git version of the repo used in the test",
    },
}
```

The following fields in `finished.json` are honored:

```json
{
    "timestamp": "seconds after UNIX epoch that the build finished",
    "result": "SUCCESS or FAILURE, the result of the build",
    "metadata": "dictionary of additional key-value pairs that will be displayed to the user",
}
```

Any artifacts from the build should be placed under `./artifacts/`. Any jUnit
XML reports should be named `junit_*.xml` and placed under `./artifacts` as well.

## Test Properties [Optional]

Test properties are a set of key value pairs defined on the test case and are optional. The test
result file `junit_*.xml` contains a list of test cases and the properties associated with it.
These properties can be later parsed by any aggregator like testgrid, and used to collect metrics
about the test case.

The properties can be defined as:

```xml
<testcase ...>
  <properties>
    <property>
        <name>key1</name>
        <value>value1</value>
    </property>
    <property>
        <name>key2</name>
        <value>value2</value>
    </property>
  </properties>
</testcase>
```

## GCS Bucket Layout

In your bucket, a number of directories are required to store the output of all
the different types of jobs:

```
.
├── logs                    # periodic and postsubmit jobs live here
│   └── other_job_name      # contains all the builds of a job
│      ├── build_number     # contains job artifacts, as above
│      └── latest-build.txt # contains the latest build id of a job
└── pr-logs
    ├── directory                # symlinks for builds live here
    │   └── job_name             # contains all symlinks for a job
    │       ├── build_number.txt # contains one line: location of artifacts for the build
    │       └── latest-build.txt # contains the latest build id of a job
    └── pull
        ├── batch                # batch jobs live here
        │   └── job_name         # contains all the builds of a job
        │       └── build_number # contains job artifacts, as above
        └── org_repo                     # jobs testing PRs for org/repo live here
            └── pull_number              # jobs running for a PR with pull_number live here
                └── job_name             # all builds for the job for this pr live here
                    └── build_number     # contains job artifacts, as above
                    └── latest-build.txt # contains the latest build id of a job
```

# Migrations

1. 2018-01-09: Github webhook and oauth secrets must be reconfigured. Visit
   /config on your deployment to enter the new values.


text:
var assert = require('assert'),
    nunjucks = require('nunjucks'),
    reverser = require('../urlreverser')

describe('urlreverser', function () {
    function eq(a, b) { assert.strictEqual(a, b) }
    function noeq(a, b) { assert.notEqual(a, b) }

    it('should return same url', function () {
        var url = '(?:/base/:base?)/(?:/base2/:base2?)/?f=:link'
        eq(reverser.url(url), url)
    })

    it('should work without name', function() {
        reverser.url('/foo/:bar')
        eq(reverser.reverse('/foo/:bar', {bar: 'bar'}), '/foo/bar')
    })

    it('should work with php links', function() {
        reverser.url('/link/?q=:q(?:&v=:v)?', 'link')
        eq(reverser.reverse('link', {q: 's', v: '1'}), '/link/?q=s&v=1')
    })

    it('should reverse simple url', function () {
        reverser.url('/foo', 'foo')
        eq(reverser.reverse('foo'), '/foo')

        reverser.url('/foo/:bar', 'bar')
        eq(reverser.reverse('bar', {bar: 'bar'}), '/foo/bar')
    })

    it('should reverse without parameters', function () {
        reverser.url('/foo/:baz?', 'baz')
        eq(reverser.reverse('baz'), '/foo/')
    })

    it('should omit groups', function () {
        reverser.url('/group(?:/foo/:bar)?', 'group')
        eq(reverser.reverse('group', {bar: 'a'}), '/group/foo/a')
        eq(reverser.reverse('group'), '/group')
    })

    it('should provide template to nunjucks', function () {
        var env = new nunjucks.Environment()
        reverser.nunjucks(env)
        reverser.url('/foo/:bar', 'bar')
        eq(env.renderString("{% url 'bar', bar='baz' %}"), '/foo/baz')
    })
})


text:
package org.apache.spark.scheduler

import java.util.Arrays
import java.util.Objects

import org.apache.spark._
import org.apache.spark.rdd.RDD

/**
 * A Partitioner that might group together one or more partitions from the parent.
 *
 * @param parent a parent partitioner
 * @param partitionStartIndices indices of partitions in parent that should create new partitions
 *   in child (this should be an array of increasing partition IDs). For example, if we have a
 *   parent with 5 partitions, and partitionStartIndices is [0, 2, 4], we get three output
 *   partitions, corresponding to partition ranges [0, 1], [2, 3] and [4] of the parent partitioner.
 */
class CoalescedPartitioner(val parent: Partitioner, val partitionStartIndices: Array[Int])
  extends Partitioner {

  @transient private lazy val parentPartitionMapping: Array[Int] = {
    val n = parent.numPartitions
    val result = new Array[Int](n)
    for (i <- 0 until partitionStartIndices.length) {
      val start = partitionStartIndices(i)
      val end = if (i < partitionStartIndices.length - 1) partitionStartIndices(i + 1) else n
      for (j <- start until end) {
        result(j) = i
      }
    }
    result
  }

  override def numPartitions: Int = partitionStartIndices.size

  override def getPartition(key: Any): Int = {
    parentPartitionMapping(parent.getPartition(key))
  }

  override def hashCode(): Int =
    31 * Objects.hashCode(parent) + Arrays.hashCode(partitionStartIndices)

  override def equals(other: Any): Boolean = other match {
    case c: CoalescedPartitioner =>
      c.parent == parent && Arrays.equals(c.partitionStartIndices, partitionStartIndices)
    case _ =>
      false
  }
}

private[spark] class CustomShuffledRDDPartition(
    val index: Int, val startIndexInParent: Int, val endIndexInParent: Int)
  extends Partition {

  override def hashCode(): Int = index

  override def equals(other: Any): Boolean = super.equals(other)
}

/**
 * A special ShuffledRDD that supports a ShuffleDependency object from outside and launching reduce
 * tasks that read multiple map output partitions.
 */
class CustomShuffledRDD[K, V, C](
    var dependency: ShuffleDependency[K, V, C],
    partitionStartIndices: Array[Int])
  extends RDD[(K, C)](dependency.rdd.context, Seq(dependency)) {

  def this(dep: ShuffleDependency[K, V, C]) = {
    this(dep, (0 until dep.partitioner.numPartitions).toArray)
  }

  override def getDependencies: Seq[Dependency[_]] = List(dependency)

  override val partitioner = {
    Some(new CoalescedPartitioner(dependency.partitioner, partitionStartIndices))
  }

  override def getPartitions: Array[Partition] = {
    val n = dependency.partitioner.numPartitions
    Array.tabulate[Partition](partitionStartIndices.length) { i =>
      val startIndex = partitionStartIndices(i)
      val endIndex = if (i < partitionStartIndices.length - 1) partitionStartIndices(i + 1) else n
      new CustomShuffledRDDPartition(i, startIndex, endIndex)
    }
  }

  override def compute(p: Partition, context: TaskContext): Iterator[(K, C)] = {
    val part = p.asInstanceOf[CustomShuffledRDDPartition]
    SparkEnv.get.shuffleManager.getReader(
      dependency.shuffleHandle, part.startIndexInParent, part.endIndexInParent, context)
      .read()
      .asInstanceOf[Iterator[(K, C)]]
  }

  override def clearDependencies() {
    super.clearDependencies()
    dependency = null
  }
}


text:
<?php

namespace Magento\LayeredNavigation\Block\Navigation;

interface FilterRendererInterface
{
    /**
     * Render filter
     *
     * @param \Magento\Catalog\Model\Layer\Filter\AbstractFilter $filter
     * @return string
     */
    public function render(\Magento\Catalog\Model\Layer\Filter\AbstractFilter $filter);
}


text:
<?php
/**
 * @author Jacky Jiang
 * @version 0.1.1
 * Only support UTF-8
 */

if(!defined('IN_APP')) {
	exit('Access Denied');
}
include APP_ROOT.'include/CActiveRecord.class.php';
include APP_ROOT.'exceptions/CDBException.php';
class dbstuff {
	var $querynum = 0;
	var $link;
	var $query;

	private $dbhost;
	private $dbuser;
	private $dbpw;
	private $dbname;
	private $pconnect;

	function connect($dbhost, $dbuser, $dbpw, $dbname = '', $pconnect = 0, $halt = TRUE) {
		$this->dbhost=$dbhost;
		$this->dbuser=$dbuser;
		$this->dbpw=$dbpw;
		$this->dbname=$dbname;
		$this->pconnect=$pconnect;
		if($pconnect) {
			if(!$this->link = @mysql_pconnect($dbhost, $dbuser, $dbpw)) {
				$halt && $this->halt('Can not connect to MySQL server');
			}
		} else {
			if(!$this->link = @mysql_connect($dbhost, $dbuser, $dbpw, 1)) {
				$halt && $this->halt('Can not connect to MySQL server');
			}
		}
		if($this->version() > '4.1') {
			$dbcharset='utf8';
			if($dbcharset) {
				@mysql_query("SET character_set_connection=$dbcharset, character_set_results=$dbcharset, character_set_client=binary", $this->link);
			}
			if($this->version() > '5.0.1') {
				@mysql_query("SET sql_mode=''", $this->link);
			}
		}

		if($dbname) {
			@mysql_select_db($dbname, $this->link);
		}
	}

	function select_db($dbname) {
		return mysql_select_db($dbname, $this->link);
	}

	function fetch_array($query, $result_type = MYSQL_ASSOC) {
		return mysql_fetch_array($query, $result_type);
	}

	function real_escape_string($str)
	{
		return mysql_real_escape_string((string)$str,$this->link);
	}

	function query($sql, $params=null, $type = '')
	{
		global $APP_ENV;
		$func = $type == 'UNBUFFERED' && @function_exists('mysql_unbuffered_query') ? 'mysql_unbuffered_query' : 'mysql_query';
		if(!empty($params) && is_array($params)) {
			foreach($params as $k=>$v)
			{
				if($v===NULL) $params[$k]='NULL';
				else $params[$k]='\''.mysql_real_escape_string((string)$v,$this->link).'\'';
			}
			$sql=str_replace(array_keys($params),array_values($params),$sql);
		}
		if(!($query = $func($sql, $this->link))) {
			if(in_array($this->errno(), array(2006, 2013)) && substr($type, 0, 5) != 'RETRY') {
				$this->close();
				$this->connect($this->dbhost,$this->dbuser,$this->dbpw, $this->dbname, $this->pconnect);
				$query = $this->query($sql, $params, 'RETRY'.$type);
			} elseif($type != 'SILENT' && substr($type, 5) != 'SILENT') {
				$this->halt('MySQL Query Error', $sql);
			}
		}

		$this->querynum++;
		if($APP_ENV['debug'])
		{
			$APP_ENV['debugInfo']['curSql']=$sql;
			$APP_ENV['debugInfo']['sqlStack'][]=$sql;
		}
		$this->query=$query;
		$this->error();
		return $query;
	}

	function affected_rows() {
		return mysql_affected_rows($this->link);
	}

	function error() {
		return (($this->link) ? mysql_error($this->link) : mysql_error());
	}

	function errno() {
		return intval(($this->link) ? mysql_errno($this->link) : mysql_errno());
	}

	function result($query, $row) {
		$query = @mysql_result($query, $row);
		return $query;
	}

	function num_rows($query=NULL) {
		if(!isset($query)) $query=$this->query;
		$query = mysql_num_rows($query);
		return $query;
	}

	function num_fields($query) {
		return mysql_num_fields($query);
	}

	function free_result($query) {
		return mysql_free_result($query);
	}

	function insert_id() {
		return ($id = mysql_insert_id($this->link)) >= 0 ? $id : $this->result($this->query("SELECT last_insert_id()"), 0);
	}

	function fetch_row($query) {
		$query = mysql_fetch_row($query);
		return $query;
	}

	function fetch_fields($query) {
		return mysql_fetch_field($query);
	}

	function version() {
		return mysql_get_server_info($this->link);
	}

	function close() {
		return mysql_close($this->link);
	}

	function halt($message = '', $sql = '')
	{
		global $APP_ENV;
		define('CACHE_FORBIDDEN', TRUE);
		throw new CDBException($message.":\n<br/>".$this->error(),$this->errno(),$sql);
	}

	/*
	$table: table name should insert records into
	$record: array contains name- value pair

	*/

	function insert($table,$record)
	{
		$sql="INSERT INTO $table (".implode(', ', array_keys($record)).") VALUES ('".implode('\', \'', array_values($record))."')";
		return $this->query($sql);
	}

		/*
	$table: table name should insert records into
	$record: array contains name- value pair

	*/

	function update($table,$record,$where)
	{
		foreach($values as $key => $val)
		{
			$valstr[] = $key." = ".$val;
		}
		$sql="UPDATE $table SET ".implode(', ', $valstr)." WHERE $where;";
		return $this->query($sql);
	}

	function fetchResultBySQL($sql,$params=null,$arg1=null,$arg2=null)
	{
		$limtText='';
		if(isset($arg2)) $limtText=" LIMIT {$arg1},{$arg2}";
		elseif (isset($arg1)) $limtText=" LIMIT {$arg1}";
		$sql.=$limtText;
		$query=$this->query($sql,$params);
		if($query===false) return false;
		while($row=mysql_fetch_array($query,MYSQL_ASSOC)) $records[]=$row;
		if($records==false) $records=array();
		return $records;
	}

	/*
	Get the Collection object from a table
	Only works on table has a primary key with the name id

	*/
	function oQuery($sql)
	{
		$r=$this->query($sql);
		$table=mysql_field_table($r,0);
		return new CCollection($table,$r);
	}

	function fetchSmartRecord($query)
	{
		if($query===false) return false;
		$table=mysql_field_table($query,0);
		$row=mysql_fetch_array ($query,MYSQL_ASSOC);
		if($row===false) return false;
		return new CSmartRecord($table,$row);
	}
	function fetchSmartRecordById($table,$id)
	{
		$query=$this->query("select * from $table where id='$id' limit 1");
		if($query===false) return false;
		$table=mysql_field_table($query,0);
		$row=mysql_fetch_array ($query,MYSQL_ASSOC);
		if($row===false) return false;
		return new CSmartRecord($table,$row);
	}

	function activeRecordQuery($sql)
	{
		$query=$this->query($sql);
		if($query===false) return false;
		$num=mysql_num_fields($query);
		$tableSet=array();

		for($i=0;$i<$num;$i++)
		{
			mysql_field_seek ($query,$i);
			$obj=mysql_fetch_field($query,$i);
			$tableSet[$obj->table]['collist'][$obj->name]=array(
													'default' => $obj->def,
													'primary_key' => $obj->primary_key);
		}
		foreach($tableSet as $key => $value)
		{
			foreach($value['collist'] as $k => $v)
			{
				if($v['primary_key']==1)
				{
					$tableSet[$key]['primarykeyName']=$k;
				}

			}
			if(!isset($tableSet[$key]['primarykeyName'])) throw new RuntimeException('Can\'t load data set without primary key from table:'.$key);
		}
		$records=array();
		while($row=mysql_fetch_array($query,MYSQL_ASSOC)) $records[]=new CActiveRecord($tableSet,$row);
		return $records;
	}
}

?>


text:
layout: page
#subheadline: Templates
title:  "How to Use SYDI Server with the Windows XP Firewall"
meta_description: "If you're able to run SYDI locally from a computer but you're not able to reach it over the network. Perhaps with a 462 -- The remote server machine does not exist or is unavailable error, this note should interest you."
#teaser: "This is a example of page/post with a sidebar on the left."
breadcrumb: true
tags:
    - post format
categories:
    - howto
#image:
#    title: "unsplash_1.jpg"
#    caption: Unsplash.com
#    caption_url: http://unsplash.com
---
If you're able to run SYDI locally from a computer but you're not able to reach it over the network. Perhaps with a 462 -- The remote server machine does not exist or is unavailable error, this note should interest you. SYDI relies on WMI and the 462 is an error code which has been returned from WMI. You will see this error in two instances, the first one being the machine doesn't exist i.e. you typed the wrong name or weren't able to resolve the name. The other instance is that when the target computer isn't reachable.
<!--more-->
If your clients are running XP and have the firewall, which was shipped in Service Pack 2, enabled the ports SYDI need are disabled by default.

SYDI relies on WMI which in turn uses RPC communication. RPC starts it's communication on TCP/135 and then a dynamic port over TCP/1023 is used, this is negotiated between the client and the server.

In order to allow this communication you would have to change the group policy for the machine you are targeting.

In a domain environment you can change this setting under; Computer Configuration -> Administrative Templates -> Network -> Network Connections -> Windows Firewall -> Domain Profile -> Windows Firewall: Allow remote administration exception. Change the state to Enabled and in the Allow unsolicited incoming messages from field enter the ip address of the machine(s) you will be running SYDI from.

If the machine resides outside a domain you can edit the local group policy by opening up a local group policy from a snap-in in mmc.exe.


text:
<?php
namespace pew\validator;

class StringValidator extends AbstractValidator {

	// TODO implement $notNull
	public static function validate($v, &$res, $notNull, &$msg) {

		$v = (string)$v;
		if ($notNull == true AND strlen($v) < 1) {
			$msg = 'stringValidator: string validation failed';
			$res = '';
			return false;
		}
		$res = $v;
		return true;
	}

}
?>


text:
@interface Member (helper)

+(Member*)createOrFetchWithIdentifier:(NSNumber*)identifier inContext:(NSManagedObjectContext*)context;

-(void)updateWithAttributes:(NSDictionary*)attributes inContext:(NSManagedObjectContext*)context withCompletion:(void(^)(void))completionBlock;

@end


text:
set -xv
## only important ones
export SUBARUDIR=/u/ki/awright/data/
export bonn=/u/ki/awright/wtgpipeline/
export subdir=${SUBARUDIR}
export subarudir=${SUBARUDIR}

export ending="OCFSRI"
export detect_filter=W-C-RC
export lensing_filter=W-C-RC
export INSTRUMENT=SUBARU
export cluster=MACS1115+01
export filter="W-C-RC"
export lens="gabodsid1554"
export lensing_coadd_type="gabodsid1554"
export lensing_image=${SUBARUDIR}/${cluster}/${filter}/SCIENCE/coadd_${cluster}_${lensing_coadd_type}/coadd.fits
export lensing_dir=`dirname ${lensing_image}`
export lensing_base=`basename ${lensing_image} .fits`
export lensing_weight=${lensing_dir}/${lensing_base}.weight.fits
export lensing_flag=${lensing_dir}/${lensing_base}.flag.fits
export LENSDIR=${SUBARUDIR}/${cluster}/LENSING_${filter}_${filter}_aper/${lensing_coadd_type}
export PHOTDIR=${subarudir}/${cluster}/PHOTOMETRY_${filter}_aper

#./adam_do_photometry_final_starcat.sh ${cluster} ${detect_filter} ${lensing_filter} aper PHOTO 2>&1 | tee -a OUT-adam_do_photometry_final_starcat-PHOTO.log
#./get_error_log.sh OUT-adam_do_photometry_final_starcat-PHOTO.log
#bjobs -a
#exit 0;

#./adam_do_photometry_final_starcat.sh ${cluster} ${detect_filter} ${lensing_filter} aper MERGE STARS 2>&1 | tee -a OUT-adam_do_photometry_final_starcat-MERGE_STARS.log
#./get_error_log.sh OUT-adam_do_photometry_final_starcat-MERGE_STARS.log
#exit 0;

#...then cd ~/ldaclensing and run the first few steps of adam_outline.sh, ending with the 2nd run of check_psf_coadd_vis_new.sh
#then run the photometric zps with bigmacs (and get the SDSS catalog)
test -f OUT-adam_do_photometry_final_starcat-SDSS.log && rm -f OUT-adam_do_photometry_final_starcat-SDSS.log
./adam_do_photometry_final_starcat.sh ${cluster} ${detect_filter} ${lensing_filter} aper SDSS 2>&1 | tee -a OUT-adam_do_photometry_final_starcat-SDSS.log
exit_stat=$? #use ${PIPESTATUS[0]} if it's <command> | tee -a OUT-command.log
if [ "${exit_stat}" -gt "0" ]; then
        exit ${exit_stat};
fi
./get_error_log.sh OUT-adam_do_photometry_final_starcat-SDSS.log

./adam_do_photometry_final_starcat.sh ${cluster} ${detect_filter} ${lensing_filter} aper BIGMACSCALIB BIGMACSAPPLY 2>&1 | tee -a OUT-adam_do_photometry_final_starcat-BIGMACSCALIB_BIGMACSAPPLY.log
exit_stat=$? #use ${PIPESTATUS[0]} if it's <command> | tee -a OUT-command.log
if [ "${exit_stat}" -gt "0" ]; then
        exit ${exit_stat};
fi
./get_error_log.sh OUT-adam_do_photometry_final_starcat-BIGMACSCALIB_BIGMACSAPPLY.log

#then run the photo-zs with BPZ
. bpz.ini
./adam_bpz_wrapper.py 2>&1 | tee -a OUT-adam_bpz_wrapper_${cluster}.log
exit_stat=$? #use ${PIPESTATUS[0]} if it's <command> | tee -a OUT-command.log
if [ "${exit_stat}" -gt "0" ]; then
        exit ${exit_stat};
fi

#...then cd ~/ldaclensing and continue on with adam_outline.sh


text:
module RoCell
  class TryTry
  end
end


text:
/**
 * This package containing classes for handling command to MPD protocol server connection and
 * response processing.
 */
package com.anpmech.mpd.connection;


text:
<template>

    <input type="text" placeholder="Search...">

</template>


text:
runify
======

.net continuous unit testing, runify automatically executes tests in your assemblies. runify only monitors for changes to files that contain "Tests" in their names

dependencies
============

1. Powershell >= 2.0
2. nunit
3. Git

installation
============

```
cd your_current_repository
```

add the submodule

```
git submodule add git://github.com/jeanres/runify.git
```

updating
============

```
cd your_current_repository
```

cd into the runify directory

```
cd runify
```

update the submodule

```
git pull
```

configuration
==============

edit the configuration file runfify\runify.config

```
<tests>
  <assembly path="MyProject.UnitTests\bin\debug\MyProject.UnitTests.dll"></assembly>
  <assembly path="MyProject.IntegrationTests\bin\debug\MyProject.IntegrationTests.dll"></assembly>
</tests>
```

point the assembly paths to your unit and integration test paths, these are relative to the root of your project.

running
=======

cd to the root of your repository run .\runify\runify.ps1

Contributing
============

- Fork it
- Create your feature branch (git checkout -b my-new-feature)
- Commit your changes (git commit -am 'Add some feature')
- Push to the branch (git push origin my-new-feature)
- Create new Pull Request


text:
package net.estinet.gFeatures.Feature.gRanks;

import java.io.File;
import java.io.IOException;
import java.util.HashMap;

import org.bukkit.Bukkit;
import org.bukkit.configuration.file.YamlConfiguration;

import net.estinet.gFeatures.Configuration.Config;

public class ConfigHub {

    private static HashMap<String, String> configFields = new HashMap<>();

    static {
        configFields.put("Config", "");
        configFields.put("Config.Cached", "true");
        configFields.put("Config.MySQL", "");
        configFields.put("Config.MySQL.Port", "3306");
        configFields.put("Config.MySQL.Address", "localhost");
        configFields.put("Config.MySQL.TableName", "granks");
        configFields.put("Config.MySQL.Username", "root");
        configFields.put("Config.MySQL.Password", "pass123");
        configFields.put("Config.ClioteSky", "");
        configFields.put("Config.ClioteSky.Enable", "true");
    }

    public static void setupConfig() {
        File f = new File("plugins/gFeatures/gRanks/config.yml");
        Config c = new Config();
        c.createDirectory("plugins/gFeatures/gRanks", "gRanks files created!");
        if (!f.exists()) {
            Bukkit.getLogger().info("[gRanks] Setting up configs....");
            try {
                f.createNewFile();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
        }
        YamlConfiguration yamlFile = YamlConfiguration.loadConfiguration(f);

        for (String key : configFields.keySet()) {
            if (!yamlFile.contains(key)) {
                yamlFile.createSection(key);
                if (!configFields.get(key).equals("")) {
                    yamlFile.set(key, configFields.get(key));
                }
            }
        }

        try {
            yamlFile.save(f);
            Bukkit.getLogger().info("[gRanks] Successfully added config!");
        } catch (IOException e) {
            e.printStackTrace();
        }

        gRanks.address = yamlFile.getString("Config.MySQL.Address");
        gRanks.password = yamlFile.getString("Config.MySQL.Password");
        gRanks.tablename = yamlFile.getString("Config.MySQL.TableName");
        gRanks.port = yamlFile.getString("Config.MySQL.Port");
        gRanks.username = yamlFile.getString("Config.MySQL.Username");
        gRanks.cliotesky = yamlFile.getBoolean("Config.ClioteSky.Enable");
        gRanks.url = "jdbc:mysql://" + gRanks.address + ":" + gRanks.port + "/" + gRanks.tablename + "?autoReconnect=true&useSSL=false";

        Bukkit.getLogger().info("[gRanks] ClioteSky Support: " + gRanks.cliotesky);
    }
}


text:
require 'google/protobuf'

Google::Protobuf::DescriptorPool.generated_pool.build do
  add_file("google/api/system_parameter.proto", :syntax => :proto3) do
    add_message "google.api.SystemParameters" do
      repeated :rules, :message, 1, "google.api.SystemParameterRule"
    end
    add_message "google.api.SystemParameterRule" do
      optional :selector, :string, 1
      repeated :parameters, :message, 2, "google.api.SystemParameter"
    end
    add_message "google.api.SystemParameter" do
      optional :name, :string, 1
      optional :http_header, :string, 2
      optional :url_query_parameter, :string, 3
    end
  end
end

module Google
  module Api
    SystemParameters = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.api.SystemParameters").msgclass
    SystemParameterRule = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.api.SystemParameterRule").msgclass
    SystemParameter = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.api.SystemParameter").msgclass
  end
end


text:
class AddKeyColumnToAuthorizations < ActiveRecord::Migration
  def change
    add_column :authorizations, :key, :text
  end
end


text:
<?php
# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: google/cloud/bigquery/datatransfer/v1/datasource.proto

namespace Google\Cloud\BigQuery\DataTransfer\V1;

if (false) {
    /**
     * This class is deprecated. Use Google\Cloud\BigQuery\DataTransfer\V1\ImportedDataInfo\TableDefinition\CsvOptions instead.
     * @deprecated
     */
    class ImportedDataInfo_TableDefinition_CsvOptions {}
}
class_exists(ImportedDataInfo\TableDefinition\CsvOptions::class);
@trigger_error('Google\Cloud\BigQuery\DataTransfer\V1\ImportedDataInfo_TableDefinition_CsvOptions is deprecated and will be removed in a future release. Use Google\Cloud\BigQuery\DataTransfer\V1\ImportedDataInfo\TableDefinition\CsvOptions instead', E_USER_DEPRECATED);


text:
package io.coala.experimental.kbase;

import io.coala.capability.BasicCapabilityStatus;
import io.coala.capability.Capability;
import io.coala.capability.CapabilityFactory;
import rx.Observable;
import rx.Observable.OnSubscribe;
import rx.Observer;
import rx.Subscriber;

import com.fasterxml.jackson.annotation.JsonIgnore;

/**
 * {@link KnowledgeBaseCapability} provides reasoning capability
 *
 * @date $Date: 2014-06-03 14:26:09 +0200 (Tue, 03 Jun 2014) $
 * @version $Revision: 296 $
 * @author Rick
 *
 * @param  the type of belief
 */
public interface KnowledgeBaseCapability extends Capability<BasicCapabilityStatus>
{

	/**
	 * {@link Factory}
	 *
	 * @version $Revision: 296 $
	 * @author Rick
	 */
	interface Factory extends CapabilityFactory<KnowledgeBaseCapability>
	{
		// empty
	}

	interface Belief
	{
		Belief negate();
	}

	interface Rule<T> // is the belief pattern matcher
	{
		Observable<T> match(Observable<Belief> beliefs);

		Rule<Belief> IDENTITY = new Rule<Belief>()
		{
			@Override
			public Observable<Belief> match(final Observable<Belief> beliefs)
			{
				return beliefs.asObservable();
			}
		};

		Rule<Belief> NEGATION = new Rule<Belief>()
		{
			@Override
			public Observable<Belief> match(final Observable<Belief> beliefs)
			{
				return Observable.create(new OnSubscribe<Belief>()
				{
					@Override
					public void call(final Subscriber<? super Belief> sub)
					{
						beliefs.subscribe(new Observer<Belief>()
						{
							@Override
							public void onCompleted()
							{
								sub.onCompleted();
							}

							@Override
							public void onError(final Throwable e)
							{
								sub.onError(e);
							}

							@Override
							public void onNext(Belief t)
							{
								sub.onNext(t.negate());
							}
						});
					}
				});
			}
		};
	}

	/** @return the rules */
	@JsonIgnore
	Observable<Rule<?>> getRules(boolean currentOnly);

	/** @return the (inferred) beliefs */
	@JsonIgnore
	Observable<Belief> getBeliefs(boolean currentOnly);

	/** @return the newly inferred and added beliefs */
	Observable<Belief> add(Rule<?> rule);

	/** @return the (recursively added) newly inferred beliefs */
	Observable<Belief> add(Belief belief);

	/** @return the resulting matched */
	<T> Observable<T> match(Rule<T> rule, boolean readOnly);

}


text:
package com.gs.collections.test.set;

import java.util.Collection;
import java.util.Iterator;
import java.util.Set;

import com.gs.collections.api.set.MutableSet;
import com.gs.collections.impl.factory.Sets;
import com.gs.collections.impl.test.SerializeTestHelper;
import com.gs.collections.test.CollectionTestCase;
import org.junit.Assert;
import org.junit.Test;

import static com.gs.collections.impl.test.Verify.assertPostSerializedEqualsAndHashCode;
import static com.gs.collections.test.IterableTestCase.assertEquals;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.isOneOf;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;

public interface SetTestCase extends CollectionTestCase
{
    @Override
    <T> Set<T> newWith(T... elements);

    @Override
    @Test
    default void Object_PostSerializedEqualsAndHashCode()
    {
        Iterable<Integer> iterable = this.newWith(3, 2, 1);
        Object deserialized = SerializeTestHelper.serializeDeserialize(iterable);
        Assert.assertNotSame(iterable, deserialized);
    }

    @Override
    @Test
    default void Object_equalsAndHashCode()
    {
        assertPostSerializedEqualsAndHashCode(this.newWith(3, 2, 1));

        assertNotEquals(this.newWith(4, 3, 2, 1), this.newWith(3, 2, 1));
        assertNotEquals(this.newWith(3, 2, 1), this.newWith(4, 3, 2, 1));

        assertNotEquals(this.newWith(2, 1), this.newWith(3, 2, 1));
        assertNotEquals(this.newWith(3, 2, 1), this.newWith(2, 1));

        assertNotEquals(this.newWith(4, 2, 1), this.newWith(3, 2, 1));
        assertNotEquals(this.newWith(3, 2, 1), this.newWith(4, 2, 1));
    }

    @Override
    @Test
    default void Iterable_next()
    {
        Set<Integer> iterable = this.newWith(3, 2, 1);

        MutableSet<Integer> mutableSet = Sets.mutable.with();

        Iterator<Integer> iterator = iterable.iterator();
        while (iterator.hasNext())
        {
            Integer integer = iterator.next();
            assertTrue(mutableSet.add(integer));
        }

        assertEquals(iterable, mutableSet);
        assertFalse(iterator.hasNext());
    }

    @Override
    @Test
    default void Iterable_remove()
    {
        Set<Integer> set = this.newWith(3, 2, 1);
        Iterator<Integer> iterator = set.iterator();
        iterator.next();
        iterator.remove();
        assertThat(set, isOneOf(
                this.newWith(1, 2),
                this.newWith(1, 3),
                this.newWith(2, 3)));
    }

    @Override
    @Test
    default void Collection_add()
    {
        Collection<Integer> collection = this.newWith(1, 2, 3);
        assertFalse(collection.add(3));
    }

    @Override
    @Test
    default void Collection_size()
    {
        assertThat(this.newWith(3, 2, 1), hasSize(3));
        assertThat(this.newWith(), hasSize(0));
    }
}


text:
package org.nd4j.parameterserver.distributed.v2.messages.pairs.params;

import lombok.NoArgsConstructor;
import org.nd4j.parameterserver.distributed.messages.RequestMessage;
import org.nd4j.parameterserver.distributed.v2.messages.impl.base.BaseRequestMessage;

/**
 * This message represents request for ModelParameters
 *
 * @author raver119@gmail.com
 */
@NoArgsConstructor
public class ModelParametersRequest extends BaseRequestMessage {
    //
}


text:
package com.syaku.core.builder;

import java.util.*;
import java.io.*;
import java.lang.reflect.*;
import java.lang.reflect.Method.*;

import org.apache.log4j.*;
import org.apache.commons.configuration.*;
import org.apache.commons.lang.*;
import org.apache.commons.io.*;
import com.ibatis.sqlmap.client.SqlMapClient;

import com.syaku.core.util.*;
import com.syaku.core.common.*;

public class MeiInstall {
  private static Logger log = Logger.getLogger(MeiInstall.class);
  private static Configuration MEI;
  private static PropertiesConfiguration CONFIG;
  private static final SqlMapClient sqlMap = MyBatis.getInstance();

  public static void main(String[] args) throws Exception {
    building();
  }

  public static void building() {
    log_start("MEI Install");

    try {
      MEI = new PropertiesConfiguration("mei.properties");
      String path_config = MEI.getString("mei.path.rec_config");

      CONFIG = new PropertiesConfiguration(path_config);
      CONFIG.clear();
      CONFIG.setAutoSave(true);

      getInstallBefore();
      getInstallAfter();

    } catch (Exception e) {
      log_println(e.toString());
    }

    log_end();
  }

  private static void getInstallBefore() throws Exception {

    try {

      List list = MEI.getList("mei.builder.module");
      int count = list.size();

      for (int i = 0; i < count; i++ ) {
        String module = (String) list.get(i);
        if ( StringUtils.isNotEmpty(module) ) {
          getInstallBeforeBuild(module);
        }
      }

    } catch (Exception e) {
      log_println(StringUtils.join(new String[] {"[MeiInstall.getInstallBefore] Exception : " , e.toString() }));
    }

  }

  private static void getInstallAfter() throws Exception {

    try {

      List list = MEI.getList("mei.builder.module");
      int count = list.size();

      for (int i = 0; i < count; i++ ) {
        String module = (String) list.get(i);
        if ( StringUtils.isNotEmpty(module) ) {
          getInstallAfterBuild(module);
        }
      }

    } catch (Exception e) {
      log_println(StringUtils.join(new String[] {"[MeiInstall.getInstallAfter] Exception : " , e.toString() }));
    }
  }

  private static void getInstallBeforeBuild(String module) throws Exception {
    String conf = "/com/syaku/modules/" + module + "/info.properties";
    Configuration info = new PropertiesConfiguration(conf);

    List list = info.getList("mei.module.install.classes");
    int count = list.size();
    for (int i = 0; i < count; i++ ) {
      String classes = (String) list.get(i);
      if ( StringUtils.isNotEmpty(classes) ) {
        Class cls =Class.forName(classes);
        Object obj = cls.newInstance();
        Method method = obj.getClass().getDeclaredMethod("getInstallBefore");
        method.invoke(obj);
        log_println(StringUtils.join(new String[] {"[MeiInstall getInstallBeforeBuild]" + module + "/" + classes + " Success" }));
      }
    }
  }

  private static void getInstallAfterBuild(String module) throws Exception {
    String conf = "/com/syaku/modules/" + module + "/info.properties";
    Configuration info = new PropertiesConfiguration(conf);

    List list = info.getList("mei.module.install.classes");
    int count = list.size();
    for (int i = 0; i < count; i++ ) {
      String classes = (String) list.get(i);
      if ( StringUtils.isNotEmpty(classes) ) {

        Class cls =Class.forName(classes);
        Object obj = cls.newInstance();
        Method method = obj.getClass().getDeclaredMethod("getInstallAfter");
        method.invoke(obj);

        log_println(StringUtils.join(new String[] {"[MeiInstall getInstallAfterBuild]" + module + "/" + classes + " Success" }));
      }
    }
  }

  public static void log_start(String title) {

    System.out.print(StringUtils.join(new String[] {
      "\r\n" , title , "\r\n" ,
      "=============================================================================\r\n" ,
      "-----------------------------------------------------------------------------\r\n" ,
      "Copyright (c) MEI (Modularize. Extension. Interaction.)\r\n",
      "Powered by Seok Kyun. Choi. 최석균.\r\n" ,
      "http://syaku.tistory.com\r\n" ,
      "GNU Lesser General Public License.\r\n" ,
      "http://www.gnu.org/licenses/lgpl.html\r\n\r\n"
    } ));

  }

  public static void log_print(String content) {
    System.out.print(content);
  }

  public static void log_println(String content) {
    System.out.println(content);
  }

  public static void log_end() {
    System.out.print(StringUtils.join(new String[] {
      "\r\n\r\n" ,
      "-----------------------------------------------------------------------------\r\n" ,
      "=============================================================================\r\n"
    } ));
  }

}


text:
package gen_tasks_logic

import (
	"log"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"go.skia.org/infra/go/cas/rbe"
	"go.skia.org/infra/task_scheduler/go/specs"
)

const (
	// If a parent path contains more than this many immediate child paths (ie.
	// files and dirs which are directly inside it as opposed to indirect
	// descendants), we will include the parent in the isolate file instead of
	// the children. This results in a simpler CasSpec which should need to be
	// changed less often.
	combinePathsThreshold = 3
)

var (
	// Any files in Git which match these patterns will be included, either
	// directly or indirectly via a parent dir.
	pathRegexes = []*regexp.Regexp{
		regexp.MustCompile(`.*\.c$`),
		regexp.MustCompile(`.*\.cc$`),
		regexp.MustCompile(`.*\.cpp$`),
		regexp.MustCompile(`.*\.gn$`),
		regexp.MustCompile(`.*\.gni$`),
		regexp.MustCompile(`.*\.h$`),
		regexp.MustCompile(`.*\.mm$`),
		regexp.MustCompile(`.*\.storyboard$`),
	}

	// These paths are always added to the inclusion list. Note that they may
	// not appear in the CasSpec if they are included indirectly via a parent
	// dir.
	explicitPaths = []string{
		".bazelrc",
		".bazelversion",
		".clang-format",
		".clang-tidy",
		".vpython",
		"BUILD.bazel",
		"WORKSPACE.bazel",
		"bazel",
		"bin/activate-emsdk",
		"bin/fetch-clang-format",
		"bin/fetch-gn",
		"buildtools",
		"example",
		"go_repositories.bzl",
		"infra/bots/assets/android_ndk_darwin/VERSION",
		"infra/bots/assets/android_ndk_linux/VERSION",
		"infra/bots/assets/android_ndk_windows/VERSION",
		"infra/bots/assets/cast_toolchain/VERSION",
		"infra/bots/assets/clang_linux/VERSION",
		"infra/bots/assets/clang_win/VERSION",
		"infra/bots/run_recipe.py",
		"infra/bots/task_drivers",
		"infra/canvaskit",
		"infra/pathkit",
		"package.json",
		"package-lock.json",
		"requirements.txt",
		"resources",
		"third_party/externals",
		"toolchain",
	}
)

// getAllCheckedInPaths returns every path checked in to the repo.
func getAllCheckedInPaths(cfg *Config) []string {
	cmd := exec.Command("git", "ls-files")
	// Use cfg.PathToSkia to get to the Skia checkout, in case this is used by
	// another repo.
	cmd.Dir = filepath.Join(CheckoutRoot(), cfg.PathToSkia)
	output, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatal(err)
	}
	split := strings.Split(string(output), "\n")
	rv := make([]string, 0, len(split))
	for _, line := range split {
		if line != "" {
			rv = append(rv, line)
		}
	}
	return rv
}

// getRelevantPaths returns all paths needed by compile tasks.
func getRelevantPaths(cfg *Config) []string {
	rv := []string{}
	for _, path := range getAllCheckedInPaths(cfg) {
		for _, regex := range pathRegexes {
			if regex.MatchString(path) {
				rv = append(rv, path)
				break
			}
		}
	}
	return append(rv, explicitPaths...)
}

// node is a single node in a directory tree of task inputs.
type node struct {
	children map[string]*node
	name     string
	isLeaf   bool
}

// newNode returns a node instance.
func newNode(name string) *node {
	return &node{
		children: map[string]*node{},
		name:     name,
		isLeaf:   false,
	}
}

// isRoot returns true iff this is the root node.
func (n *node) isRoot() bool {
	return n.name == ""
}

// add the given entry (given as a slice of path components) to the node.
func (n *node) add(entry []string) {
	// Remove the first element if we're not the root node.
	if !n.isRoot() {
		if entry[0] != n.name {
			log.Fatalf("Failed to compute compile CAS inputs; attempting to add entry %v to node %q", entry, n.name)
		}
		entry = entry[1:]

		// If the entry is now empty, this node is a leaf.
		if len(entry) == 0 {
			n.isLeaf = true
			return
		}
	}

	// Add a child node.
	if !n.isLeaf {
		name := entry[0]
		child, ok := n.children[name]
		if !ok {
			child = newNode(name)
			n.children[name] = child
		}
		child.add(entry)

		// If we have more than combinePathsThreshold immediate children,
		// combine them into this node.
		immediateChilden := 0
		for _, child := range n.children {
			if child.isLeaf {
				immediateChilden++
			}
			if !n.isRoot() && immediateChilden >= combinePathsThreshold {
				n.isLeaf = true
				n.children = map[string]*node{}
			}
		}
	}
}

// entries returns the entries represented by this node and its children.
// Will not return children in the following cases:
// - This Node is a leaf, ie. it represents an entry which was explicitly
//   inserted into the Tree, as opposed to only part of a path to other
//   entries.
// - This Node has immediate children exceeding combinePathsThreshold and
//   thus has been upgraded to a leaf node.
func (n *node) entries() [][]string {
	if n.isLeaf {
		return [][]string{{n.name}}
	}
	rv := [][]string{}
	for _, child := range n.children {
		for _, entry := range child.entries() {
			if !n.isRoot() {
				entry = append([]string{n.name}, entry...)
			}
			rv = append(rv, entry)
		}
	}
	return rv
}

// tree represents a directory tree of task inputs.
type tree struct {
	root *node
}

// newTree returns a tree instance.
func newTree() *tree {
	return &tree{
		root: newNode(""),
	}
}

// add the given path to the tree. Entries may be combined as defined by
// combinePathsThreshold.
func (t *tree) add(p string) {
	split := strings.Split(p, "/")
	t.root.add(split)
}

// entries returns all entries in the tree. Entries may be combined as defined
// by combinePathsThreshold.
func (t *tree) entries() []string {
	entries := t.root.entries()
	rv := make([]string, 0, len(entries))
	for _, entry := range entries {
		rv = append(rv, strings.Join(append([]string{"skia"}, entry...), "/"))
	}
	sort.Strings(rv)
	return rv
}

// generateCompileCAS creates the CasSpec used for tasks which build Skia.
func generateCompileCAS(b *specs.TasksCfgBuilder, cfg *Config) {
	t := newTree()
	for _, path := range getRelevantPaths(cfg) {
		t.add(path)
	}
	spec := &specs.CasSpec{
		Root:     "..",
		Paths:    t.entries(),
		Excludes: []string{rbe.ExcludeGitDir},
	}
	b.MustAddCasSpec(CAS_COMPILE, spec)
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<!-- Copyright(C), Xplus Co., Ltd, 2016. All rights reserved. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>com.xplus.server</groupId>
    <artifactId>xplus-server</artifactId>
    <version>0.1-SNAPSHOT</version>
  </parent>
  <artifactId>xplus-api</artifactId>
  <name>XPlus API</name>
  <description>XPlus API.</description>

  <dependencies>
  </dependencies>

  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-javadoc-plugin</artifactId>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>jar</goal>
            </goals>
          </execution>
        </executions>
        <configuration>
          <doctitle><![CDATA[${app_name} - Maven (${project.version})]]></doctitle>
          <quiet>true</quiet>
          <detectJavaApiLink>true</detectJavaApiLink>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-source-plugin</artifactId>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>jar-no-fork</goal>
            </goals>
            <configuration>
            </configuration>
          </execution>
        </executions>
      </plugin>
    </plugins>
  </build>

</project>


text:
/* eslint-disable react/react-in-jsx-scope, react/prefer-stateless-function, react/prop-types  */

class Hello extends React.Component {
  render() {
    return
Hello {this.props.name}
;
  }
}


text:
define(function (require) {
  'use strict';

  return require('./src/TabsCollection');
});


text:
package test.github1461;

import org.testng.TestNG;
import org.testng.annotations.Test;

/**
 * This class reproduces a memory leak problem when running TestNG tests. The same (memory behavior
 * will be shown when running this test as normal TestNG test (e.g. using Intellij)
 *
 * <p>See https://github.com/cbeust/testng/issues/1461
 */
public class MemoryLeakTestNg {

  @Test(timeOut = 10_000)
  public void testMemoryLeak() throws Exception {

    // we run the test programmatically
    runTest();
    // lets wait for garbage collection
    waitForAllObjectsDestructed();
  }

  private static void waitForAllObjectsDestructed() throws InterruptedException {
    while (true) {
      System.out.println("waiting for clean up...");
      // enforce a full gc
      System.gc();

      // check if there are still instances of our test class
      if (MyTestClassWithGlobalReferenceCounterSample.currentNumberOfMyTestObjects == 0) {
        // if we reach this point, all test instances are gone,
        // ... however this never happens ...
        break;
      }
      // let's wait 1 seconds and try again ...
      Thread.sleep(1_000);
      System.out.println(
          "["
              + MyTestClassWithGlobalReferenceCounterSample.currentNumberOfMyTestObjects
              + "] test object(s) still exist.");
    }
  }

  private static void runTest() {

    // create TestNG class
    TestNG testng =
        new TestNG() {
          @Override
          protected void finalize() {
            // it seems that this object will never be finalized !!!
            System.out.println("TestNG finalized");
          }
        };

    // and set a test (which also will never be finalized ... see later)
    testng.setTestClasses(
        new Class[] {
          MyTestClassWithGlobalReferenceCounterSample.class,
        });

    // lets run the test
    testng.run();

    // At this point the test run through and we expect both instances
    // - testng and
    // - the test object of type (MyTest)
    // will be garbage collected when leaving this method
    // ...
  }
}


text:
<?php

namespace Veridu\API;

use Veridu\API\Exception;

class Profile extends AbstractEndpoint
{
    const FILTER_ALL = 0xFFFF;
    const FILTER_STATE = 0x0001;
    const FILTER_USER = 0x0002;
    const FILTER_DETAILS = 0x0004;
    const FILTER_DOCUMENT = 0x0008;
    const FILTER_BADGES = 0x0010;
    const FILTER_CERTIFICATE = 0x020;
    const FILTER_FLAGS = 0x0040;
    const FILTER_FACTS = 0x0080;
    const FILTER_PROVIDER = 0x0100;
    const FILTER_CPR = 0x0200;
    const FILTER_KBA = 0x0400;
    const FILTER_NEMID = 0x0800;
    const FILTER_OTP = 0x1000;
    const FILTER_PERSONAL = 0x2000;

    private function createFilter($map)
    {
        if (($map & self::FILTER_ALL) == self::FILTER_ALL) {
            return 'all';
        }

        $filter = array();

        if (($map & self::FILTER_STATE) == self::FILTER_STATE) {
            $filter[] = 'state';
        }

        if (($map & self::FILTER_USER) == self::FILTER_USER) {
            $filter[] = 'user';
        }

        if (($map & self::FILTER_DETAILS) == self::FILTER_DETAILS) {
            $filter[] = 'details';
        }

        if (($map & self::FILTER_DOCUMENT) == self::FILTER_DOCUMENT) {
            $filter[] = 'document';
        }

        if (($map & self::FILTER_BADGES) == self::FILTER_BADGES) {
            $filter[] = 'badges';
        }

        if (($map & self::FILTER_CERTIFICATE) == self::FILTER_CERTIFICATE) {
            $filter[] = 'certificate';
        }

        if (($map & self::FILTER_FLAGS) == self::FILTER_FLAGS) {
            $filter[] = 'flags';
        }

        if (($map & self::FILTER_FACTS) == self::FILTER_FACTS) {
            $filter[] = 'facts';
        }

        if (($map & self::FILTER_PROVIDER) == self::FILTER_PROVIDER) {
            $filter[] = 'provider';
        }

        if (($map & self::FILTER_CPR) == self::FILTER_CPR) {
            $filter[] = 'cpr';
        }

        if (($map & self::FILTER_KBA) == self::FILTER_KBA) {
            $filter[] = 'kba';
        }

        if (($map & self::FILTER_NEMID) == self::FILTER_NEMID) {
            $filter[] = 'nemid';
        }

        if (($map & self::FILTER_OTP) == self::FILTER_OTP) {
            $filter[] = 'otp';
        }

        if (($map & self::FILTER_PERSONAL) == self::FILTER_PERSONAL) {
            $filter[] = 'personal';
        }

        return $filter;
    }

    /**
    * Retrieves the consolidates profile of a given user
    *
    * @link https://veridu.com/wiki/Profile_Resource#How_to_retrieve_the_consolidated_profile_of_a_given_user
    *
    * @param integer $filter Filter map
    * @param string $username User identification
    *
    * @return array
    */
    public function retrieve($filter = Profile::FILTER_ALL, $username = null)
    {
        $this->validateNotEmptySessionOrFail();
        $username = empty($username) ? $this->storage->getUsername() : $username;
        self::validateUsernameOrFail($username);
        $filters = $this->createFilter($filter);

        if ($filters != 'all') {
            $filters = implode(',', $filters);
            $json = $this->fetch(
                'GET',
                sprintf(
                    'profile/%s',
                    $username
                ),
                array(
                    'filter' => $filters
                )
            );
        } else {
            $json = $this->fetch(
                'GET',
                sprintf(
                    'profile/%s',
                    $username
                )
            );
        }
        return $json;
    }
}


text:
import { connect } from '../../../redux';

import AbstractDialogContainer, {
    abstractMapStateToProps
} from '../AbstractDialogContainer';

/**
 * Implements a DialogContainer responsible for showing all dialogs. We will
 * need a separate container so we can handle multiple dialogs by showing them
 * simultaneously or queueing them.
 *
 * @extends AbstractDialogContainer
 */
class DialogContainer extends AbstractDialogContainer {
    /**
     * Implements React's {@link Component#render()}.
     *
     * @inheritdoc
     * @returns {ReactElement}
     */
    render() {
        return this._renderDialogContent();
    }
}

export default connect(abstractMapStateToProps)(DialogContainer);


text:
package ru.fabrikant.api.multi_integration.easuz.downloadfile;

import javax.xml.bind.annotation.XmlRegistry;

/**
 * This object contains factory methods for each
 * Java content interface and Java element interface
 * generated in the ru.fabrikant.api.multi_integration.easuz.downloadfile package.
 * <p>An ObjectFactory allows you to programatically
 * construct new instances of the Java representation
 * for XML content. The Java representation of XML
 * content can consist of schema derived interfaces
 * and classes representing the binding of schema
 * type definitions, element declarations and model
 * groups.  Factory methods for each of these are
 * provided in this class.
 *
 */
@XmlRegistry
public class ObjectFactory {

    /**
     * Create a new ObjectFactory that can be used to create new instances of schema derived classes for package: ru.fabrikant.api.multi_integration.easuz.downloadfile
     *
     */
    public ObjectFactory() {
    }

    /**
     * Create an instance of {@link GetFile }
     *
     */
    public GetFile createGetFile() {
        return new GetFile();
    }

    /**
     * Create an instance of {@link GetFileResponse }
     *
     */
    public GetFileResponse createGetFileResponse() {
        return new GetFileResponse();
    }

}