text:
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "cdc_rsync/base/cdc_interface.h"

#include <vector>

#include "absl/strings/str_format.h"
#include "blake3.h"
#include "cdc_rsync/base/message_pump.h"
#include "cdc_rsync/protos/messages.pb.h"
#include "common/buffer.h"
#include "common/path.h"
#include "common/status.h"
#include "common/util.h"
#include "fastcdc/fastcdc.h"

#if PLATFORM_LINUX
#include <fcntl.h>
#endif

namespace cdc_ft {
namespace {

// The average chunk size should be as low as possible, but not too low.
// Lower sizes mean better delta-encoding and hence less data uploads.
// However, chunking becomes slower for lower sizes. At 8 KB, a gamelet can
// still process close to 700 MB/sec, which matches hard drive speed.
// Signature data rate is another factor. The gamelet generates signature data
// at a rate of 700 MB/sec / kAvgChunkSize * sizeof(Chunk) = 1.7 MB/sec for 8 KB
// chunks. That means, the client needs at least 16 MBit download bandwidth to
// stream signatures or else this part becomes slower. 4 KB chunks would require
// a 32 MBit connection.
constexpr size_t kAvgChunkSize = 8 * 1024;
constexpr size_t kMinChunkSize = kAvgChunkSize / 2;
constexpr size_t kMaxChunkSize = kAvgChunkSize * 4;

// This number was found by experimentally optimizing chunking throughput.
constexpr size_t kFileIoBufferSize = kMaxChunkSize * 4;

// Limits the size of contiguous patch chunks where data is copied from the
// basis file. Necessary since the server copies chunks in one go and doesn't
// split them up (would be possible, but unnecessarily complicates code).
constexpr size_t kCombinedChunkSizeThreshold = 64 * 1024;

// Number of hashing tasks in flight at a given point of time.
constexpr size_t kMaxNumHashTasks = 64;

#pragma pack(push, 1)
// 16 byte hashes guarantee a sufficiently low chance of hash collisions. For
// 8 byte the chance of a hash collision is actually quite high for large files
// 0.0004% for a 100 GB file and 8 KB chunks.
struct Hash {
  uint64_t low;
  uint64_t high;

  bool operator==(const Hash& other) const {
    return low == other.low && high == other.high;
  }
  bool operator!=(const Hash& other) const { return !(*this == other); }
};
#pragma pack(pop)

static_assert(sizeof(Hash) <= BLAKE3_OUT_LEN, "");

}  // namespace
}  // namespace cdc_ft

namespace std {

template <>
struct hash<cdc_ft::Hash> {
  size_t operator()(const cdc_ft::Hash& hash) const { return hash.low; }
};

}  // namespace std

namespace cdc_ft {
namespace {

// Send a batch of signatures every 8 MB of processed data (~90 packets per
// second at 700 MB/sec processing rate). The size of each signature batch is
// kMinNumChunksPerBatch * sizeof(Chunk), e.g. 20 KB for an avg chunk size of
// 8 KB.
constexpr int kMinSigBatchDataSize = 8 * 1024 * 1024;
constexpr int kMinNumChunksPerBatch = kMinSigBatchDataSize / kAvgChunkSize;

// Send patch commands in batches of at least that size for efficiency.
constexpr int kPatchRequestSizeThreshold = 65536;

// 16 bytes hash, 4 bytes size = 20 bytes.
struct Chunk {
  Hash hash;
  uint32_t size = 0;
  Chunk(const Hash& hash, uint32_t size) : hash(hash), size(size) {}
};

Hash ComputeHash(const void* data, size_t size) {
  assert(data);
  Hash hash;
  blake3_hasher hasher;
  blake3_hasher_init(&hasher);
  blake3_hasher_update(&hasher, data, size);
  blake3_hasher_finalize(&hasher, reinterpret_cast<uint8_t*>(&hash),
                         sizeof(hash));
  return hash;
}

// Task that computes hashes for a single chunk and adds the result to
// AddSignaturesResponse.
class HashTask : public Task {
 public:
  HashTask() {}
  ~HashTask() {}

  HashTask(const HashTask& other) = delete;
  HashTask& operator=(HashTask&) = delete;

  // Sets the data to compute the hash of.
  // Should be called before queuing the task.
  void SetData(const void* data, size_t size) {
    buffer_.reserve(size);
    buffer_.resize(size);
    memcpy(buffer_.data(), data, size);
  }

  // Appends the computed hash to |response|.
  // Should be called once the task is finished.
  void AppendHash(AddSignaturesResponse* response) const {
    response->add_sizes(static_cast<uint32_t>(buffer_.size()));
    std::string* hashes = response->mutable_hashes();
    hashes->append(reinterpret_cast<const char*>(&hash_), sizeof(hash_));
  }

  void ThreadRun(IsCancelledPredicate is_cancelled) override {
    hash_ = ComputeHash(buffer_.data(), buffer_.size());
  }

 private:
  Buffer buffer_;
  struct Hash hash_ = {0};
};

class ServerChunkReceiver {
 public:
  explicit ServerChunkReceiver(MessagePump* message_pump)
      : message_pump_(message_pump) {
    assert(message_pump_);
  }

  // Receives server signature packets and places the data into a map
  // (chunk hash) -> (server-side file offset).
  // If |block| is false, returns immediately if no data is available.
  // If |block| is true, blocks until some data is available.
  // |num_server_bytes_processed| is set to the total size of the chunks
  // received.
  absl::Status Receive(bool block, uint64_t* num_server_bytes_processed) {
    assert(num_server_bytes_processed);
    *num_server_bytes_processed = 0;

    // Already all server chunks received?
    if (all_chunks_received_) {
      return absl::OkStatus();
    }

    // If no data is available, early out (unless blocking is requested).
    if (!block && !message_pump_->CanReceive()) {
      return absl::OkStatus();
    }

    // Receive signatures.
    AddSignaturesResponse response;
    absl::Status status =
        message_pump_->ReceiveMessage(PacketType::kAddSignatures, &response);
    if (!status.ok()) {
      return WrapStatus(status, "Failed to receive AddSignaturesResponse");
    }

    // Validate size of packed hashes, just in case.
    const int num_chunks = response.sizes_size();
    if (response.hashes().size() != num_chunks * sizeof(Hash)) {
      return MakeStatus("Bad hashes size. Expected %u. Actual %u.",
                        num_chunks * sizeof(Hash), response.hashes().size());
    }

    // An empty packet marks the end of the server chunks.
    if (num_chunks == 0) {
      all_chunks_received_ = true;
      return absl::OkStatus();
    }

    // Copy the data over to |server_chunk_offsets|.
    const Hash* hashes =
        reinterpret_cast<const Hash*>(response.hashes().data());
    for (int n = 0; n < num_chunks; ++n) {
      uint32_t size = response.sizes(n);
      chunk_offsets_.insert({hashes[n], curr_offset_});
      curr_offset_ += size;
      *num_server_bytes_processed += size;
    }

    return absl::OkStatus();
  }

  // True if all server chunks have been received.
  bool AllChunksReceived() const { return all_chunks_received_; }

  // Returns a map (server chunk hash) -> (offset of that chunk in server file).
  const std::unordered_map<Hash, uint64_t>& ChunkOffsets() const {
    return chunk_offsets_;
  }

 private:
  MessagePump* message_pump_;

  // Maps server chunk hashes to the file offset in the server file.
  std::unordered_map<Hash, uint64_t> chunk_offsets_;

  // Current server file offset.
  uint64_t curr_offset_ = 0;

  // Whether all server files have been received.
  bool all_chunks_received_ = false;
};

class PatchSender {
  // 1 byte for source, 8 bytes for offset and 4 bytes for size.
  static constexpr size_t kPatchMetadataSize =
      sizeof(uint8_t) + sizeof(uint64_t) + sizeof(uint32_t);

 public:
  PatchSender(FILE* file, MessagePump* message_pump)
      : file_(file), message_pump_(message_pump) {}

  // Tries to send patch data for the next chunk in |client_chunks|. The class
  // keeps an internal counter for the current chunk index. Patch data is not
  // sent if the current client chunk is not found among the server chunks and
  // there are outstanding server chunks. In that case, the method returns
  // with an OK status and should be called later as soon as additional server
  // chunks have been received.
  // |num_client_bytes_processed| is set to the total size of the chunks added.
  absl::Status TryAddChunks(const std::vector<Chunk>& client_chunks,
                            const ServerChunkReceiver& server_chunk_receiver,
                            uint64_t* num_client_bytes_processed) {
    assert(num_client_bytes_processed);
    *num_client_bytes_processed = 0;

    while (curr_chunk_idx_ < client_chunks.size()) {
      const Chunk& chunk = client_chunks[curr_chunk_idx_];
      auto it = server_chunk_receiver.ChunkOffsets().find(chunk.hash);
      bool exists = it != server_chunk_receiver.ChunkOffsets().end();

      // If there are outstanding server chunks and the client hash is not
      // found, do not send the patch data yet. A future server chunk might
      // contain the data.
      if (!exists && !server_chunk_receiver.AllChunksReceived()) {
        return absl::OkStatus();
      }

      absl::Status status = exists ? AddExistingChunk(it->second, chunk.size)
                                   : AddNewChunk(chunk.size);
      if (!status.ok()) {
        return WrapStatus(status, "Failed to add chunk");
      }

      ++curr_chunk_idx_;
      *num_client_bytes_processed += chunk.size;

      // Break loop if all server chunks are received. Otherwise, progress
      // reporting is blocked.
      if (server_chunk_receiver.AllChunksReceived()) {
        break;
      }
    }

    return absl::OkStatus();
  }

  // Sends the remaining patch commands and an EOF marker.
  absl::Status Flush() {
    if (request_size_ > 0) {
      absl::Status status =
          message_pump_->SendMessage(PacketType::kAddPatchCommands, request_);
      if (!status.ok()) {
        return WrapStatus(status, "Failed to send final patch commands");
      }
      total_request_size_ += request_size_;
      request_.Clear();
    }

    // Send an empty patch commands request as EOF marker.
    absl::Status status =
        message_pump_->SendMessage(PacketType::kAddPatchCommands, request_);
    if (!status.ok()) {
      return WrapStatus(status, "Failed to send patch commands EOF marker");
    }

    return absl::OkStatus();
  }

  // Returns the (estimated) total size of all patch data sent.
  uint64_t GetTotalRequestSize() const { return total_request_size_; }

  // Index of the next client chunk.
  size_t CurrChunkIdx() const { return curr_chunk_idx_; }

 private:
  // Adds patch data for a client chunk that has a matching server chunk of
  // given |size| at given |offset| in the server file.
  absl::Status AddExistingChunk(uint64_t offset, uint32_t size) {
    int last_idx = request_.sources_size() - 1;
    if (last_idx >= 0 &&
        request_.sources(last_idx) ==
            AddPatchCommandsRequest::SOURCE_BASIS_FILE &&
        request_.offsets(last_idx) + request_.sizes(last_idx) == offset &&
        request_.sizes(last_idx) < kCombinedChunkSizeThreshold) {
      // Same source and contiguous data -> Append to last entry.
      request_.set_sizes(last_idx, request_.sizes(last_idx) + size);
    } else {
      // Different source or first chunk -> Create new entry.
      request_.add_sources(AddPatchCommandsRequest::SOURCE_BASIS_FILE);
      request_.add_offsets(offset);
      request_.add_sizes(size);
      request_size_ += kPatchMetadataSize;
    }

    return OnChunkAdded(size);
  }

  absl::Status AddNewChunk(uint32_t size) {
    std::string* data = request_.mutable_data();
    int last_idx = request_.sources_size() - 1;
    if (last_idx >= 0 &&
        request_.sources(last_idx) == AddPatchCommandsRequest::SOURCE_DATA) {
      // Same source -> Append to last entry.
      request_.set_sizes(last_idx, request_.sizes(last_idx) + size);
    } else {
      // Different source or first chunk -> Create new entry.
      request_.add_sources(AddPatchCommandsRequest::SOURCE_DATA);
      request_.add_offsets(data->size());
      request_.add_sizes(size);
      request_size_ += kPatchMetadataSize;
    }

    // Read data from client file into |data|. Be sure to restore the previous
    // file offset as the chunker might still be processing the file.
    size_t prev_size = data->size();
    data->resize(prev_size + size);
    int64_t prev_offset = ftell64(file_);
    if (fseek64(file_, file_offset_, SEEK_SET) != 0 ||
        fread(&(*data)[prev_size], 1, size, file_) != size ||
        fseek64(file_, prev_offset, SEEK_SET) != 0) {
      return MakeStatus("Failed to read %u bytes at offset %u", size,
                        file_offset_);
    }
    request_size_ += size;

    return OnChunkAdded(size);
  }

  absl::Status OnChunkAdded(uint32_t size) {
    file_offset_ += size;

    // Send patch commands if there's enough data.
    if (request_size_ > kPatchRequestSizeThreshold) {
      absl::Status status =
          message_pump_->SendMessage(PacketType::kAddPatchCommands, request_);
      if (!status.ok()) {
        return WrapStatus(status, "Failed to send patch commands");
      }
      total_request_size_ += request_size_;
      request_size_ = 0;
      request_.Clear();
    }

    return absl::OkStatus();
  }

  FILE* file_;
  MessagePump* message_pump_;

  AddPatchCommandsRequest request_;
  size_t request_size_ = 0;
  size_t total_request_size_ = 0;
  uint64_t file_offset_ = 0;
  size_t curr_chunk_idx_ = 0;
};

}  // namespace

CdcInterface::CdcInterface(MessagePump* message_pump)
    : message_pump_(message_pump) {}

absl::Status CdcInterface::CreateAndSendSignature(const std::string& filepath) {
  absl::StatusOr<FILE*> file = path::OpenFile(filepath, "rb");
  if (!file.ok()) {
    return file.status();
  }
#if PLATFORM_LINUX
  // Tell the kernel we'll load the file sequentially (improves IO bandwidth).
  posix_fadvise(fileno(*file), 0, 0, POSIX_FADV_SEQUENTIAL);
#endif

  // Use a background thread for computing hashes on the server.
  // Allocate lazily since it is not needed on the client.
  // MUST NOT use more than 1 worker thread since the order of finished tasks
  // would then not necessarily match the pushing order. However, the order is
  // important for computing offsets.
  if (!hash_pool_) hash_pool_ = std::make_unique<Threadpool>(1);

  // |chunk_handler| is called for each CDC chunk. It pushes a hash task to the
  // pool. Tasks are "recycled" from |free_tasks_|, so that buffers don't have
  // to reallocated constantly.
  size_t num_hash_tasks = 0;
  auto chunk_handler = [pool = hash_pool_.get(), &num_hash_tasks,
                        free_tasks = &free_tasks_](const void* data,
                                                   size_t size) {
    ++num_hash_tasks;
    if (free_tasks->empty()) {
      free_tasks->push_back(std::make_unique<HashTask>());
    }
    std::unique_ptr<Task> task = std::move(free_tasks->back());
    free_tasks->pop_back();
    static_cast<HashTask*>(task.get())->SetData(data, size);
    pool->QueueTask(std::move(task));
  };

  fastcdc::Config config(kMinChunkSize, kAvgChunkSize, kMaxChunkSize);
  fastcdc::Chunker chunker(config, chunk_handler);

  AddSignaturesResponse response;
  auto read_handler = [&chunker, &response, pool = hash_pool_.get(),
                       &num_hash_tasks, free_tasks = &free_tasks_,
                       message_pump = message_pump_](const void* data,
                                                     size_t size) {
    chunker.Process(static_cast<const uint8_t*>(data), size);

    // Finish hashing tasks. Block if there are too many of them in flight.
    for (;;) {
      std::unique_ptr<Task> task = num_hash_tasks >= kMaxNumHashTasks
                                       ? pool->GetCompletedTask()
                                       : pool->TryGetCompletedTask();
      if (!task) break;
      num_hash_tasks--;
      static_cast<HashTask*>(task.get())->AppendHash(&response);
      free_tasks->push_back(std::move(task));
    }

    // Send data if we have enough chunks.
    if (response.sizes_size() >= kMinNumChunksPerBatch) {
      absl::Status status =
          message_pump->SendMessage(PacketType::kAddSignatures, response);
      if (!status.ok()) {
        return WrapStatus(status, "Failed to send signatures");
      }
      response.Clear();
    }

    return absl::OkStatus();
  };

  absl::Status status =
      path::StreamReadFileContents(*file, kFileIoBufferSize, read_handler);
  fclose(*file);
  if (!status.ok()) {
    return WrapStatus(status, "Failed to compute signatures");
  }
  chunker.Finalize();

  // Finish hashing tasks.
  hash_pool_->Wait();
  std::unique_ptr<Task> task = hash_pool_->TryGetCompletedTask();
  while (task) {
    static_cast<HashTask*>(task.get())->AppendHash(&response);
    free_tasks_.push_back(std::move(task));
    task = hash_pool_->TryGetCompletedTask();
  }

  // Send the remaining chunks, if any.
  if (response.sizes_size() > 0) {
    status = message_pump_->SendMessage(PacketType::kAddSignatures, response);
    if (!status.ok()) {
      return WrapStatus(status, "Failed to send final signatures");
    }
    response.Clear();
  }

  // Send an empty response as EOF marker.
  status = message_pump_->SendMessage(PacketType::kAddSignatures, response);
  if (!status.ok()) {
    return WrapStatus(status, "Failed to send signatures EOF marker");
  }

  return absl::OkStatus();
}

absl::Status CdcInterface::ReceiveSignatureAndCreateAndSendDiff(
    FILE* file, ReportCdcProgress* progress) {
  //
  // Compute signatures from client |file| and send patches while receiving
  // server signatures.
  //
  std::vector<Chunk> client_chunks;
  ServerChunkReceiver server_chunk_receiver(message_pump_);
  PatchSender patch_sender(file, message_pump_);

  auto chunk_handler = [&client_chunks](const void* data, size_t size) {
    client_chunks.emplace_back(ComputeHash(data, size),
                               static_cast<uint32_t>(size));
  };

  fastcdc::Config config(kMinChunkSize, kAvgChunkSize, kMaxChunkSize);
  fastcdc::Chunker chunker(config, chunk_handler);

  uint64_t file_size = 0;
  auto read_handler = [&chunker, &client_chunks, &server_chunk_receiver,
                       &file_size, progress,
                       &patch_sender](const void* data, size_t size) {
    // Process client chunks for the data read.
    chunker.Process(static_cast<const uint8_t*>(data), size);
    file_size += size;

    const bool all_client_chunks_read = data == nullptr;
    if (all_client_chunks_read) {
      chunker.Finalize();
    }

    do {
      // Receive any server chunks available.
      uint64_t num_server_bytes_processed = 0;
      absl::Status status = server_chunk_receiver.Receive(
          /*block=*/all_client_chunks_read, &num_server_bytes_processed);
      if (!status.ok()) {
        return WrapStatus(status, "Failed to receive server chunks");
      }

      // Try to send patch data.
      uint64_t num_client_bytes_processed = 0;
      status = patch_sender.TryAddChunks(client_chunks, server_chunk_receiver,
                                         &num_client_bytes_processed);
      if (!status.ok()) {
        return WrapStatus(status, "Failed to send patch data");
      }

      progress->ReportSyncProgress(num_client_bytes_processed,
                                   num_server_bytes_processed);
    } while (all_client_chunks_read &&
             (!server_chunk_receiver.AllChunksReceived() ||
              patch_sender.CurrChunkIdx() < client_chunks.size()));

    return absl::OkStatus();
  };

  absl::Status status =
      path::StreamReadFileContents(file, kFileIoBufferSize, read_handler);
  if (!status.ok()) {
    return WrapStatus(status, "Failed to stream file");
  }

  // Should have sent all client chunks by now.
  assert(patch_sender.CurrChunkIdx() == client_chunks.size());

  // Flush remaining patches.
  status = patch_sender.Flush();
  if (!status.ok()) {
    return WrapStatus(status, "Failed to flush patches");
  }

  return absl::OkStatus();
}

absl::Status CdcInterface::ReceiveDiffAndPatch(
    const std::string& basis_filepath, FILE* patched_file,
    bool* is_executable) {
  Buffer buffer;
  *is_executable = false;

  absl::StatusOr<FILE*> basis_file = path::OpenFile(basis_filepath, "rb");
  if (!basis_file.ok()) {
    return basis_file.status();
  }
#if PLATFORM_LINUX
  // Tell the kernel we'll load the file sequentially (improves IO bandwidth).
  // It is not strictly true that the basis file is accessed sequentially, but
  // for larger parts of this file this should be the case.
  posix_fadvise(fileno(*basis_file), 0, 0, POSIX_FADV_SEQUENTIAL);
#endif

  bool first_chunk = true;
  for (;;) {
    AddPatchCommandsRequest request;
    absl::Status status =
        message_pump_->ReceiveMessage(PacketType::kAddPatchCommands, &request);
    if (!status.ok()) {
      fclose(*basis_file);
      return WrapStatus(status, "Failed to receive AddPatchCommandsRequest");
    }

    // All arrays must be of the same size.
    int num_chunks = request.sources_size();
    if (num_chunks != request.offsets_size() ||
        num_chunks != request.sizes_size()) {
      fclose(*basis_file);
      return MakeStatus(
          "Corrupted patch command arrays: Expected sizes %i. Actual %i/%i.",
          num_chunks, request.offsets_size(), request.sizes_size());
    }

    if (num_chunks == 0) {
      // A zero-size request marks the end of patch commands.
      break;
    }

    for (int n = 0; n < num_chunks; ++n) {
      AddPatchCommandsRequest::Source source = request.sources(n);
      uint64_t chunk_offset = request.offsets(n);
      uint32_t chunk_size = request.sizes(n);

      const char* chunk_data = nullptr;
      if (source == AddPatchCommandsRequest::SOURCE_BASIS_FILE) {
        // Copy [chunk_offset, chunk_offset + chunk_size) from |basis_file|.
        buffer.resize(chunk_size);
        if (fseek64(*basis_file, chunk_offset, SEEK_SET) != 0 ||
            fread(buffer.data(), 1, chunk_size, *basis_file) != chunk_size) {
          fclose(*basis_file);
          return MakeStatus(
              "Failed to read %u bytes at offset %u from basis file",
              chunk_size, chunk_offset);
        }
        chunk_data = buffer.data();
      } else {
        // Write [chunk_offset, chunk_offset + chunk_size) from request data.
        assert(source == AddPatchCommandsRequest::SOURCE_DATA);
        if (request.data().size() < chunk_offset + chunk_size) {
          fclose(*basis_file);
          return MakeStatus(
              "Insufficient data in patch commands. Required %u. Actual %u.",
              chunk_offset + chunk_size, request.data().size());
        }
        chunk_data = &request.data()[chunk_offset];
      }

      if (first_chunk && chunk_size > 0) {
        first_chunk = false;
        *is_executable = Util::IsExecutable(chunk_data, chunk_size);
      }
      if (fwrite(chunk_data, 1, chunk_size, patched_file) != chunk_size) {
        fclose(*basis_file);
        return MakeStatus("Failed to write %u bytes to patched file",
                          chunk_size);
      }
    }
  }
  fclose(*basis_file);

  return absl::OkStatus();
}

}  // namespace cdc_ft


text:
/*
 * @file MedianFilter.hpp
 *
 * @brief Implementation of a median filter with C qsort.
 *
 */

#pragma once

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

namespace math
{

template<typename T, int WINDOW = 3>
class MedianFilter
{
public:
	static_assert(WINDOW >= 3, "MedianFilter window size must be >= 3");
	static_assert(WINDOW % 2, "MedianFilter window size must be odd"); // odd

	MedianFilter() = default;

	void insert(const T &sample)
	{
		_head = (_head + 1) % WINDOW;
		_buffer[_head] = sample;
	}

	T median()
	{
		T sorted[WINDOW];
		memcpy(sorted, _buffer, sizeof(_buffer));
		qsort(&sorted, WINDOW, sizeof(T), cmp);

		return sorted[WINDOW / 2];
	}

	T apply(const T &sample)
	{
		insert(sample);
		return median();
	}

private:

	static int cmp(const void *a, const void *b)
	{
		return (*(T *)a >= *(T *)b) ? 1 : -1;
	}

	T _buffer[WINDOW] {};
	uint8_t _head{0};
};

} // namespace math

redpajama_set_name: RedPajamaGithub


text:
@interface AppDelegate (CDVLean)

@end


text:
package com.intellij.java.refactoring;

import com.intellij.JavaTestUtil;
import com.intellij.psi.PsiClass;
import com.intellij.psi.PsiField;
import com.intellij.psi.PsiMethod;
import com.intellij.psi.search.GlobalSearchScope;
import com.intellij.refactoring.BaseRefactoringProcessor;
import com.intellij.refactoring.MultiFileTestCase;
import com.intellij.refactoring.wrapreturnvalue.WrapReturnValueProcessor;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;

/**
 * @author anna
 */
public class WrapReturnValueTest extends MultiFileTestCase {
  @Override
  protected String getTestDataPath() {
    return JavaTestUtil.getJavaTestDataPath();
  }

  @NotNull
  @Override
  protected String getTestRoot() {
    return "/refactoring/wrapReturnValue/";
  }

  public void testSimple() { doTest(false); }
  public void testGenerics() { doTest(false); }
  public void testInconsistentWrapper() { doTest(true, "Existing class does not have getter for selected field"); }
  public void testWrapper() { doTest(true); }
  public void testStrip() { doTest(true); }
  public void testNoConstructor() { doTest(true, "Existing class does not have appropriate constructor"); }
  public void testInnerClass() { doTest(false, null, true); }
  public void testHierarchy() { doTest(false, null, true); }
  public void testAnonymous() { doTest(true, null, false); }
  public void testWrongFieldAssignment() { doTest(true, "Existing class does not have appropriate constructor", false); }
  public void testInferFieldType() { doTest(true, null, false); }
  public void testInferFieldTypeArg() { doTest(true, null, false); }
  public void testWrongFieldType() { doTest(true, "Existing class does not have appropriate constructor", false); }
  public void testStaticMethodInnerClass() { doTest(false, null, true); }
  public void testOpenMethodReference() { doTest(false, null, true); }
  public void testRawReturnType() { doTest(true, "Existing class does not have appropriate constructor"); }
  public void testReturnInsideLambda() { doTest(false, null, true); }
  public void testTypeAnnotations() { doTest(false); }
  public void testWithLambdaInside() { doTest(true); }

  private void doTest(final boolean existing) {
    doTest(existing, null);
  }

  private void doTest(final boolean existing, @NonNls String exceptionMessage) {
    doTest(existing, exceptionMessage, false);
  }

  private void doTest(final boolean existing, String exceptionMessage, final boolean createInnerClass) {
    try {
      doTest((rootDir, rootAfter) -> {
        PsiClass aClass = myJavaFacade.findClass("Test", GlobalSearchScope.projectScope(getProject()));
        assertNotNull("Class Test not found", aClass);
        PsiMethod method = aClass.findMethodsByName("foo", false)[0];
        String wrapperClassName = "Wrapper";
        PsiClass wrapperClass = myJavaFacade.findClass(wrapperClassName, GlobalSearchScope.projectScope(getProject()));
        assertTrue(!existing || wrapperClass != null);
        PsiField delegateField = existing ? wrapperClass.findFieldByName("myField", false) : null;
        new WrapReturnValueProcessor(wrapperClassName, "", null, method, existing, createInnerClass, delegateField).run();
      });
    }
    catch (BaseRefactoringProcessor.ConflictsInTestsException e) {
      if (exceptionMessage != null) {
        assertEquals(exceptionMessage, e.getMessage());
        return;
      }
      throw e;
    }
    if (exceptionMessage != null) {
      fail("Conflict was not found");
    }
  }
}


text:
using System;

namespace Desktop_App
{
#if WINDOWS || LINUX
    /// <summary>
    /// The main class.
    /// </summary>
    public static class Program
    {
        /// <summary>
        /// The main entry point for the application.
        /// </summary>
        [STAThread]
        static void Main()
        {
            using (var game = new GameLogic())
                game.Run();
        }
    }
#endif
}


text:
My Organizations

  <app-loading [loading]="isLoading" [retainContent]="false">

      You have no organizations. Read our
        docs
        to learn more.

    <ng-template #userHasOrganizations>
      <mat-form-field class="w-100">
        <input
          data-cy="filterOrganizationsInput"
          matInput
          [(ngModel)]="filterText"
          (keyup)="onTextChange($event)"
          placeholder="Find organizations..."
        />
      </mat-form-field>

          {{ organization.displayName }}

No matching organizations

    </ng-template>
  </app-loading>


text:
a = 12
b = 15
c = a * b
CAP = "a string"

print c

print CAP

# variable names cannot start with a
# number:
#1a = 12

# variable names can contain numbers,
# just not at the start:
a1 = 12

# underscores are allowed:
_a = 12
a_ = 13

#   a-z  A-Z  0-9  _

# everything is an object
# this "rebinds" the name 'a' to a new
# object:
a = a + 12

# variable names are case sensitive
# meaning that:
x = 12
# is a different variable from
X = 13
print x, X

y = 102
# so this is an error:
print Y


text:
from django.db import models

import amo.models

class DiscoveryModule(amo.models.ModelBase):
    """
    Keeps the application, ordering, and locale metadata for a module.

    The modules are defined statically in modules.py and linked to a database
    row through the module's name.
    """
    app = models.ForeignKey('applications.Application')
    module = models.CharField(max_length=255)
    ordering = models.IntegerField(null=True, blank=True)
    locales = models.CharField(max_length=255, blank=True, default='')

    class Meta:
        db_table = 'discovery_modules'
        unique_together = ('app', 'module')


text:
import configparser

def main():
  parser = configparser.ConfigParser()
  parser.read("Pipfile")

  packages = "packages"
  with open("readthedocs-requirements.txt", "w") as f:
    for key in parser[packages]:
      value = parser[packages][key]

      if key == "sphinxcontrib-httpexample":
        f.write(key + value.replace("\"", "") + "\n")

if __name__ == "__main__":
  main()


text:
namespace Bridge.QUnit
{
    [Ignore]
    [Name("QUnit")]
    public partial class QUnit
    {
        /// <summary>
        /// Configuration for QUnit.
        /// </summary>
        public static Configuration Config;

        /// <summary>
        /// Advanced and extensible data dumping for JavaScript.
        /// </summary>
        public static Dump Dump;
    }
}


text:
package com.vmware.vim25;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlType;

/**
 *
Java class for HostConnectInfoNetworkInfo complex type.
 *
 *
The following schema fragment specifies the expected content contained within this class.
 *
 *

 * <complexType name="HostConnectInfoNetworkInfo">
 *   <complexContent>
 *     <extension base="{urn:vim25}DynamicData">
 *       <sequence>
 *         <element name="summary" type="{urn:vim25}NetworkSummary"/>
 *       </sequence>
 *     </extension>
 *   </complexContent>
 * </complexType>
 *

 *
 *
 */
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(name = "HostConnectInfoNetworkInfo", propOrder = {
    "summary"
})
@XmlSeeAlso({
    HostNewNetworkConnectInfo.class
})
public class HostConnectInfoNetworkInfo
    extends DynamicData
{

    @XmlElement(required = true)
    protected NetworkSummary summary;

    /**
     * Gets the value of the summary property.
     *
     * @return
     *     possible object is
     *     {@link NetworkSummary }
     *
     */
    public NetworkSummary getSummary() {
        return summary;
    }

    /**
     * Sets the value of the summary property.
     *
     * @param value
     *     allowed object is
     *     {@link NetworkSummary }
     *
     */
    public void setSummary(NetworkSummary value) {
        this.summary = value;
    }

}


text:
<meta charset="UTF-8">
<title>TN.<init> - biomedicus-core</title>
<link rel="stylesheet" href="../../../style.css">

biomedicus-core / edu.umn.biomedicus.concepts / TN / <init>

<init>

TN(<span class="identifier" id="edu.umn.biomedicus.concepts.TN$<init>(kotlin.String)/tnString">tnString: String)


text:
<?php
/******************************************************************************
	ADD_VALIDATION.PHP

	Adds a new inversion validation to the database.
	It is executed when adding inversion validations by the "New validation" field from the "Validations and genotyping" section of the current inversion's report webpage.
	After adding the new inversion validation to the database, it executes automatically run_breakseq.sh for its BreakSeq annotation
*******************************************************************************/
?>

<?php
	session_start(); //Inicio la sesión
	include('security_layer.php');

?>

<!DOCTYPE html>

<?php

function addvalidation() {

    $inv_id=$_POST["inv_id"];
    $chr=$_POST["chr"];
    $research_name=$_POST["research_name"];
    $method=$_POST["method"];
    $status=$_POST["status"];
    $checked=$_POST["checked"];
    if (isset($_POST['between_bp1'])) { $betweenbp1="TRUE"; }
    else { $betweenbp1="FALSE"; }
    if (isset($_POST['between_bp2'])) { $betweenbp2="TRUE"; }
    else { $betweenbp2="FALSE"; }
    $validation=$_POST["validation"]; //puede ser: experimental o bioinformatics

    // Si es experimental, recibimos esta informacion:
    $experimental_conditions=$_POST["experimental_conditions"];
    $primers=$_POST["primers"];
    $commentE=$_POST["commentE"];

    //Si es bioinformatics, recibimos esta otra informacion
    $fosmids=$_POST["searchFosmids"];
    $commentB=$_POST["commentB"];
    $results=$_POST["fosmids_results"]; //obligatorio cuando fosmids esta relleno

    //FALTA COMPROBAR ERRORES AL ESCOGER ENTRE LAS DOS OPCIONES
    $bp1s=$_POST["bp1s"]; //numerico
    $bp1e=$_POST["bp1e"]; //numerico
    $bp2s=$_POST["bp2s"]; //numerico
    $bp2e=$_POST["bp2e"]; //numerico
    $description=$_POST["description"];

    //Frequency without genotypes
    $fng_population=$_POST["fng_population"];
    $fng_individuals=$_POST["fng_individuals"];
    $fng_invalleles=$_POST["fng_invalleles"];
    $fng_stdfreq=$_POST["fng_stdfreq"];
    $fng_invfreq=$_POST["fng_invfreq"];
    ///
    $indiv_array = $_POST["indiv"];
	// }

    $message=""; $warning_ind='';

    //Comprobaciones
    if ($checked != 'yes') { $checked='not'; }
    $order_bp='ko';
    if ($bp2e >= $bp2s && $bp2s > $bp1e && $bp1e >= $bp1s) { $order_bp='ok'; }
    if ($bp1s == "" && $bp1e == "" && $bp2s == "" && $bp2e == "" ) { $order_bp='ok'; }

    if ($research_name == "" || $research_name == null )
        { echo "Research Name is not defined
"; }
    elseif ($method == "" || $method == null)
        { echo "Method is not defined
"; }
    elseif ($status == "" || $status == null)
        { echo "Status is not defined
"; }
    /*
    elseif (!preg_match('/PCR/',$method) && !preg_match('/FISH/',$method) && !preg_match('/MLPA/',$method) && $method != "" && $fosmids == "")
	    { echo "Fosmids information from Validation details is not defined
"; }
    elseif (!preg_match('/PCR/',$method) && !preg_match('/FISH/',$method) && !preg_match('/MLPA/',$method) && $method != "" && $results == "")
	    { echo "Results from Validation details is not defined
"; }
    */
    //elseif ($fosmids != "" && $results == "") { echo "Results from Bioinformatic validation is not defined
";}
    elseif (($bp1s != "" || $bp1e!="" || $bp2s!="" || $bp2e!="") && ($bp1s=="" || $bp1e=="" || $bp2s=="" || $bp2e==""))
        { echo "All fields from Add Breakpoints must be defined
"; }
    elseif ($bp1s != "" && !preg_match('/^[0-9]+$/', $bp1s) && !preg_match('/[1-9]/', $bp1s))
        { echo"Breakpoint 1 start is not a number
"; }
    elseif ($bp1e != "" && !preg_match('/^[0-9]*$/', $bp1e) && !preg_match('/[1-9]/', $bp1e))
        { echo"Breakpoint 1 end is not a number
"; }
    elseif ($bp2s != "" && !preg_match('/^[0-9]*$/', $bp2s) && !preg_match('/[1-9]/', $bp2s))
        { echo"Breakpoint 2 start is not a number
"; }
    elseif ($bp2e != "" && !preg_match('/^[0-9]*$/', $bp2e) && !preg_match('/[1-9]/', $bp2e))
        { echo"Breakpoint 2 end is not a number
"; }
    elseif ($order_bp != 'ok')
        {echo "Positions of the breakpoints are not correct
";}
    elseif ($_FILES["individuals"]["name"]!="" && $_FILES["individuals"]["type"]!="text/plain")
        { echo "File type not valid
"; }
    elseif ($_FILES["individuals"]["name"]!="" && $_FILES["individuals"]["error"]>0)
        { echo "Error: ".$_FILES["individuals"]["error"]."
"; }

    else { //Todo es correcto, por lo tantos conectamos a la BBDD:
	    include_once('db_conexion.php');

	    /*
        function add_validation
	    IN `inv_id_val` VARCHAR(255),
	    IN `research_name_val` VARCHAR(255),
	    IN `validation_val` VARCHAR(255),
	    IN `valiadtion_method_val` VARCHAR(255),
	    IN `PCRconditions_val` VARCHAR(255),
	    IN `primer_val` VARCHAR(255),
	    IN `validation_comment_val` VARCHAR(255) ,
	    IN `checked_val` VARCHAR(255))
	    */

	    //mysql_query('CALL miProcedure()');
	    //mysql_query('SELECT miFunction()');

	    //Llamamos a la funcion add_validation:
        $a="SELECT add_validation('$inv_id', '$research_name', '$status', '$method', '$experimental_conditions', '$primers','$commentE','$checked', '".$_SESSION["userID"]."') AS id;";
	    $sql_validation_id = mysql_query("SELECT add_validation('$inv_id', '$research_name', '$status', '$method', '$experimental_conditions', '$primers','$commentE','$checked', '".$_SESSION["userID"]."') AS id;");
	    $r= mysql_fetch_array($sql_validation_id);
	    $validation_id = $r['id'];

	    if ($validation_id != '') {
	        //echo "validacion: ".$r['id']."
";
	        //echo "SELECT add_validation('$inv_id', '$research_name', '$status', '$method', '$experimental_conditions', 			'$primers','$commentE','$checked') AS id
";

            if ($fosmids != '' || $fosmids != NULL) {
			    /*
                add_fosmid_validation`(
			    IN validation_id_val INT,
			    IN inv_id_val INT,
			    IN fosmid_id_val INT,
			    IN research_val  VARCHAR(255),
			    IN result_val  VARCHAR(255),
			    IN comment_val  VARCHAR(255))
			    */

			    //Despues de la funcion add_validation, si hay fosmidos, llamamos al procedure add_fosmid_validation:
			    mysql_query("CALL add_fosmid_validation('$validation_id','$inv_id','$fosmids','$research_name','$results','$commentB', '".$_SESSION["userID"]."');");
	            //echo "CALL add_fosmid_validation('$validation_id','$inv_id','$fosmids','$research_name','$results','$commentB');";
		    }

		    if ($_FILES["individuals"]["name"]!="") {
			    //CUANDO PASAN INFORMACION DE INDIVIDUALS, ABRIMOS EL FICHERO, LO LEEMOS LINEA A LINEA
	            // OJO!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
	            //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
	            // SI HAY ERRORES, SE CREARA LA VALIDACION
	            // HABRIA QUE CHEQUEAR PREVIAMENTE!!!!!!!!!!!!!!!!!

			    $fileind = fopen($_FILES["individuals"]["tmp_name"], "r") or exit("Unable to open file!");
			   	$indiv_array = array();

			    while(!feof($fileind)) {
			    	$line=fgets($fileind);
			    	$indiv_array[] = $line;
			    }
			    fclose($fileind);
		    }

		    if ($indiv_array != "" || $indiv_array != NULL){
			    foreach ($indiv_array as $line) {

				    	if( !strncmp($line, '#', 1) )continue;

	                    //Eliminamos los saltos de linea del final
					    $line=trim($line, "\n");
					    $line=trim($line, "\r");

	                    //PARA CADA LINEA LLAMAMOS ADD_INDIVIDUAL_GENOTIPY
					    /*
	                    add_individual_genotipy`(
					    IN `code_val` VARCHAR(255), -> del fichero
					    IN `inv_name_val` VARCHAR(255), -> hidden --------------------->>>>>>>>>>>> lo cambiamos por inv_id
					    IN `genotype_val` VARCHAR(255), -> fichero
					    IN `valid_id_val` INT(11))-> devuelto por add_validation
					    --- Nuevo fromato del fichero
			            IN code_val VARCHAR(255),              -> del fichero
			            IN gender_val VARCHAR(255),            -> del fichero
			            IN population_val VARCHAR(255),        -> del fichero
			            IN region_val VARCHAR(255),            -> del fichero
			            IN family_val VARCHAR(255),            -> del fichero
			            IN relationship_val VARCHAR(255),      -> del fichero
			            IN `genotype_val` VARCHAR(255),        -> del fichero
			            IN allele_comment_val VARCHAR(255),    -> del fichero
			            IN allele_level_val INT,               -> del fichero
			            IN panel_val VARCHAR(255),             -> del fichero
			            IN other_code_val VARCHAR(255),        -> del fichero
			            IN `inv_id_val` INT(11),               -> hidden
			            IN `valid_id_val` INT(11),             -> hidden
			            IN user_id_val INT                     -> hidden
			            */

	                    $code=""; $geder=""; $population=""; $region=""; $family=""; $panel=""; $relationship=""; $allele_level=""; $genotype="";
					    $allele_comment=""; $other_code="";

	                    if ($line != "") {
						    //PARSEAMOS EL FICHERO
						    //El fichero tiene 'codigo individuo' - 'genotipo' -> puede estar separado por ; \t o espacio
						    $separator="";
						    $semicolon=strpos($line,';');
						    if ($semicolon!==false) {           //; es el separador
							    $separator=';';
						    }
						    else {
							    $tab=strpos($line,"\t");
							    if ($tab!==false) {             //\t es el separador
								    $separator="\t";
							    } else {
								    $space=strpos($line," ");
								    if ($space!==false) {       //El separador es el espacio
									    $separator=' ';
								    }
							    }
						    }
						    if ($separator!="") {
							    $info=explode($separator,$line);
							    $code=$info[0];
			                    $geder = ($info[1] == "" ? NULL : $info[1]);
			                    $population= ($info[2] == "" ? NULL : $info[2]);
			                    $region= ($info[3] == "" ? NULL : $info[3]);
			                    $family= ($info[4] == "" ? NULL : $info[4]);
			                    $relationship= ($info[5] == "" ? NULL : $info[5]);
							        $genotype= $info[6];
			                    $allele_comment= ($info[6] == "" ? NULL : $info[6]);
			                    $allele_level= ($info[7] == "" ? NULL : $info[7]);
			                    $panel= ($info[8] == "" ? NULL : $info[8]);
			                    $other_code= ($info[9] == "" ? NULL : $info[9]);

							    if ($code !="" && ($genotype=="INV/INV" || $genotype=="STD/INV" || $genotype=="STD/STD" || $genotype=="STD" || $genotype=="INV" || $genotype=="NA" || $genotype=="ND")) {
							        mysql_query("CALL add_individual_genotype('$code' , '$geder', '$population', '$region', '$family', '$relationship', '$genotype', '$allele_comment', '$allele_level', '$panel', '$other_code', '$inv_id', '$validation_id', '".$_SESSION["userID"]."');");
	                            	// $a="CALL add_individual_genotype('$code' , '$geder', '$population', '$region', '$family', '$relationship', '$genotype', '$allele_comment', '$allele_level', '$panel', '$other_code', '$inv_id', '$validation_id', '".$_SESSION["userID"]."');";
		                            // echo "CALL add_individual_genotipy('$code','$inv_id','$genotype','$validation_id');";
							    } else {
								    $warning_ind="yes";
							    }
						    } else {
							    $warning_ind='yes';
						    }
					    }
				}
			}

		    if ($bp1s != "" || $bp1e!="" || $bp2s!="" || $bp2e!="") {
			    $changed=mysql_query("SELECT add_BP('$validation_id','$inv_id','$chr','$bp1s','$bp1e','$bp2s','$bp2e','$description', '".$_SESSION["userID"]."') AS chang");
			    $r= mysql_fetch_array($changed);

			    if ($r['chang']=='YES') {
				    $message="Some predictions status have changed
";
			    }

			    #INSERT THE "BETWEEN BREAKPOINTS" INFORMATION TO THE DB
				$sql_between = "UPDATE breakpoints SET bp1_between='$betweenbp1', bp2_between = '$betweenbp2' WHERE inv_id = $inv_id;";
				$result_between = mysql_query($sql_between);
				if (!$result_between) {
					die('Error when passing the checkbox input to the db: ' . mysql_error());
				}

				sleep(1);

			    //BREAKSEQ ANNOTATION
			    exec("kill $(ps aux | grep 'breakseq-1.3' | awk '{print $2}') > /dev/null 2>&1");
			    $gff_file = fopen("/home/invfest/BPSeq/breakseq_annotated_gff/input.gff", "w") or die("Unable to create gff file!");
                //Select inversions
                $query2="SELECT i.name, b.id, b.chr, b.bp1_start, b.bp1_end, b.bp2_start, b.bp2_end, i.status, b.GC FROM inversions i, breakpoints b  WHERE i.id=b.inv_id AND b.GC is null AND b.chr NOT IN ('chrM');";
                print "$sql_bp".'
';
                $result_bp = mysql_query($query2) or die("Query fail: " . mysql_error());
                while($bprow = mysql_fetch_array($result_bp)) {
	                $midpoint_BP1=round(($bprow['bp1_end']-$bprow['bp1_start'])/2+$bprow['bp1_start']);
    	                $midpoint_BP2=round(($bprow['bp2_end']-$bprow['bp2_end'])/2+$bprow['bp2_start']);
    	                $chr=$bprow['chr'];
	                $name=$bprow['name'];
	                $id_bp= $bprow['id'];
	                //$gene_id= $bprow['gene_id'];
	                $inverion_gff_line= "$chr\t$name\tInversion\t$midpoint_BP1\t$midpoint_BP2\t.\t.\t.\t$id_bp\n";

    	            fwrite($gff_file, $inverion_gff_line);
                }

                fclose($gff_file);

                //BreakSeq execution
                //---------------------------------------------------------------------------
                exec("nohup ./run_breakseq.sh > /dev/null 2>&1 &");
		    }

		    // Frequencies without genotypes
		    if (($fng_population != 'null') && ($fng_invfreq != '')) {

			    if ($fng_individuals == '') { $fng_individuals='NULL'; }
			    if ($fng_invalleles == '') { $fng_invalleles='NULL'; }

			    $add_freq_nogenotypes = mysql_query("INSERT INTO population_distribution (inv_id, population_name, validation_id, validation_research_name, individuals, inverted_alleles, inv_frequency) VALUES ($inv_id, '$fng_population', $validation_id, '$research_name', $fng_individuals, $fng_invalleles, $fng_invfreq);");
		    }

		    mysql_close($con);

		    if ($warning_ind != '') {
			    $message.="Some individuals have not been correctly introduced
";
		    }
		    //if ($validation_id) HAY Q FORZAR A QUE SALGA MAL PARA SABER Q DEVUELVE!!
	        //echo "Validation added succesfully
".$message;

		    //Con el siguiente botón se refresca la página principal y por lo tanto, también se cierra el iframe -->
	        //echo "
<input type='submit' value='Close' name='gsubmit'  onclick='parent.location.reload();' />";

		    header('Location: ../report.php?q='.$inv_id.'&o=add_val#validations'.$validation_id);

	    } else { //Fin if que ha funcionado bien el add_validation
		    mysql_close($con);
		    header('Location: ../report.php?q='.$inv_id.'&o=add_valError#validations');
	    }
    }

}

if (!isset ($including)){
	$indiv_array = ""; #deja en blanco indiv_array, se llenará con un archivo individuals
	addvalidation();
}

?>


text:
layout: post
title: Big Team Science for Studying Small People
tags: [MB, CDS, workshop]
---

### [CDS 2022 Pre-Conference](https://cogdevsoc.org/pre-conference-workshops-2/)
#### April 21, 2022 in Madison, Wisconsin, United States

### [VIEW THE WEBINAR RECORDING](https://stanford.zoom.us/rec/share/pQXzvhH5PbZwSLdObTd5JQiRWpU4JLy7oYVRRjIoTt219NKv45QW1NPwIe3TIcJT.u9zqXW9NCpBYsRP1)

## Big Team Science for Studying Small People

The growth in recent years of 'Big Team Science' (BTS) networks in developmental psychology (and beyond) has allowed scientists to come together to conduct projects and create tools that are beyond the scope of any individual lab. Collaborative networks such as the [Open Science Collaboration](https://osf.io/vmrgu/), the [Psychological Science Accelerator (PSA)](https://psysciacc.org), and [ManyBabies (MB)](https://manybabies.github.io) have produced large-scale replications and expansions of landmark findings. Further, groups like [Children Helping Science (CHS)](https://childrenhelpingscience.com), [Databrary](https://nyu.databrary.org), and [Lookit](https://lookit.mit.edu) have created novel tools and infrastructure for collecting and sharing data.

As these developmental BTS organizations have grown and matured, they have unlocked new potential for collaborations and best practices, but they have also uncovered new challenges. These organizations are increasingly working together and incorporating insights gained from BTS collaborations beyond developmental psychology (e.g., from the PSA) to address these challenges and advance their missions. This workshop will bring these groups together as a 'network of networks' to present their findings, discuss successes and challenges, and share best practices and guidance for conducting and participating in large-scale collaborations and/or using the tools developed by these organizations.

This pre-conference will consist of research talks and updates from members of MB, PLAY, Databrary, Lookit, CHS, and PSA as well as practical tutorials on how to get involved in BTS and use the research tools and infrastructure developed by these organizations. Current members of these groups AND anyone interested in getting involved or using these tools will benefit from the workshop and leave with a better understanding of the opportunities and challenges of conducting Big Team Science for Studying Small People.

Cost:

Faculty: $20

Students/Postdocs: Free

Includes lunch and coffee breaks

### People
* [Heidi Baumgartner](https://www-csli.stanford.edu/people/baumgartner-heidi) (ManyBabies & Stanford University)
* [Kiley Hamlin](https://psych.ubc.ca/profile/kiley-hamlin/) (ManyBabies & University of British Columbia)
* [Aaron Chuey](https://sll.stanford.edu/people.html) (Stanford University)
* [Nicholas Coles](https://hai.stanford.edu/people/nicholas-coles) (Psychological Science Accelerator & Stanford University)
* [Moira Dillon](https://as.nyu.edu/content/nyu-as/as/faculty/Moira-Dillon.html) (New York University)
* [Michael C. Frank](https://web.stanford.edu/~mcfrank/) (ManyBabies & Stanford University)
* [Rick Gilmore](https://www.rick-gilmore.com) (PLAY project, Databrary, & Penn State University)
* [Melissa Kline Struhl](http://www.melissaklinestruhl.com) (Lookit, ManyBabies, & MIT)
* [Casey Lew-Williams](https://psych.princeton.edu/person/casey-lew-williams) (ManyBabies & Princeton University)
* [Mark Sheskin](https://www.marksheskin.com) (Children Helping Science, CRADLE, & Minerva University)

> [E-mail the Organizer](mailto:heidib@stanford.edu)

***

### Schedule
#### *(all times in CDT, local Madison time)*
**Morning Session:**

9:00 - 9:15 – Welcome: Introduction/goals of the workshop (Organizers: Heidi Baumgartner, Michael Frank, Melissa Kline Struhl)

9:15 - 10:00 – ManyBabies updates and research results (Casey Lew-Williams, Kiley Hamlin, ManyBabies Governing Board)

10:00 - 10:30 – The PLAY Project (Rick Gilmore)

10:30 - 10:45 – **Coffee break**

10:45 - 11:15 – Insights from the Psychological Science Accelerator (Nicholas Coles)

11:15 - 12:00 – The potential for online data collection to facilitate Big Team Science (BTS), including:
* Comparison of online and lab studies (Aaron Chuey)
* Strengths/limits of online studies (Moira Dillon)

12:00 - 1:00 - **Lunch break**

**Afternoon Session: **

1:00 - 1:30 – Doing asynchronous studies using Lookit (Lookit, Melissa Kline Struhl)

1:30 - 2:00 – Doing synchronous studies on Zoom (Children Helping Science, Mark Sheskin)

2:00 - 3:00 – Breakout discussions: Creating inclusive BTS projects (ManyBabies, Psych Science Accelerator, PLAY project)

3:00 - 3:15 – **Coffee break**

3:15 - 3:45 – Data management for BTS (Rick Gilmore)

3:45 - 4:30 – Closing session: Open discussion to discuss ideas/concerns/future directions & brainstorm to create communal "wish list" for BTS infrastructure and/or policies

***

### Funding
**Big Team Science for Studying Small People** gratefully acknowledges funding from:

* [Social Sciences and Humanities Research Council of Canada](https://www.sshrc-crsh.gc.ca/)
* [Center for the Study of Language and Information](https://www-csli.stanford.edu/), Stanford University
* [ManyBabies](https://manybabies.github.io)
* [PLAY project](https://www.play-project.org) & [Databrary](https://nyu.databrary.org)
* [Princeton Baby Lab](https://babylab.princeton.edu)


text:
from flask import Flask
import os

app = Flask(__name__)                                       #生成Flask框架的对象
app.secret_key=os.urandom(32)                               #设定session_id

import views


text:
#ifndef OSKAR_CONVERT_GEODETIC_SPHERICAL_TO_ECEF_H_
#define OSKAR_CONVERT_GEODETIC_SPHERICAL_TO_ECEF_H_

/**
 * @file oskar_convert_geodetic_spherical_to_ecef.h
 */

#include <oskar_global.h>

#ifdef __cplusplus
extern "C" {
#endif

/**
 * @brief
 * Convert coordinates from geodetic spherical to geocentric Cartesian (ECEF).
 *
 * @details
 * This function converts station positions from the geodetic spherical
 * (longitude, latitude, altitude above ellipsoid) to the geocentric Cartesian
 * (Earth-Centred-Earth-Fixed, or ECEF) system.
 *
 * The output coordinates are with respect to the origin at the centre of the
 * Earth, and have the x-axis pointing towards the meridian of zero longitude,
 * the y-axis to +90 degrees East, and the z-axis to the North Celestial Pole.
 *
 * The function uses WGS84 ellipsoid parameters:
 * Equatorial radius: 6378137 metres.
 * Polar radius: 6356752.314 metres.
 *
 * The algorithm is from "Explanatory Supplement to the Astronomical Almanac"
 * (Seidelmann, 2006), Chapter 4, page 206.
 *
 * @param[in]  num_points  Number of points.
 * @param[in]  lon_rad     Vector of longitudes, in radians.
 * @param[in]  lat_rad     Vector of latitudes, in radians.
 * @param[in]  alt_metres  Vector of altitudes above ellipsoid, in metres.
 * @param[out] x           Vector of ECEF x-positions, in metres.
 * @param[out] y           Vector of ECEF y-positions, in metres.
 * @param[out] z           Vector of ECEF z-positions, in metres.
 */
OSKAR_EXPORT
void oskar_convert_geodetic_spherical_to_ecef(int num_points,
        const double* lon_rad, const double* lat_rad, const double* alt_metres,
        double* x, double* y, double* z);

#ifdef __cplusplus
}
#endif

#endif /* OSKAR_CONVERT_GEODETIC_SPHERICAL_TO_ECEF_H_ */


text:
package com.esri.hadoop.hive.serde;

import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeStats;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonProcessingException;
import org.codehaus.jackson.JsonToken;

import com.esri.core.geometry.GeometryEngine;
import com.esri.core.geometry.MapGeometry;
import com.esri.core.geometry.ogc.OGCGeometry;
import com.esri.hadoop.hive.GeometryUtils;
import com.esri.hadoop.shims.HiveShims;

public class JsonSerde implements SerDe {

	static final Log LOG = LogFactory.getLog(JsonSerde.class.getName());

	static JsonFactory jsonFactory = new JsonFactory();
	static String columnNameConstant = null;
	static String columnTypeConstant = null;

	StructObjectInspector rowOI; // contains the type information for the fields returned

	/* rowBase keeps a base copy of the Writable for each field so they can be reused for
	 * all records. When deserialize is called, row is initially nulled out. Then for each attribute
	 * found in the JSON record the Writable reference is copied from rowBase to row
	 * and set to the appropriate value.  Then row is returned.  This why values don't linger from
	 * previous records.
	 */
	ArrayList<Writable> rowBase;
	ArrayList<Writable> row;

	int numColumns;
	int geometryColumn = -1;
	ArrayList<String> columnNames;
	ArrayList<ObjectInspector> columnOIs;

	boolean [] columnSet;

	@Override
	public void initialize(Configuration arg0, Properties tbl)
			throws SerDeException {

		geometryColumn = -1;

	    // Read the configuration parameters
		String columnNameProperty = tbl.getProperty(HiveShims.serdeConstants.LIST_COLUMNS);
		String columnTypeProperty = tbl.getProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES);

		ArrayList<TypeInfo> typeInfos = TypeInfoUtils
				.getTypeInfosFromTypeString(columnTypeProperty);

		columnNames = new ArrayList<String>();
		columnNames.addAll(Arrays.asList(columnNameProperty.toLowerCase().split(",")));

		numColumns = columnNames.size();

		columnOIs = new ArrayList<ObjectInspector>(numColumns);
		columnSet = new boolean[numColumns];

		for (int c = 0; c < numColumns; c++) {

			TypeInfo colTypeInfo = typeInfos.get(c);

			if (colTypeInfo.getCategory() != Category.PRIMITIVE){
				throw new SerDeException("Only primitive field types are accepted");
			}

			if (colTypeInfo.getTypeName().equals("binary")) {

				if (geometryColumn >= 0) {
					// only one column can be defined as binary for geometries
					throw new SerDeException(
							"Multiple binary columns defined.  Define only one binary column for geometries");
				}

				columnOIs.add(GeometryUtils.geometryTransportObjectInspector);
				geometryColumn = c;
			} else {
				columnOIs.add(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(colTypeInfo));
			}
		}

		// standardStruct uses ArrayList to store the row.
		rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(
				columnNames, columnOIs);

		// constructing the row objects, etc, which will be reused for all rows.
		rowBase = new ArrayList<Writable>(numColumns);
		row = new ArrayList<Writable>(numColumns);

		// set each value in rowBase to the writable that corresponds with its PrimitiveObjectInspector
		for (int c = 0; c < numColumns; c++) {

			PrimitiveObjectInspector poi = (PrimitiveObjectInspector)columnOIs.get(c);
			Writable writable;

			try {
				writable = (Writable)poi.getPrimitiveWritableClass().newInstance();
			} catch (InstantiationException e) {
				throw new SerDeException("Error creating Writable from ObjectInspector", e);
			} catch (IllegalAccessException e) {
				throw new SerDeException("Error creating Writable from ObjectInspector", e);
			}

			rowBase.add(writable);
			row.add(null); // default all values to null
		}
	}

	/**
	 * Copies the Writable at fieldIndex from rowBase to row, then sets the value of the Writable
	 * to the value in parser
	 *
	 * @param fieldIndex column index of field in row
	 * @param parser JsonParser pointing to the attribute
	 * @throws JsonParseException
	 * @throws IOException
	 */
	private void setRowFieldFromParser(int fieldIndex, JsonParser parser) throws JsonParseException, IOException{

		PrimitiveObjectInspector poi = (PrimitiveObjectInspector)this.columnOIs.get(fieldIndex);

		// set the field in the row to the writable from rowBase
		row.set(fieldIndex, rowBase.get(fieldIndex));

		switch (poi.getPrimitiveCategory()){
		case SHORT:
			((ShortWritable)row.get(fieldIndex)).set(parser.getShortValue());
			break;
		case INT:
			((IntWritable)row.get(fieldIndex)).set(parser.getIntValue());
			break;
		case LONG:
			((LongWritable)row.get(fieldIndex)).set(parser.getLongValue());
			break;
		case DOUBLE:
			((DoubleWritable)row.get(fieldIndex)).set(parser.getDoubleValue());
			break;
		case FLOAT:
			((FloatWritable)row.get(fieldIndex)).set(parser.getFloatValue());
			break;
		case BOOLEAN:
			((BooleanWritable)row.get(fieldIndex)).set(parser.getBooleanValue());
			break;
		case STRING:
			((Text)row.get(fieldIndex)).set(parser.getText());
			break;
		default:
			((Text)row.get(fieldIndex)).set(parser.getText());
			break;
		}
	}

	/**
	 * Send to the generator, the value of the Writable, using column type
	 *
	 * @param value The attribute value as a Writable
	 * @param fieldIndex column index of field in row
	 * @param jsonGen JsonGenerator
	 * @throws JsonProcessingException
	 * @throws IOException
	 */
	private void generateJsonFromValue(Writable value, int fieldIndex, JsonGenerator jsonGen)
		throws JsonProcessingException, IOException {

		if (value == null) {
			jsonGen.writeObjectField(columnNames.get(fieldIndex), null);
			return;
		}

		PrimitiveObjectInspector poi = (PrimitiveObjectInspector)this.columnOIs.get(fieldIndex);

		switch (poi.getPrimitiveCategory()) {
		case SHORT:
			jsonGen.writeObjectField(columnNames.get(fieldIndex), ((ShortWritable)value).get());
			break;
		case INT:
			jsonGen.writeObjectField(columnNames.get(fieldIndex), ((IntWritable)value).get());
			break;
		case LONG:
			jsonGen.writeObjectField(columnNames.get(fieldIndex), ((LongWritable)value).get());
			break;
		case DOUBLE:
			jsonGen.writeObjectField(columnNames.get(fieldIndex), ((DoubleWritable)value).get());
			break;
		case FLOAT:
			jsonGen.writeObjectField(columnNames.get(fieldIndex), ((FloatWritable)value).get());
			break;
		case BOOLEAN:
			jsonGen.writeObjectField(columnNames.get(fieldIndex), ((BooleanWritable)value).get());
			break;
		default:	/* especially:	case STRING: */
			jsonGen.writeObjectField(columnNames.get(fieldIndex), value.toString());
			break;
		}
	}

	@Override
	public Object deserialize(Writable json_in) throws SerDeException {
		Text json = (Text) json_in;

		// null out array because we reuse it and we don't want values persisting
		// from the last record
		for (int i=0;i<numColumns;i++)
			row.set(i, null);

		try {
			JsonParser parser = jsonFactory.createJsonParser(json.toString());

			JsonToken token = parser.nextToken();

			while (token != null) {

				if (token == JsonToken.START_OBJECT) {
					if (parser.getCurrentName() == "geometry") {
						if (geometryColumn > -1) {
							// create geometry and insert into geometry field
							MapGeometry mapGeom = GeometryEngine.jsonToGeometry(parser);
							row.set(geometryColumn, mapGeom == null ? null :
									GeometryUtils.geometryToEsriShapeBytesWritable(OGCGeometry.createFromEsriGeometry(mapGeom.getGeometry(),
																													  mapGeom.getSpatialReference())));
						} else {
							// no geometry in select field set, don't even bother parsing
							parser.skipChildren();
						}
					} else if (parser.getCurrentName() == "attributes") {

						token = parser.nextToken();

						while (token != JsonToken.END_OBJECT && token != null) {

							// hive makes all column names in the queries column list lower case
							String name = parser.getText().toLowerCase();

							parser.nextToken();

							// figure out which column index corresponds with the attribute name
							int fieldIndex = columnNames.indexOf(name);

							if (fieldIndex >= 0) {
								setRowFieldFromParser(fieldIndex, parser);
							}

							token = parser.nextToken();
						}

						token = parser.nextToken();
					}
				}

				token = parser.nextToken();
			}

		} catch (JsonParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		return row;
	}

	@Override
	public ObjectInspector getObjectInspector() throws SerDeException {
		return rowOI;
	}

	@Override
	public SerDeStats getSerDeStats() {
		return null;
	}

	@Override
	public Class<? extends Writable> getSerializedClass() {
		return Text.class;
	}

	@Override
	public Writable serialize(Object obj, ObjectInspector oi)
			throws SerDeException {

		StandardStructObjectInspector structOI = (StandardStructObjectInspector) oi;

		// get list of writables, one for each field in the row
		List<Object> fieldWritables = structOI.getStructFieldsDataAsList(obj);

		StringWriter writer = new StringWriter();

		try {
			JsonGenerator jsonGen = jsonFactory.createJsonGenerator(writer);

			jsonGen.writeStartObject();

			// first write attributes
			jsonGen.writeObjectFieldStart("attributes");

			for (int i = 0; i < fieldWritables.size(); i++) {
				if (i == geometryColumn)
					continue; // skip geometry, it comes later

				Writable writable;
				Object tmpObj = fieldWritables.get(i);
				if (tmpObj instanceof LazyPrimitive<?,?>) {  // usually Text, but have seen LazyString
					writable = ((LazyPrimitive<?,?>)(tmpObj)).getWritableObject();
				} else {
					writable = (Writable)tmpObj;
				}

				try {
					generateJsonFromValue(writable, i, jsonGen);
				} catch (JsonProcessingException e) {
					e.printStackTrace();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}

			jsonGen.writeEndObject();

			// if geometry column exists, write it
			if (geometryColumn > -1) {
				BytesWritable bytesWritable = (BytesWritable)fieldWritables.get(geometryColumn);
				if (bytesWritable == null) {
					jsonGen.writeObjectField("geometry", null);
				} else {
					OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(bytesWritable);
					jsonGen.writeRaw(",\"geometry\":" + GeometryEngine.geometryToJson(ogcGeometry.getEsriSpatialReference(),
																				  ogcGeometry.getEsriGeometry()));
				}
			}

			jsonGen.writeEndObject();

			jsonGen.close();

		} catch (JsonGenerationException e) {
			LOG.error("Error generating JSON", e);
			return null;
		} catch (IOException e) {
			LOG.error("Error generating JSON", e);
			return null;
		}

		return new Text(writer.toString());
	}
}


text:
from neutron.db import model_base

def get_metadata():
    return model_base.BASEV2.metadata


text:
from __future__ import absolute_import, division, print_function
__metaclass__ = type

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = r'''
---
module: mso_schema_template_externalepg
short_description: Manage external EPGs in schema templates
description:
- Manage external EPGs in schema templates on Cisco ACI Multi-Site.
author:
- Dag Wieers (@dagwieers)
version_added: '2.8'
options:
  schema:
    description:
    - The name of the schema.
    type: str
    required: yes
  template:
    description:
    - The name of the template.
    type: str
    required: yes
  externalepg:
    description:
    - The name of the external EPG to manage.
    type: str
    aliases: [ name ]
  display_name:
    description:
    - The name as displayed on the MSO web interface.
    type: str
  vrf:
    description:
    - The VRF associated to this ANP.
    type: dict
  state:
    description:
    - Use C(present) or C(absent) for adding or removing.
    - Use C(query) for listing an object or multiple objects.
    type: str
    choices: [ absent, present, query ]
    default: present
extends_documentation_fragment: mso
'''

EXAMPLES = r'''
- name: Add a new external EPG
  mso_schema_template_externalepg:
    host: mso_host
    username: admin
    password: SomeSecretPassword
    schema: Schema 1
    template: Template 1
    externalepg: External EPG 1
    state: present
  delegate_to: localhost

- name: Remove an external EPG
  mso_schema_template_externalepg:
    host: mso_host
    username: admin
    password: SomeSecretPassword
    schema: Schema 1
    template: Template 1
    externalepg: external EPG1
    state: absent
  delegate_to: localhost

- name: Query a specific external EPGs
  mso_schema_template_externalepg:
    host: mso_host
    username: admin
    password: SomeSecretPassword
    schema: Schema 1
    template: Template 1
    externalepg: external EPG1
    state: query
  delegate_to: localhost
  register: query_result

- name: Query all external EPGs
  mso_schema_template_externalepg:
    host: mso_host
    username: admin
    password: SomeSecretPassword
    schema: Schema 1
    template: Template 1
    state: query
  delegate_to: localhost
  register: query_result
'''

RETURN = r'''
'''

from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.network.aci.mso import MSOModule, mso_argument_spec, mso_reference_spec, issubset

def main():
    argument_spec = mso_argument_spec()
    argument_spec.update(
        schema=dict(type='str', required=True),
        template=dict(type='str', required=True),
        externalepg=dict(type='str', aliases=['name']),  # This parameter is not required for querying all objects
        display_name=dict(type='str'),
        vrf=dict(type='dict', options=mso_reference_spec()),
        state=dict(type='str', default='present', choices=['absent', 'present', 'query']),
    )

    module = AnsibleModule(
        argument_spec=argument_spec,
        supports_check_mode=True,
        required_if=[
            ['state', 'absent', ['externalepg']],
            ['state', 'present', ['externalepg', 'vrf']],
        ],
    )

    schema = module.params['schema']
    template = module.params['template']
    externalepg = module.params['externalepg']
    display_name = module.params['display_name']
    vrf = module.params['vrf']
    state = module.params['state']

    mso = MSOModule(module)

    # Get schema_id
    schema_obj = mso.get_obj('schemas', displayName=schema)
    if schema_obj:
        schema_id = schema_obj['id']
    else:
        mso.fail_json(msg="Provided schema '{0}' does not exist".format(schema))

    schema_path = 'schemas/{id}'.format(**schema_obj)

    # Get template
    templates = [t['name'] for t in schema_obj['templates']]
    if template not in templates:
        mso.fail_json(msg="Provided template '{0}' does not exist. Existing templates: {1}".format(template, ', '.join(templates)))
    template_idx = templates.index(template)

    # Get external EPGs
    externalepgs = [e['name'] for e in schema_obj['templates'][template_idx]['externalEpgs']]

    if externalepg is not None and externalepg in externalepgs:
        externalepg_idx = externalepgs.index(externalepg)
        mso.existing = schema_obj['templates'][template_idx]['externalEpgs'][externalepg_idx]

    if state == 'query':
        if externalepg is None:
            mso.existing = schema_obj['templates'][template_idx]['externalEpgs']
        elif not mso.existing:
            mso.fail_json(msg="External EPG '{externalepg}' not found".format(externalepg=externalepg))
        mso.exit_json()

    eepgs_path = '/templates/{0}/externalEpgs'.format(template)
    eepg_path = '/templates/{0}/externalEpgs/{1}'.format(template, externalepg)
    ops = []

    mso.previous = mso.existing
    if state == 'absent':
        if mso.existing:
            mso.sent = mso.existing = {}
            ops.append(dict(op='remove', path=eepg_path))

    elif state == 'present':
        vrf_ref = mso.make_reference(vrf, 'vrf', schema_id, template)

        if display_name is None and not mso.existing:
            display_name = externalepg

        payload = dict(
            name=externalepg,
            displayName=display_name,
            vrfRef=vrf_ref,
            # FIXME
            subnets=[],
            contractRelationships=[],
        )

        mso.sanitize(payload, collate=True)

        if mso.existing:
            ops.append(dict(op='replace', path=eepg_path, value=mso.sent))
        else:
            ops.append(dict(op='add', path=eepgs_path + '/-', value=mso.sent))

        mso.existing = mso.proposed

    if not module.check_mode:
        mso.request(schema_path, method='PATCH', data=ops)

    mso.exit_json()

if __name__ == "__main__":
    main()


text:
import React, { Component } from 'react';
import withStyles from 'isomorphic-style-loader/lib/withStyles';
import s from './Suggestions.scss';
import {connect} from 'react-redux';
import n from '../Navigation/Navigation.scss';
import Link from '../Link';
import biennalequestions from './SampleQuestions/biennale_en.json';
import biennalequestions_fr from './SampleQuestions/biennale_fr.json';
import samplequestions from './SampleQuestions/questions_en.json';

@connect((store) => {
  return {
    language: store.lang.language,
    kb: store.knowledgebase.knowledgebase,
  }
})

class Suggestions extends Component {

  shuffle(array) {
    let counter = array.length;

    // While there are elements in the array
    while (counter > 0) {
      // Pick a random index
      let index = Math.floor(Math.random() * counter);

      // Decrease counter by 1
      counter--;

      // And swap the last element with it
      let temp = array[counter];
      array[counter] = array[index];
      array[index] = temp;
  }

  return array;
}

  render() {

    var list = samplequestions;

    return (

            {this.shuffle(list[0].list).map(function (samplequestion, index) {
              if (samplequestion.lang==this.props.language && this.props.kb.includes(samplequestion.kb)) {
                var link = "/question?query=" + encodeURI(samplequestion.question) + "&lang=" + this.props.language +"&kb=" + this.props.kb.join(',') ;
                return (

                  <Link className={n.darklink}
                        to={link}>{samplequestion.question}</Link>

);
              }
            }.bind(this))}

    );
  }

}

export default withStyles(Suggestions, s, n);


text:
<?php
isset($GLOBALS['_kint_settings']) or $GLOBALS['_kint_settings'] = array();
$_kintSettings = &$GLOBALS['_kint_settings'];

/** @var bool if set to false, kint will become silent, same as Kint::enabled(false) */
$_kintSettings['enabled'] = true;

/**
 * @var callback
 *
 * @param string $file filename where the function was called
 * @param int|null $line the line number in the file (not applicable when used in resource dumps)
 */
$_kintSettings['pathDisplayCallback'] = "kint::_debugPath";

/**
 * @var callback|null
 *
 * @param array $step each step of the backtrace is passed to this callback to clean it up or skip it entirely
 * @return array|null you can return null if you want to bypass outputting this step
 */
$_kintSettings['traceCleanupCallback'] = null;

/** @var int max length of string before it is truncated and displayed separately in full */
$_kintSettings['maxStrLength'] = 60;

/** @var bool whether to add a right colored gutter based on the location of the call to the dump */
$_kintSettings['colorCodeLoops'] = true;

/** @var int max array/object levels to go deep, if zero no limits are applied */
$_kintSettings['maxLevels'] = 5;

/** @var bool whether dumped indexed arrays that are in ideal sequence are displayed */
$_kintSettings['hideSequentialKeys'] = true;

/** @var string the css file to format the output of kint */
$_kintSettings['skin'] = 'kint.css';

/** @var string|null if set, prepends a <head> tag with appropriate meta charset value (issue 7) */
$_kintSettings['charset'] = null;

/** @var bool only set to true if you want to develop kint and know what you're doing */
$_kintSettings['devel'] = false;

unset($_kintSettings);


text:
package com.mocircle.cidrawing.persistence;

import org.json.JSONObject;

import java.util.Map;

public class ExportData {

    private JSONObject metaData;
    private Map<String, byte[]> resources;

    public JSONObject getMetaData() {
        return metaData;
    }

    public void setMetaData(JSONObject metaData) {
        this.metaData = metaData;
    }

    public Map<String, byte[]> getResources() {
        return resources;
    }

    public void setResources(Map<String, byte[]> resources) {
        this.resources = resources;
    }
}


text:
package dark

import (
	"github.com/google/gxui"
	"github.com/google/gxui/math"
	"github.com/google/gxui/mixins"
)

type Button struct {
	mixins.Button
	theme *Theme
}

func CreateButton(theme *Theme) gxui.Button {
	b := &Button{}
	b.Init(b, theme)
	b.theme = theme
	b.SetPadding(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	b.SetMargin(math.Spacing{L: 3, T: 3, R: 3, B: 3})
	b.SetBackgroundBrush(b.theme.ButtonDefaultStyle.Brush)
	b.SetBorderPen(b.theme.ButtonDefaultStyle.Pen)
	b.OnMouseEnter(func(gxui.MouseEvent) { b.Redraw() })
	b.OnMouseExit(func(gxui.MouseEvent) { b.Redraw() })
	b.OnMouseDown(func(gxui.MouseEvent) { b.Redraw() })
	b.OnMouseUp(func(gxui.MouseEvent) { b.Redraw() })
	b.OnGainedFocus(b.Redraw)
	b.OnLostFocus(b.Redraw)
	return b
}

// Button internal overrides
func (b *Button) Paint(c gxui.Canvas) {
	pen := b.Button.BorderPen()
	brush := b.Button.BackgroundBrush()
	fontColor := b.theme.ButtonDefaultStyle.FontColor

	switch {
	case b.IsMouseDown(gxui.MouseButtonLeft) && b.IsMouseOver():
		pen = b.theme.ButtonPressedStyle.Pen
		brush = b.theme.ButtonPressedStyle.Brush
		fontColor = b.theme.ButtonPressedStyle.FontColor
	case b.IsMouseOver():
		pen = b.theme.ButtonOverStyle.Pen
		brush = b.theme.ButtonOverStyle.Brush
		fontColor = b.theme.ButtonOverStyle.FontColor
	}

	if l := b.Label(); l != nil {
		l.SetColor(fontColor)
	}

	r := b.Size().Rect()

	c.DrawRoundedRect(r, 2, 2, 2, 2, gxui.TransparentPen, brush)

	b.PaintChildren.Paint(c)

	c.DrawRoundedRect(r, 2, 2, 2, 2, pen, gxui.TransparentBrush)

	if b.IsChecked() {
		pen = b.theme.HighlightStyle.Pen
		brush = b.theme.HighlightStyle.Brush
		c.DrawRoundedRect(r, 2.0, 2.0, 2.0, 2.0, pen, brush)
	}

	if b.HasFocus() {
		pen = b.theme.FocusedStyle.Pen
		brush = b.theme.FocusedStyle.Brush
		c.DrawRoundedRect(r.ContractI(int(pen.Width)), 3.0, 3.0, 3.0, 3.0, pen, brush)
	}
}


text:
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/**
 * ResponsiveSearchAd.java
 *
 * This file was auto-generated from WSDL
 * by the Apache Axis 1.4 Mar 02, 2009 (07:08:06 PST) WSDL2Java emitter.
 */

package com.google.api.ads.adwords.axis.v201809.cm;

/**
 * Represents a responsive search ad.
 *
 *
Important

 *
 *

 *

 *
Responsive search ads are in beta and may not be available
 * to all AdWords advertisers.

 *
Per the AdWords
 * Terms &
 *             Conditions for features in beta, you may not disclose
 * any non-public information.

 *
Responsive search ads will learn and improve over
 * time, so make sure to regularly monitor the
 *             performance and status of your ads.

 *
Responsive search ads are currently available in English,
 * French, German, Spanish,
 *             Portuguese, Italian, Swedish, Dutch, Russian, Japanese,
 * Polish, Turkish, Danish, and
 *             Norwegian, with other languages on the way.

 *
Assets can be shown in any order, so make sure that
 * they make sense individually or in
 *             combination, and don?t violate our policies or local law.

 *
Even after ads are assembled, they may not serve.

 *
If you have text that should appear in every ad, then
 * you must pin it to either Headline
 *             position 1, Headline position 2, or Description position
 * 1, and also make sure it is less
 *             than 80 characters long.

 *

 *

 *
 *
To increase the likelihood that your ad shows, be sure
 * to provide at least 5 distinct
 *             headlines and 2 distinct descriptions that don't repeat
 * the same or similar phrases. Providing
 *             redundant content will restrict the system's ability to
 * assemble combinations.
 *             See
 * example ads

 *
 *
Help center documentation
 *

 *
About
 * responsive search
 *             ads

 *
Create
 * a responsive search ad
 *             (see examples)

 *

 *

 *
 *
Caution: Responsive search ads
 * do not use {@link #url url}, {@link
 *             #displayUrl displayUrl}, {@link #finalAppUrls finalAppUrls},
 * or {@link #devicePreference
 *             devicePreference}; setting these fields on a responsive
 * search ad will cause an error.

 *             This is enabled for
 * AdX.
 */
public class ResponsiveSearchAd  extends com.google.api.ads.adwords.axis.v201809.cm.Ad  implements java.io.Serializable {
    /* Headlines. When the Ad serves, the headlines will be selected
     * from this list.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdHeadlines".
     * This field is required and should
     * not be {@code null} when it is contained within {@link Operator}s
     * : ADD. */
    private com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] headlines;

    /* Descriptions. When the Ad serves, the descriptions will be
     * selected from this list.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdDescriptions".
     * This field is required and should
     * not be {@code null} when it is contained within {@link Operator}s
     * : ADD. */
    private com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] descriptions;

    /* Text that appears in the ad with the displayed URL.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdPath1".This field can be filtered on. */
    private java.lang.String path1;

    /* In addition to {@link #path1}, more text that appears with
     * the displayed URL.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdPath2".This field can be filtered on. */
    private java.lang.String path2;

    public ResponsiveSearchAd() {
    }

    public ResponsiveSearchAd(
           java.lang.Long id,
           java.lang.String url,
           java.lang.String displayUrl,
           java.lang.String[] finalUrls,
           java.lang.String[] finalMobileUrls,
           com.google.api.ads.adwords.axis.v201809.cm.AppUrl[] finalAppUrls,
           java.lang.String trackingUrlTemplate,
           java.lang.String finalUrlSuffix,
           com.google.api.ads.adwords.axis.v201809.cm.CustomParameters urlCustomParameters,
           com.google.api.ads.adwords.axis.v201809.cm.UrlData[] urlData,
           java.lang.Boolean automated,
           com.google.api.ads.adwords.axis.v201809.cm.AdType type,
           java.lang.Long devicePreference,
           com.google.api.ads.adwords.axis.v201809.cm.SystemManagedEntitySource systemManagedEntitySource,
           java.lang.String adType,
           com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] headlines,
           com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] descriptions,
           java.lang.String path1,
           java.lang.String path2) {
        super(
            id,
            url,
            displayUrl,
            finalUrls,
            finalMobileUrls,
            finalAppUrls,
            trackingUrlTemplate,
            finalUrlSuffix,
            urlCustomParameters,
            urlData,
            automated,
            type,
            devicePreference,
            systemManagedEntitySource,
            adType);
        this.headlines = headlines;
        this.descriptions = descriptions;
        this.path1 = path1;
        this.path2 = path2;
    }

    @Override
    public String toString() {
        return com.google.common.base.MoreObjects.toStringHelper(this.getClass())
            .omitNullValues()
            .add("adType", getAdType())
            .add("automated", getAutomated())
            .add("descriptions", getDescriptions())
            .add("devicePreference", getDevicePreference())
            .add("displayUrl", getDisplayUrl())
            .add("finalAppUrls", getFinalAppUrls())
            .add("finalMobileUrls", getFinalMobileUrls())
            .add("finalUrlSuffix", getFinalUrlSuffix())
            .add("finalUrls", getFinalUrls())
            .add("headlines", getHeadlines())
            .add("id", getId())
            .add("path1", getPath1())
            .add("path2", getPath2())
            .add("systemManagedEntitySource", getSystemManagedEntitySource())
            .add("trackingUrlTemplate", getTrackingUrlTemplate())
            .add("type", getType())
            .add("url", getUrl())
            .add("urlCustomParameters", getUrlCustomParameters())
            .add("urlData", getUrlData())
            .toString();
    }

    /**
     * Gets the headlines value for this ResponsiveSearchAd.
     *
     * @return headlines   * Headlines. When the Ad serves, the headlines will be selected
     * from this list.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdHeadlines".
     * This field is required and should
     * not be {@code null} when it is contained within {@link Operator}s
     * : ADD.
     */
    public com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] getHeadlines() {
        return headlines;
    }

    /**
     * Sets the headlines value for this ResponsiveSearchAd.
     *
     * @param headlines   * Headlines. When the Ad serves, the headlines will be selected
     * from this list.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdHeadlines".
     * This field is required and should
     * not be {@code null} when it is contained within {@link Operator}s
     * : ADD.
     */
    public void setHeadlines(com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] headlines) {
        this.headlines = headlines;
    }

    public com.google.api.ads.adwords.axis.v201809.cm.AssetLink getHeadlines(int i) {
        return this.headlines[i];
    }

    public void setHeadlines(int i, com.google.api.ads.adwords.axis.v201809.cm.AssetLink _value) {
        this.headlines[i] = _value;
    }

    /**
     * Gets the descriptions value for this ResponsiveSearchAd.
     *
     * @return descriptions   * Descriptions. When the Ad serves, the descriptions will be
     * selected from this list.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdDescriptions".
     * This field is required and should
     * not be {@code null} when it is contained within {@link Operator}s
     * : ADD.
     */
    public com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] getDescriptions() {
        return descriptions;
    }

    /**
     * Sets the descriptions value for this ResponsiveSearchAd.
     *
     * @param descriptions   * Descriptions. When the Ad serves, the descriptions will be
     * selected from this list.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdDescriptions".
     * This field is required and should
     * not be {@code null} when it is contained within {@link Operator}s
     * : ADD.
     */
    public void setDescriptions(com.google.api.ads.adwords.axis.v201809.cm.AssetLink[] descriptions) {
        this.descriptions = descriptions;
    }

    public com.google.api.ads.adwords.axis.v201809.cm.AssetLink getDescriptions(int i) {
        return this.descriptions[i];
    }

    public void setDescriptions(int i, com.google.api.ads.adwords.axis.v201809.cm.AssetLink _value) {
        this.descriptions[i] = _value;
    }

    /**
     * Gets the path1 value for this ResponsiveSearchAd.
     *
     * @return path1   * Text that appears in the ad with the displayed URL.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdPath1".This field can be filtered on.
     */
    public java.lang.String getPath1() {
        return path1;
    }

    /**
     * Sets the path1 value for this ResponsiveSearchAd.
     *
     * @param path1   * Text that appears in the ad with the displayed URL.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdPath1".This field can be filtered on.
     */
    public void setPath1(java.lang.String path1) {
        this.path1 = path1;
    }

    /**
     * Gets the path2 value for this ResponsiveSearchAd.
     *
     * @return path2   * In addition to {@link #path1}, more text that appears with
     * the displayed URL.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdPath2".This field can be filtered on.
     */
    public java.lang.String getPath2() {
        return path2;
    }

    /**
     * Sets the path2 value for this ResponsiveSearchAd.
     *
     * @param path2   * In addition to {@link #path1}, more text that appears with
     * the displayed URL.
     *                     This field
     * can be selected using the value "ResponsiveSearchAdPath2".This field can be filtered on.
     */
    public void setPath2(java.lang.String path2) {
        this.path2 = path2;
    }

    private java.lang.Object __equalsCalc = null;
    public synchronized boolean equals(java.lang.Object obj) {
        if (!(obj instanceof ResponsiveSearchAd)) return false;
        ResponsiveSearchAd other = (ResponsiveSearchAd) obj;
        if (obj == null) return false;
        if (this == obj) return true;
        if (__equalsCalc != null) {
            return (__equalsCalc == obj);
        }
        __equalsCalc = obj;
        boolean _equals;
        _equals = super.equals(obj) &&
            ((this.headlines==null && other.getHeadlines()==null) ||
             (this.headlines!=null &&
              java.util.Arrays.equals(this.headlines, other.getHeadlines()))) &&
            ((this.descriptions==null && other.getDescriptions()==null) ||
             (this.descriptions!=null &&
              java.util.Arrays.equals(this.descriptions, other.getDescriptions()))) &&
            ((this.path1==null && other.getPath1()==null) ||
             (this.path1!=null &&
              this.path1.equals(other.getPath1()))) &&
            ((this.path2==null && other.getPath2()==null) ||
             (this.path2!=null &&
              this.path2.equals(other.getPath2())));
        __equalsCalc = null;
        return _equals;
    }

    private boolean __hashCodeCalc = false;
    public synchronized int hashCode() {
        if (__hashCodeCalc) {
            return 0;
        }
        __hashCodeCalc = true;
        int _hashCode = super.hashCode();
        if (getHeadlines() != null) {
            for (int i=0;
                 i<java.lang.reflect.Array.getLength(getHeadlines());
                 i++) {
                java.lang.Object obj = java.lang.reflect.Array.get(getHeadlines(), i);
                if (obj != null &&
                    !obj.getClass().isArray()) {
                    _hashCode += obj.hashCode();
                }
            }
        }
        if (getDescriptions() != null) {
            for (int i=0;
                 i<java.lang.reflect.Array.getLength(getDescriptions());
                 i++) {
                java.lang.Object obj = java.lang.reflect.Array.get(getDescriptions(), i);
                if (obj != null &&
                    !obj.getClass().isArray()) {
                    _hashCode += obj.hashCode();
                }
            }
        }
        if (getPath1() != null) {
            _hashCode += getPath1().hashCode();
        }
        if (getPath2() != null) {
            _hashCode += getPath2().hashCode();
        }
        __hashCodeCalc = false;
        return _hashCode;
    }

    // Type metadata
    private static org.apache.axis.description.TypeDesc typeDesc =
        new org.apache.axis.description.TypeDesc(ResponsiveSearchAd.class, true);

    static {
        typeDesc.setXmlType(new javax.xml.namespace.QName("https://adwords.google.com/api/adwords/cm/v201809", "ResponsiveSearchAd"));
        org.apache.axis.description.ElementDesc elemField = new org.apache.axis.description.ElementDesc();
        elemField.setFieldName("headlines");
        elemField.setXmlName(new javax.xml.namespace.QName("https://adwords.google.com/api/adwords/cm/v201809", "headlines"));
        elemField.setXmlType(new javax.xml.namespace.QName("https://adwords.google.com/api/adwords/cm/v201809", "AssetLink"));
        elemField.setMinOccurs(0);
        elemField.setNillable(false);
        elemField.setMaxOccursUnbounded(true);
        typeDesc.addFieldDesc(elemField);
        elemField = new org.apache.axis.description.ElementDesc();
        elemField.setFieldName("descriptions");
        elemField.setXmlName(new javax.xml.namespace.QName("https://adwords.google.com/api/adwords/cm/v201809", "descriptions"));
        elemField.setXmlType(new javax.xml.namespace.QName("https://adwords.google.com/api/adwords/cm/v201809", "AssetLink"));
        elemField.setMinOccurs(0);
        elemField.setNillable(false);
        elemField.setMaxOccursUnbounded(true);
        typeDesc.addFieldDesc(elemField);
        elemField = new org.apache.axis.description.ElementDesc();
        elemField.setFieldName("path1");
        elemField.setXmlName(new javax.xml.namespace.QName("https://adwords.google.com/api/adwords/cm/v201809", "path1"));
        elemField.setXmlType(new javax.xml.namespace.QName("http://www.w3.org/2001/XMLSchema", "string"));
        elemField.setMinOccurs(0);
        elemField.setNillable(false);
        typeDesc.addFieldDesc(elemField);
        elemField = new org.apache.axis.description.ElementDesc();
        elemField.setFieldName("path2");
        elemField.setXmlName(new javax.xml.namespace.QName("https://adwords.google.com/api/adwords/cm/v201809", "path2"));
        elemField.setXmlType(new javax.xml.namespace.QName("http://www.w3.org/2001/XMLSchema", "string"));
        elemField.setMinOccurs(0);
        elemField.setNillable(false);
        typeDesc.addFieldDesc(elemField);
    }

    /**
     * Return type metadata object
     */
    public static org.apache.axis.description.TypeDesc getTypeDesc() {
        return typeDesc;
    }

    /**
     * Get Custom Serializer
     */
    public static org.apache.axis.encoding.Serializer getSerializer(
           java.lang.String mechType,
           java.lang.Class _javaType,
           javax.xml.namespace.QName _xmlType) {
        return
          new  org.apache.axis.encoding.ser.BeanSerializer(
            _javaType, _xmlType, typeDesc);
    }

    /**
     * Get Custom Deserializer
     */
    public static org.apache.axis.encoding.Deserializer getDeserializer(
           java.lang.String mechType,
           java.lang.Class _javaType,
           javax.xml.namespace.QName _xmlType) {
        return
          new  org.apache.axis.encoding.ser.BeanDeserializer(
            _javaType, _xmlType, typeDesc);
    }

}