text:
package io.druid.indexing.kafka;

import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import com.metamx.emitter.EmittingLogger;
import io.druid.data.input.Committer;
import io.druid.data.input.InputRow;
import io.druid.data.input.impl.InputRowParser;
import io.druid.indexing.appenderator.ActionBasedSegmentAllocator;
import io.druid.indexing.appenderator.ActionBasedUsedSegmentChecker;
import io.druid.indexing.common.TaskStatus;
import io.druid.indexing.common.TaskToolbox;
import io.druid.indexing.common.actions.ResetDataSourceMetadataAction;
import io.druid.indexing.common.actions.SegmentTransactionalInsertAction;
import io.druid.indexing.common.actions.TaskActionClient;
import io.druid.indexing.common.task.AbstractTask;
import io.druid.indexing.common.task.TaskResource;
import io.druid.java.util.common.ISE;
import io.druid.java.util.common.guava.Sequence;
import io.druid.java.util.common.parsers.ParseException;
import io.druid.query.DruidMetrics;
import io.druid.query.NoopQueryRunner;
import io.druid.query.Query;
import io.druid.query.QueryRunner;
import io.druid.segment.indexing.DataSchema;
import io.druid.segment.indexing.RealtimeIOConfig;
import io.druid.segment.realtime.FireDepartment;
import io.druid.segment.realtime.FireDepartmentMetrics;
import io.druid.segment.realtime.RealtimeMetricsMonitor;
import io.druid.segment.realtime.appenderator.Appenderator;
import io.druid.segment.realtime.appenderator.Appenderators;
import io.druid.segment.realtime.appenderator.FiniteAppenderatorDriver;
import io.druid.segment.realtime.appenderator.SegmentIdentifier;
import io.druid.segment.realtime.appenderator.SegmentsAndMetadata;
import io.druid.segment.realtime.appenderator.TransactionalSegmentPublisher;
import io.druid.segment.realtime.firehose.ChatHandler;
import io.druid.segment.realtime.firehose.ChatHandlerProvider;
import io.druid.timeline.DataSegment;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetOutOfRangeException;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.joda.time.DateTime;

import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

public class KafkaIndexTask extends AbstractTask implements ChatHandler
{
  public static final long PAUSE_FOREVER = -1L;

  public enum Status
  {
    NOT_STARTED,
    STARTING,
    READING,
    PAUSED,
    PUBLISHING
  }

  private static final EmittingLogger log = new EmittingLogger(KafkaIndexTask.class);
  private static final String TYPE = "index_kafka";
  private static final Random RANDOM = new Random();
  private static final long POLL_TIMEOUT = 100;
  private static final long POLL_RETRY_MS = 30000;
  private static final long LOCK_ACQUIRE_TIMEOUT_SECONDS = 15;
  private static final String METADATA_NEXT_PARTITIONS = "nextPartitions";

  private final DataSchema dataSchema;
  private final InputRowParser<ByteBuffer> parser;
  private final KafkaTuningConfig tuningConfig;
  private final KafkaIOConfig ioConfig;
  private final Optional<ChatHandlerProvider> chatHandlerProvider;

  private final Map<Integer, Long> endOffsets = new ConcurrentHashMap<>();
  private final Map<Integer, Long> nextOffsets = new ConcurrentHashMap<>();

  private ObjectMapper mapper;

  private volatile Appenderator appenderator = null;
  private volatile FireDepartmentMetrics fireDepartmentMetrics = null;
  private volatile DateTime startTime;
  private volatile Status status = Status.NOT_STARTED; // this is only ever set by the task runner thread (runThread)
  private volatile Thread runThread = null;
  private volatile boolean stopRequested = false;
  private volatile boolean publishOnStop = false;

  // The pause lock and associated conditions are to support coordination between the Jetty threads and the main
  // ingestion loop. The goal is to provide callers of the API a guarantee that if pause() returns successfully
  // the ingestion loop has been stopped at the returned offsets and will not ingest any more data until resumed. The
  // fields are used as follows (every step requires acquiring [pauseLock]):
  //   Pausing:
  //   - In pause(), [pauseRequested] is set to true and then execution waits for [status] to change to PAUSED, with the
  //     condition checked when [hasPaused] is signalled.
  //   - In possiblyPause() called from the main loop, if [pauseRequested] is true, [status] is set to PAUSED,
  //     [hasPaused] is signalled, and execution pauses until [pauseRequested] becomes false, either by being set or by
  //     the [pauseMillis] timeout elapsing. [pauseRequested] is checked when [shouldResume] is signalled.
  //   Resuming:
  //   - In resume(), [pauseRequested] is set to false, [shouldResume] is signalled, and execution waits for [status] to
  //     change to something other than PAUSED, with the condition checked when [shouldResume] is signalled.
  //   - In possiblyPause(), when [shouldResume] is signalled, if [pauseRequested] has become false the pause loop ends,
  //     [status] is changed to STARTING and [shouldResume] is signalled.

  private final Lock pauseLock = new ReentrantLock();
  private final Condition hasPaused = pauseLock.newCondition();
  private final Condition shouldResume = pauseLock.newCondition();

  // [pollRetryLock] and [isAwaitingRetry] is used when the Kafka consumer returns an OffsetOutOfRangeException and we
  // pause polling from Kafka for POLL_RETRY_MS before trying again. This allows us to signal the sleeping thread and
  // resume the main run loop in the case of a pause or stop request from a Jetty thread.
  private final Lock pollRetryLock = new ReentrantLock();
  private final Condition isAwaitingRetry = pollRetryLock.newCondition();

  // [statusLock] is used to synchronize the Jetty thread calling stopGracefully() with the main run thread. It prevents
  // the main run thread from switching into a publishing state while the stopGracefully() thread thinks it's still in
  // a pre-publishing state. This is important because stopGracefully() will try to use the [stopRequested] flag to stop
  // the main thread where possible, but this flag is not honored once publishing has begun so in this case we must
  // interrupt the thread. The lock ensures that if the run thread is about to transition into publishing state, it
  // blocks until after stopGracefully() has set [stopRequested] and then does a final check on [stopRequested] before
  // transitioning to publishing state.
  private final Object statusLock = new Object();

  private volatile boolean pauseRequested = false;
  private volatile long pauseMillis = 0;

  @JsonCreator
  public KafkaIndexTask(
      @JsonProperty("id") String id,
      @JsonProperty("resource") TaskResource taskResource,
      @JsonProperty("dataSchema") DataSchema dataSchema,
      @JsonProperty("tuningConfig") KafkaTuningConfig tuningConfig,
      @JsonProperty("ioConfig") KafkaIOConfig ioConfig,
      @JsonProperty("context") Map<String, Object> context,
      @JacksonInject ChatHandlerProvider chatHandlerProvider
  )
  {
    super(
        id == null ? makeTaskId(dataSchema.getDataSource(), RANDOM.nextInt()) : id,
        String.format("%s_%s", TYPE, dataSchema.getDataSource()),
        taskResource,
        dataSchema.getDataSource(),
        context
    );

    this.dataSchema = Preconditions.checkNotNull(dataSchema, "dataSchema");
    this.parser = Preconditions.checkNotNull((InputRowParser<ByteBuffer>) dataSchema.getParser(), "parser");
    this.tuningConfig = Preconditions.checkNotNull(tuningConfig, "tuningConfig");
    this.ioConfig = Preconditions.checkNotNull(ioConfig, "ioConfig");
    this.chatHandlerProvider = Optional.fromNullable(chatHandlerProvider);

    this.endOffsets.putAll(ioConfig.getEndPartitions().getPartitionOffsetMap());
  }

  private static String makeTaskId(String dataSource, int randomBits)
  {
    final StringBuilder suffix = new StringBuilder(8);
    for (int i = 0; i < Ints.BYTES * 2; ++i) {
      suffix.append((char) ('a' + ((randomBits >>> (i * 4)) & 0x0F)));
    }
    return Joiner.on("_").join(TYPE, dataSource, suffix);
  }

  @Override
  public String getType()
  {
    return TYPE;
  }

  @Override
  public boolean isReady(TaskActionClient taskActionClient) throws Exception
  {
    return true;
  }

  @JsonProperty
  public DataSchema getDataSchema()
  {
    return dataSchema;
  }

  @JsonProperty
  public KafkaTuningConfig getTuningConfig()
  {
    return tuningConfig;
  }

  @JsonProperty("ioConfig")
  public KafkaIOConfig getIOConfig()
  {
    return ioConfig;
  }

  @Override
  public TaskStatus run(final TaskToolbox toolbox) throws Exception
  {
    log.info("Starting up!");
    startTime = DateTime.now();
    mapper = toolbox.getObjectMapper();
    status = Status.STARTING;

    if (chatHandlerProvider.isPresent()) {
      log.info("Found chat handler of class[%s]", chatHandlerProvider.get().getClass().getName());
      chatHandlerProvider.get().register(getId(), this, false);
    } else {
      log.warn("No chat handler detected");
    }

    runThread = Thread.currentThread();

    // Set up FireDepartmentMetrics
    final FireDepartment fireDepartmentForMetrics = new FireDepartment(
        dataSchema,
        new RealtimeIOConfig(null, null, null),
        null
    );
    fireDepartmentMetrics = fireDepartmentForMetrics.getMetrics();
    toolbox.getMonitorScheduler().addMonitor(
        new RealtimeMetricsMonitor(
            ImmutableList.of(fireDepartmentForMetrics),
            ImmutableMap.of(DruidMetrics.TASK_ID, new String[]{getId()})
        )
    );

    try (
        final Appenderator appenderator0 = newAppenderator(fireDepartmentMetrics, toolbox);
        final FiniteAppenderatorDriver driver = newDriver(appenderator0, toolbox, fireDepartmentMetrics);
        final KafkaConsumer<byte[], byte[]> consumer = newConsumer()
    ) {
      appenderator = appenderator0;

      final String topic = ioConfig.getStartPartitions().getTopic();

      // Start up, set up initial offsets.
      final Object restoredMetadata = driver.startJob();
      if (restoredMetadata == null) {
        nextOffsets.putAll(ioConfig.getStartPartitions().getPartitionOffsetMap());
      } else {
        final Map<String, Object> restoredMetadataMap = (Map) restoredMetadata;
        final KafkaPartitions restoredNextPartitions = toolbox.getObjectMapper().convertValue(
            restoredMetadataMap.get(METADATA_NEXT_PARTITIONS),
            KafkaPartitions.class
        );
        nextOffsets.putAll(restoredNextPartitions.getPartitionOffsetMap());

        // Sanity checks.
        if (!restoredNextPartitions.getTopic().equals(ioConfig.getStartPartitions().getTopic())) {
          throw new ISE(
              "WTF?! Restored topic[%s] but expected topic[%s]",
              restoredNextPartitions.getTopic(),
              ioConfig.getStartPartitions().getTopic()
          );
        }

        if (!nextOffsets.keySet().equals(ioConfig.getStartPartitions().getPartitionOffsetMap().keySet())) {
          throw new ISE(
              "WTF?! Restored partitions[%s] but expected partitions[%s]",
              nextOffsets.keySet(),
              ioConfig.getStartPartitions().getPartitionOffsetMap().keySet()
          );
        }
      }

      // Set up sequenceNames.
      final Map<Integer, String> sequenceNames = Maps.newHashMap();
      for (Integer partitionNum : nextOffsets.keySet()) {
        sequenceNames.put(partitionNum, String.format("%s_%s", ioConfig.getBaseSequenceName(), partitionNum));
      }

      // Set up committer.
      final Supplier<Committer> committerSupplier = new Supplier<Committer>()
      {
        @Override
        public Committer get()
        {
          final Map<Integer, Long> snapshot = ImmutableMap.copyOf(nextOffsets);

          return new Committer()
          {
            @Override
            public Object getMetadata()
            {
              return ImmutableMap.of(
                  METADATA_NEXT_PARTITIONS, new KafkaPartitions(
                      ioConfig.getStartPartitions().getTopic(),
                      snapshot
                  )
              );
            }

            @Override
            public void run()
            {
              // Do nothing.
            }
          };
        }
      };

      Set<Integer> assignment = assignPartitionsAndSeekToNext(consumer, topic);

      // Main loop.
      // Could eventually support leader/follower mode (for keeping replicas more in sync)
      boolean stillReading = !assignment.isEmpty();
      status = Status.READING;
      try {
        while (stillReading) {
          if (possiblyPause(assignment)) {
            // The partition assignments may have changed while paused by a call to setEndOffsets() so reassign
            // partitions upon resuming. This is safe even if the end offsets have not been modified.
            assignment = assignPartitionsAndSeekToNext(consumer, topic);

            if (assignment.isEmpty()) {
              log.info("All partitions have been fully read");
              publishOnStop = true;
              stopRequested = true;
            }
          }

          if (stopRequested) {
            break;
          }

          // The retrying business is because the KafkaConsumer throws OffsetOutOfRangeException if the seeked-to
          // offset is not present in the topic-partition. This can happen if we're asking a task to read from data
          // that has not been written yet (which is totally legitimate). So let's wait for it to show up.
          ConsumerRecords<byte[], byte[]> records = ConsumerRecords.empty();
          try {
            records = consumer.poll(POLL_TIMEOUT);
          }
          catch (OffsetOutOfRangeException e) {
            log.warn("OffsetOutOfRangeException with message [%s]", e.getMessage());
            possiblyResetOffsetsOrWait(e.offsetOutOfRangePartitions(), consumer, toolbox);
            stillReading = ioConfig.isPauseAfterRead() || !assignment.isEmpty();
          }

          for (ConsumerRecord<byte[], byte[]> record : records) {
            if (log.isTraceEnabled()) {
              log.trace(
                  "Got topic[%s] partition[%d] offset[%,d].",
                  record.topic(),
                  record.partition(),
                  record.offset()
              );
            }

            if (record.offset() < endOffsets.get(record.partition())) {
              if (record.offset() != nextOffsets.get(record.partition())) {
                throw new ISE(
                    "WTF?! Got offset[%,d] after offset[%,d] in partition[%d].",
                    record.offset(),
                    nextOffsets.get(record.partition()),
                    record.partition()
                );
              }

              try {
                final byte[] valueBytes = record.value();
                if (valueBytes == null) {
                  throw new ParseException("null value");
                }

                final InputRow row = Preconditions.checkNotNull(parser.parse(ByteBuffer.wrap(valueBytes)), "row");

                if (!ioConfig.getMinimumMessageTime().isPresent() ||
                    !ioConfig.getMinimumMessageTime().get().isAfter(row.getTimestamp())) {

                  final SegmentIdentifier identifier = driver.add(
                      row,
                      sequenceNames.get(record.partition()),
                      committerSupplier
                  );

                  if (identifier == null) {
                    // Failure to allocate segment puts determinism at risk, bail out to be safe.
                    // May want configurable behavior here at some point.
                    // If we allow continuing, then consider blacklisting the interval for a while to avoid constant checks.
                    throw new ISE("Could not allocate segment for row with timestamp[%s]", row.getTimestamp());
                  }

                  fireDepartmentMetrics.incrementProcessed();
                } else {
                  fireDepartmentMetrics.incrementThrownAway();
                }
              }
              catch (ParseException e) {
                if (tuningConfig.isReportParseExceptions()) {
                  throw e;
                } else {
                  log.debug(
                      e,
                      "Dropping unparseable row from partition[%d] offset[%,d].",
                      record.partition(),
                      record.offset()
                  );

                  fireDepartmentMetrics.incrementUnparseable();
                }
              }

              nextOffsets.put(record.partition(), record.offset() + 1);
            }

            if (nextOffsets.get(record.partition()).equals(endOffsets.get(record.partition()))
                && assignment.remove(record.partition())) {
              log.info("Finished reading topic[%s], partition[%,d].", record.topic(), record.partition());
              assignPartitions(consumer, topic, assignment);
              stillReading = ioConfig.isPauseAfterRead() || !assignment.isEmpty();
            }
          }
        }
      }
      finally {
        driver.persist(committerSupplier.get()); // persist pending data
      }

      synchronized (statusLock) {
        if (stopRequested && !publishOnStop) {
          throw new InterruptedException("Stopping without publishing");
        }

        status = Status.PUBLISHING;
      }

      final TransactionalSegmentPublisher publisher = new TransactionalSegmentPublisher()
      {
        @Override
        public boolean publishSegments(Set<DataSegment> segments, Object commitMetadata) throws IOException
        {
          final KafkaPartitions finalPartitions = toolbox.getObjectMapper().convertValue(
              ((Map) commitMetadata).get(METADATA_NEXT_PARTITIONS),
              KafkaPartitions.class
          );

          // Sanity check, we should only be publishing things that match our desired end state.
          if (!endOffsets.equals(finalPartitions.getPartitionOffsetMap())) {
            throw new ISE("WTF?! Driver attempted to publish invalid metadata[%s].", commitMetadata);
          }

          final SegmentTransactionalInsertAction action;

          if (ioConfig.isUseTransaction()) {
            action = new SegmentTransactionalInsertAction(
                segments,
                new KafkaDataSourceMetadata(ioConfig.getStartPartitions()),
                new KafkaDataSourceMetadata(finalPartitions)
            );
          } else {
            action = new SegmentTransactionalInsertAction(segments, null, null);
          }

          log.info("Publishing with isTransaction[%s].", ioConfig.isUseTransaction());

          return toolbox.getTaskActionClient().submit(action).isSuccess();
        }
      };

      final SegmentsAndMetadata published = driver.finish(publisher, committerSupplier.get());
      if (published == null) {
        throw new ISE("Transaction failure publishing segments, aborting");
      } else {
        log.info(
            "Published segments[%s] with metadata[%s].",
            Joiner.on(", ").join(
                Iterables.transform(
                    published.getSegments(),
                    new Function<DataSegment, String>()
                    {
                      @Override
                      public String apply(DataSegment input)
                      {
                        return input.getIdentifier();
                      }
                    }
                )
            ),
            published.getCommitMetadata()
        );
      }
    }
    catch (InterruptedException | RejectedExecutionException e) {
      // handle the InterruptedException that gets wrapped in a RejectedExecutionException
      if (e instanceof RejectedExecutionException
          && (e.getCause() == null || !(e.getCause() instanceof InterruptedException))) {
        throw e;
      }

      // if we were interrupted because we were asked to stop, handle the exception and return success, else rethrow
      if (!stopRequested) {
        Thread.currentThread().interrupt();
        throw e;
      }

      log.info("The task was asked to stop before completing");
    }
    finally {
      if (chatHandlerProvider.isPresent()) {
        chatHandlerProvider.get().unregister(getId());
      }
    }

    return success();
  }

  @Override
  public boolean canRestore()
  {
    return true;
  }

  @POST
  @Path("/stop")
  @Override
  public void stopGracefully()
  {
    log.info("Stopping gracefully (status: [%s])", status);
    stopRequested = true;

    synchronized (statusLock) {
      if (status == Status.PUBLISHING) {
        runThread.interrupt();
        return;
      }
    }

    try {
      if (pauseLock.tryLock(LOCK_ACQUIRE_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
        try {
          if (pauseRequested) {
            pauseRequested = false;
            shouldResume.signalAll();
          }
        }
        finally {
          pauseLock.unlock();
        }
      } else {
        log.warn("While stopping: failed to acquire pauseLock before timeout, interrupting run thread");
        runThread.interrupt();
        return;
      }

      if (pollRetryLock.tryLock(LOCK_ACQUIRE_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
        try {
          isAwaitingRetry.signalAll();
        }
        finally {
          pollRetryLock.unlock();
        }
      } else {
        log.warn("While stopping: failed to acquire pollRetryLock before timeout, interrupting run thread");
        runThread.interrupt();
      }
    }
    catch (Exception e) {
      Throwables.propagate(e);
    }
  }

  @Override
  public <T> QueryRunner<T> getQueryRunner(Query<T> query)
  {
    if (appenderator == null) {
      // Not yet initialized, no data yet, just return a noop runner.
      return new NoopQueryRunner<>();
    }

    return new QueryRunner<T>()
    {
      @Override
      public Sequence<T> run(final Query<T> query, final Map<String, Object> responseContext)
      {
        return query.run(appenderator, responseContext);
      }
    };
  }

  @GET
  @Path("/status")
  @Produces(MediaType.APPLICATION_JSON)
  public Status getStatus()
  {
    return status;
  }

  @GET
  @Path("/offsets/current")
  @Produces(MediaType.APPLICATION_JSON)
  public Map<Integer, Long> getCurrentOffsets()
  {
    return nextOffsets;
  }

  @GET
  @Path("/offsets/end")
  @Produces(MediaType.APPLICATION_JSON)
  public Map<Integer, Long> getEndOffsets()
  {
    return endOffsets;
  }

  @POST
  @Path("/offsets/end")
  @Consumes(MediaType.APPLICATION_JSON)
  @Produces(MediaType.APPLICATION_JSON)
  public Response setEndOffsets(
      Map<Integer, Long> offsets,
      @QueryParam("resume") @DefaultValue("false") final boolean resume
  ) throws InterruptedException
  {
    if (offsets == null) {
      return Response.status(Response.Status.BAD_REQUEST)
                     .entity("Request body must contain a map of { partition:endOffset }")
                     .build();
    } else if (!endOffsets.keySet().containsAll(offsets.keySet())) {
      return Response.status(Response.Status.BAD_REQUEST)
                     .entity(
                         String.format(
                             "Request contains partitions not being handled by this task, my partitions: %s",
                             endOffsets.keySet()
                         )
                     )
                     .build();
    }

    pauseLock.lockInterruptibly();
    try {
      if (!isPaused()) {
        return Response.status(Response.Status.BAD_REQUEST)
                       .entity("Task must be paused before changing the end offsets")
                       .build();
      }

      for (Map.Entry<Integer, Long> entry : offsets.entrySet()) {
        if (entry.getValue().compareTo(nextOffsets.get(entry.getKey())) < 0) {
          return Response.status(Response.Status.BAD_REQUEST)
                         .entity(
                             String.format(
                                 "End offset must be >= current offset for partition [%s] (current: %s)",
                                 entry.getKey(),
                                 nextOffsets.get(entry.getKey())
                             )
                         )
                         .build();
        }
      }

      endOffsets.putAll(offsets);
      log.info("endOffsets changed to %s", endOffsets);
    }
    finally {
      pauseLock.unlock();
    }

    if (resume) {
      resume();
    }

    return Response.ok(endOffsets).build();
  }

  /**
   * Signals the ingestion loop to pause.
   *
   * @param timeout how long to pause for before resuming in milliseconds, <= 0 means indefinitely
   *
   * @return one of the following Responses: 400 Bad Request if the task has started publishing; 202 Accepted if the
   * method has timed out and returned before the task has paused; 200 OK with a map of the current partition offsets
   * in the response body if the task successfully paused
   */
  @POST
  @Path("/pause")
  @Produces(MediaType.APPLICATION_JSON)
  public Response pause(@QueryParam("timeout") @DefaultValue("0") final long timeout)
      throws InterruptedException
  {
    if (!(status == Status.PAUSED || status == Status.READING)) {
      return Response.status(Response.Status.BAD_REQUEST)
                     .entity(String.format("Can't pause, task is not in a pausable state (state: [%s])", status))
                     .build();
    }

    pauseLock.lockInterruptibly();
    try {
      pauseMillis = timeout <= 0 ? PAUSE_FOREVER : timeout;
      pauseRequested = true;

      pollRetryLock.lockInterruptibly();
      try {
        isAwaitingRetry.signalAll();
      }
      finally {
        pollRetryLock.unlock();
      }

      if (isPaused()) {
        shouldResume.signalAll(); // kick the monitor so it re-awaits with the new pauseMillis
      }

      long nanos = TimeUnit.SECONDS.toNanos(2);
      while (!isPaused()) {
        if (nanos <= 0L) {
          return Response.status(Response.Status.ACCEPTED)
                         .entity("Request accepted but task has not yet paused")
                         .build();
        }
        nanos = hasPaused.awaitNanos(nanos);
      }
    }
    finally {
      pauseLock.unlock();
    }

    try {
      return Response.ok().entity(mapper.writeValueAsString(getCurrentOffsets())).build();
    }
    catch (JsonProcessingException e) {
      throw Throwables.propagate(e);
    }
  }

  @POST
  @Path("/resume")
  public void resume() throws InterruptedException
  {
    pauseLock.lockInterruptibly();
    try {
      pauseRequested = false;
      shouldResume.signalAll();

      long nanos = TimeUnit.SECONDS.toNanos(5);
      while (isPaused()) {
        if (nanos <= 0L) {
          throw new RuntimeException("Resume command was not accepted within 5 seconds");
        }
        nanos = shouldResume.awaitNanos(nanos);
      }
    }
    finally {
      pauseLock.unlock();
    }
  }

  @GET
  @Path("/time/start")
  @Produces(MediaType.APPLICATION_JSON)
  public DateTime getStartTime()
  {
    return startTime;
  }

  @VisibleForTesting
  FireDepartmentMetrics getFireDepartmentMetrics()
  {
    return fireDepartmentMetrics;
  }

  private boolean isPaused()
  {
    return status == Status.PAUSED;
  }

  private Appenderator newAppenderator(FireDepartmentMetrics metrics, TaskToolbox toolbox)
  {
    final int maxRowsInMemoryPerPartition = (tuningConfig.getMaxRowsInMemory() /
                                             ioConfig.getStartPartitions().getPartitionOffsetMap().size());
    return Appenderators.createRealtime(
        dataSchema,
        tuningConfig.withBasePersistDirectory(new File(toolbox.getTaskWorkDir(), "persist"))
                    .withMaxRowsInMemory(maxRowsInMemoryPerPartition),
        metrics,
        toolbox.getSegmentPusher(),
        toolbox.getObjectMapper(),
        toolbox.getIndexIO(),
        tuningConfig.getBuildV9Directly() ? toolbox.getIndexMergerV9() : toolbox.getIndexMerger(),
        toolbox.getQueryRunnerFactoryConglomerate(),
        toolbox.getSegmentAnnouncer(),
        toolbox.getEmitter(),
        toolbox.getQueryExecutorService(),
        toolbox.getCache(),
        toolbox.getCacheConfig()
    );
  }

  private FiniteAppenderatorDriver newDriver(
      final Appenderator appenderator,
      final TaskToolbox toolbox,
      final FireDepartmentMetrics metrics
  )
  {
    return new FiniteAppenderatorDriver(
        appenderator,
        new ActionBasedSegmentAllocator(toolbox.getTaskActionClient(), dataSchema),
        toolbox.getSegmentHandoffNotifierFactory(),
        new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()),
        toolbox.getObjectMapper(),
        tuningConfig.getMaxRowsPerSegment(),
        tuningConfig.getHandoffConditionTimeout(),
        metrics
    );
  }

  private KafkaConsumer<byte[], byte[]> newConsumer()
  {
    ClassLoader currCtxCl = Thread.currentThread().getContextClassLoader();
    try {
      Thread.currentThread().setContextClassLoader(getClass().getClassLoader());

      final Properties props = new Properties();

      for (Map.Entry<String, String> entry : ioConfig.getConsumerProperties().entrySet()) {
        props.setProperty(entry.getKey(), entry.getValue());
      }

      props.setProperty("enable.auto.commit", "false");
      props.setProperty("auto.offset.reset", "none");
      props.setProperty("key.deserializer", ByteArrayDeserializer.class.getName());
      props.setProperty("value.deserializer", ByteArrayDeserializer.class.getName());

      return new KafkaConsumer<>(props);
    }
    finally {
      Thread.currentThread().setContextClassLoader(currCtxCl);
    }
  }

  private static void assignPartitions(
      final KafkaConsumer consumer,
      final String topic,
      final Set<Integer> partitions
  )
  {
    consumer.assign(
        Lists.newArrayList(
            Iterables.transform(
                partitions,
                new Function<Integer, TopicPartition>()
                {
                  @Override
                  public TopicPartition apply(Integer n)
                  {
                    return new TopicPartition(topic, n);
                  }
                }
            )
        )
    );
  }

  private Set<Integer> assignPartitionsAndSeekToNext(KafkaConsumer consumer, String topic)
  {
    // Initialize consumer assignment.
    final Set<Integer> assignment = Sets.newHashSet();
    for (Map.Entry<Integer, Long> entry : nextOffsets.entrySet()) {
      final long endOffset = endOffsets.get(entry.getKey());
      if (entry.getValue() < endOffset) {
        assignment.add(entry.getKey());
      } else if (entry.getValue() == endOffset) {
        log.info("Finished reading partition[%d].", entry.getKey());
      } else {
        throw new ISE(
            "WTF?! Cannot start from offset[%,d] > endOffset[%,d]",
            entry.getValue(),
            endOffset
        );
      }
    }

    assignPartitions(consumer, topic, assignment);

    // Seek to starting offsets.
    for (final int partition : assignment) {
      final long offset = nextOffsets.get(partition);
      log.info("Seeking partition[%d] to offset[%,d].", partition, offset);
      consumer.seek(new TopicPartition(topic, partition), offset);
    }

    return assignment;
  }

  /**
   * Checks if the pauseRequested flag was set and if so blocks:
   * a) if pauseMillis == PAUSE_FOREVER, until pauseRequested is cleared
   * b) if pauseMillis != PAUSE_FOREVER, until pauseMillis elapses -or- pauseRequested is cleared
   * <p/>
   * If pauseMillis is changed while paused, the new pause timeout will be applied. This allows adjustment of the
   * pause timeout (making a timed pause into an indefinite pause and vice versa is valid) without having to resume
   * and ensures that the loop continues to stay paused without ingesting any new events. You will need to signal
   * shouldResume after adjusting pauseMillis for the new value to take effect.
   * <p/>
   * Sets paused = true and signals paused so callers can be notified when the pause command has been accepted.
   * <p/>
   * Additionally, pauses if all partitions assignments have been read and pauseAfterRead flag is set.
   *
   * @return true if a pause request was handled, false otherwise
   */
  private boolean possiblyPause(Set<Integer> assignment) throws InterruptedException
  {
    pauseLock.lockInterruptibly();
    try {
      if (ioConfig.isPauseAfterRead() && assignment.isEmpty()) {
        pauseMillis = PAUSE_FOREVER;
        pauseRequested = true;
      }

      if (pauseRequested) {
        status = Status.PAUSED;
        long nanos = 0;
        hasPaused.signalAll();

        while (pauseRequested) {
          if (pauseMillis == PAUSE_FOREVER) {
            log.info("Pausing ingestion until resumed");
            shouldResume.await();
          } else {
            if (pauseMillis > 0) {
              log.info("Pausing ingestion for [%,d] ms", pauseMillis);
              nanos = TimeUnit.MILLISECONDS.toNanos(pauseMillis);
              pauseMillis = 0;
            }
            if (nanos <= 0L) {
              pauseRequested = false; // timeout elapsed
            }
            nanos = shouldResume.awaitNanos(nanos);
          }
        }

        status = Status.READING;
        shouldResume.signalAll();
        log.info("Ingestion loop resumed");
        return true;
      }
    }
    finally {
      pauseLock.unlock();
    }

    return false;
  }

  private void possiblyResetOffsetsOrWait(
      Map<TopicPartition, Long> outOfRangePartitions,
      KafkaConsumer<byte[], byte[]> consumer,
      TaskToolbox taskToolbox
  ) throws InterruptedException, IOException
  {
    final Map<TopicPartition, Long> resetPartitions = Maps.newHashMap();
    boolean doReset = false;
    if (tuningConfig.isResetOffsetAutomatically()) {
      for (Map.Entry<TopicPartition, Long> outOfRangePartition : outOfRangePartitions.entrySet()) {
        final TopicPartition topicPartition = outOfRangePartition.getKey();
        final long nextOffset = outOfRangePartition.getValue();
        // seek to the beginning to get the least available offset
        consumer.seekToBeginning(topicPartition);
        final long leastAvailableOffset = consumer.position(topicPartition);
        // reset the seek
        consumer.seek(topicPartition, nextOffset);
        // Reset consumer offset if resetOffsetAutomatically is set to true
        // and the current message offset in the kafka partition is more than the
        // next message offset that we are trying to fetch
        if (leastAvailableOffset > nextOffset) {
          doReset = true;
          resetPartitions.put(topicPartition, nextOffset);
        }
      }
    }

    if (doReset) {
      sendResetRequestAndWait(resetPartitions, taskToolbox);
    } else {
      log.warn("Retrying in %dms", POLL_RETRY_MS);
      pollRetryLock.lockInterruptibly();
      try {
        long nanos = TimeUnit.MILLISECONDS.toNanos(POLL_RETRY_MS);
        while (nanos > 0L && !pauseRequested && !stopRequested) {
          nanos = isAwaitingRetry.awaitNanos(nanos);
        }
      }
      finally {
        pollRetryLock.unlock();
      }
    }
  }

  private void sendResetRequestAndWait(Map<TopicPartition, Long> outOfRangePartitions, TaskToolbox taskToolbox)
      throws IOException
  {
    Map<Integer, Long> partitionOffsetMap = Maps.newHashMap();
    for (Map.Entry<TopicPartition, Long> outOfRangePartition : outOfRangePartitions.entrySet()) {
      partitionOffsetMap.put(outOfRangePartition.getKey().partition(), outOfRangePartition.getValue());
    }
    boolean result = taskToolbox.getTaskActionClient()
                                .submit(new ResetDataSourceMetadataAction(
                                    getDataSource(),
                                    new KafkaDataSourceMetadata(new KafkaPartitions(
                                        ioConfig.getStartPartitions()
                                                .getTopic(),
                                        partitionOffsetMap
                                    ))
                                ));

    if (result) {
      log.makeAlert("Resetting Kafka offsets for datasource [%s]", getDataSource())
         .addData("partitions", partitionOffsetMap.keySet())
         .emit();
      // wait for being killed by supervisor
      try {
        Thread.sleep(Long.MAX_VALUE);
      }
      catch (InterruptedException e) {
        throw new RuntimeException("Got interrupted while waiting to be killed");
      }
    } else {
      log.makeAlert("Failed to send reset request for partitions [%s]", partitionOffsetMap.keySet()).emit();
    }
  }
}


text:
using Aspose.Tasks.Util;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

/*
This project uses Automatic Package Restore feature of NuGet to resolve Aspose.Tasks for .NET API reference
when the project is build. Please check https:// Docs.nuget.org/consume/nuget-faq for more information.
If you do not wish to use NuGet, you can manually download Aspose.Tasks for .NET API from https://www.nuget.org/packages/Aspose.Tasks/,
install it and then add its reference to this project. For any issues, questions or suggestions
please feel free to contact us using https://forum.aspose.com/c/tasks
*/

namespace Aspose.Tasks.Examples.CSharp.WorkingWithTasks
{
    class ReadActualTaskProperties
    {
        public static void Run()
        {
            // ExStart:ReadActualTaskProperties
            // The path to the documents directory.
            string dataDir = RunExamples.GetDataDir(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType.FullName);

            // Create Project instance
            Project project = new Project(dataDir + "ActualTaskProperties.mpp");

            // Create a ChildTasksCollector instance
            ChildTasksCollector collector = new ChildTasksCollector();

            // Collect all the tasks from RootTask using TaskUtils
            TaskUtils.Apply(project.RootTask, collector, 0);

            // Parse through all the collected tasks
            foreach (Task task in collector.Tasks)
            {
                Console.WriteLine("Task Name : " + task.Get(Tsk.Name));
                Console.WriteLine("Actual Start: " + task.Get(Tsk.ActualStart).ToLongDateString());
                Console.WriteLine("Actual Finish: " + task.Get(Tsk.ActualFinish).ToLongDateString());
                Console.WriteLine("Actual Duration: " + task.Get(Tsk.ActualDuration).TimeSpan.Hours.ToString());
                Console.WriteLine("Actual Cost: " + task.Get(Tsk.ActualCost).ToString());
                Console.WriteLine("---------------------------------------------");
            }
            // ExEnd:ReadActualTaskProperties
        }
    }
}

redpajama_set_name: RedPajamaGithub


text:
package org.apache.causeway.core.metamodel.facets.object.domainobject.entitychangepublishing;

import org.apache.causeway.core.metamodel.facetapi.FacetHolder;

public class EntityChangePublishingFacetForDomainObjectAnnotationAsConfigured extends EntityChangePublishingFacetForDomainObjectAnnotation {

    public EntityChangePublishingFacetForDomainObjectAnnotationAsConfigured(final FacetHolder facetHolder, final boolean enabled) {
        super(facetHolder, enabled);
    }

}


text:
<?php

namespace Modules\Recipe\Providers;

use Illuminate\Contracts\Support\DeferrableProvider;
use Illuminate\Support\ServiceProvider;

class DeferredServiceProvider extends ServiceProvider implements DeferrableProvider
{
    /**
     * Register the service provider.
     *
     * @return void
     */
    public function register()
    {
        app()->bind('foo', function () {
            return 'bar';
        });

        app()->bind('deferred', function () {
            return;
        });
    }

    /**
     * Get the services provided by the provider.
     *
     * @return array
     */
    public function provides()
    {
        return ['deferred'];
    }
}


text:
package tablesaw;

import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 Created by bhawkins on 3/13/14.
 */
public class DependencyAnnotations
	{
	private Map<String, Method> m_consumesMethods;
	private Map<String, Method> m_providesMethods;

	public DependencyAnnotations()
		{
		m_consumesMethods = new HashMap<String, Method>();
		m_providesMethods = new HashMap<String, Method>();
		}

	public void addConsumesMethod(String name, Method method)
		{
		m_consumesMethods.put(name, method);
		}

	public List<String> getConsumesTypes()
		{
		return new ArrayList<String>(m_consumesMethods.keySet());
		}

	public Method getConsumesMethod(String type)
		{
		return m_consumesMethods.get(type);
		}

	public void addProvidesMethod(String name, Method method)
		{
		m_providesMethods.put(name, method);
		}

	public List<String> getProvidesTypes()
		{
		return new ArrayList<String>(m_providesMethods.keySet());
		}

	public Method getProvidesMethod(String type)
		{
		return (m_providesMethods.get(type));
		}
	}


text:
#ifndef SERIALIZE_H
#define SERIALIZE_H
#include "rapidjson/document.h"
#include <string>
#include <sstream>
#include <functional>
#include "helper.h"

struct NvpHelper {
    template <class T>
    inline static std::pair<const char *, T> make_nvp(const char *name, T &&value) {
        return{name, std::forward<T>(value)};
    }
};

#define NVP(T) NvpHelper::make_nvp(#T, T)
#define DESERIALIZE(...) DeserializeHelper::deserialize(val, __VA_ARGS__)

struct DeserializeHelper {
    static void deserialize(const rapidjson::Value &val)
    {
        (void)val;
    }

    template <typename Last>
    static void deserialize(const rapidjson::Value &val, const Last &last)
    {
        __deserialize(val, last.first, last.second);
    }

    template <typename First, typename... Rest>
    static void deserialize(const rapidjson::Value &val, const First &first, const Rest&... rest)
    {
        __deserialize(val, first.first, first.second);
        deserialize(val, rest...);
    }

private:

    template<typename _Ty>
    static void __deserialize_array_part(const rapidjson::Value &val, _Ty &target)
    {
        target.Deserialize(val);
    }

    template<typename _Ty>
    static void __deserialize_array_part(const rapidjson::Value &arr, std::vector<_Ty> &target)
    {
        if(arr.IsArray())
        {
            for(decltype(arr.Size()) i = 0; i < arr.Size(); i++)
            {
                _Ty part;
                __deserialize_array_part(arr[i], part);
                target.push_back(std::move(part));
            }
        }
    }

    template<typename _Ty>
    static void __deserialize(const rapidjson::Value &val, const char *s, _Ty &target)
    {
        if(!val.HasMember(s))
            throw std::invalid_argument(sge::string_format("value has no %s member", s));

        const rapidjson::Value &v = val[s];
        target.Deserialize(v);
    }

    template<typename _Ty>
    static void __deserialize(const rapidjson::Value &val, const char *s, std::vector<_Ty> &target)
    {

        if(val.HasMember(s) && val[s].IsArray())
        {
            const rapidjson::Value &arr = val[s];
            for(decltype(arr.Size()) i = 0; i < arr.Size(); i++)
            {
                _Ty part;
                __deserialize_array_part(arr[i], part);
                target.push_back(std::move(part));
            }
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, int &target)
    {
        if(val.HasMember(s))
        {
            if(!val[s].IsInt())
                throw std::invalid_argument(sge::string_format("value %s is not a integer", s));
            target = val[s].GetInt();
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, std::string &target)
    {
        if(val.HasMember(s))
        {
            if(!val[s].IsString())
                throw std::invalid_argument(sge::string_format("value %s is not a string", s));
            target = val[s].GetString();
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, float &target)
    {
        if(val.HasMember(s))
        {
            if(!val[s].IsNumber())
                throw std::invalid_argument(sge::string_format("value %s is not a number", s));
            target = static_cast<float>(val[s].GetDouble());
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, bool &target)
    {
        if(val.HasMember(s))
        {
            if(!val[s].IsBool_())
                throw std::invalid_argument(sge::string_format("value %s is not a bool", s));
            target = val[s].GetBool_();
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, glm::vec2 &target)
    {
        if(val.HasMember(s) && val[s].IsArray())
        {
            const rapidjson::Value &arr = val[s];
            if(arr.Size() != 2)          throw std::invalid_argument(sge::string_format("value %s is not vec2", s));
            if(!arr.Begin()->IsNumber()) throw std::invalid_argument(sge::string_format("value %s[0] is not a number", s));
            if(!arr[1].IsNumber())       throw std::invalid_argument(sge::string_format("value %s[1] is not a number", s));

            target.x = static_cast<float>(arr.Begin()->GetDouble());
            target.y = static_cast<float>(arr[1].GetDouble());
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, glm::vec3 &target)
    {
        if(val.HasMember(s) && val[s].IsArray())
        {
            const rapidjson::Value &arr = val[s];
            if(arr.Size() != 3)          throw std::invalid_argument("value is not vec3");
            if(!arr.Begin()->IsNumber()) throw std::invalid_argument(sge::string_format("value %s[0] is not a number", s));
            if(!arr[1].IsNumber())       throw std::invalid_argument(sge::string_format("value %s[1] is not a number", s));
            if(!arr[2].IsNumber())       throw std::invalid_argument(sge::string_format("value %s[2] is not a number", s));

            target.x = static_cast<float>(arr.Begin()->GetDouble());
            target.y = static_cast<float>(arr[1].GetDouble());
            target.z = static_cast<float>(arr[2].GetDouble());
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, glm::vec4 &target)
    {
        if(val.HasMember(s) && val[s].IsArray())
        {
            const rapidjson::Value &arr = val[s];
            if(arr.Size() != 4)          throw std::invalid_argument("target variable is not vec4");
            if(!arr.Begin()->IsNumber()) throw std::invalid_argument(sge::string_format("value %s[0] is not a number", s));
            if(!arr[1].IsNumber())       throw std::invalid_argument(sge::string_format("value %s[1] is not a number", s));
            if(!arr[2].IsNumber())       throw std::invalid_argument(sge::string_format("value %s[2] is not a number", s));
            if(!arr[3].IsNumber())       throw std::invalid_argument(sge::string_format("value %s[3] is not a number", s));

            target.x = static_cast<float>(arr.Begin()->GetDouble());
            target.y = static_cast<float>(arr[1].GetDouble());
            target.z = static_cast<float>(arr[2].GetDouble());
            target.w = static_cast<float>(arr[3].GetDouble());
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, std::vector<int> &target)
    {
        if(val.HasMember(s))
        {
            const rapidjson::Value &arr = val[s];
            for(decltype(arr.Size()) i = 0; i < arr.Size(); i++)
            {
                if(!arr[i].IsInt())
                    throw std::invalid_argument(sge::string_format("value %s[%d] is not a integer", s, i));
                target.push_back(arr[i].GetInt());
            }
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, std::vector<std::string> &target)
    {
        if(val.HasMember(s) && val[s].IsArray())
        {
            const rapidjson::Value &arr = val[s];
            for(decltype(arr.Size()) i = 0; i < arr.Size(); i++)
            {
                if(!arr[i].IsString())
                    throw std::invalid_argument(sge::string_format("value %s[%d] is not a string", s, i));
                target.push_back(arr[i].GetString());
            }
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, std::vector<bool> &target)
    {
        if(val.HasMember(s) && val[s].IsArray())
        {
            const rapidjson::Value &arr = val[s];
            for(decltype(arr.Size()) i = 0; i < arr.Size(); i++)
            {
                if(!arr[i].IsBool_())
                    throw std::invalid_argument(sge::string_format("value %s[%d] is not a bool", s, i));
                target.push_back(arr[i].GetBool_());
            }
        }
    }

    template<>
    static void __deserialize(const rapidjson::Value &val, const char *s, std::vector<float> &target)
    {
        if(val.HasMember(s) && val[s].IsArray())
        {
            const rapidjson::Value &arr = val[s];
            for(decltype(arr.Size()) i = 0; i < arr.Size(); i++)
            {
                if(!arr[i].IsDouble())
                    throw std::invalid_argument(sge::string_format("value %s[%d] is not a number", s, i));
                target.push_back(static_cast<float>(arr[i].GetDouble()));
            }
        }
    }
};

#endif // SERIALIZE_H


text:
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
  <id>pdi-tde-output-plugin-assembly</id>
  <formats>
    <format>zip</format>
  </formats>
  <includeBaseDirectory>true</includeBaseDirectory>
  <baseDirectory>${project.artifactId}</baseDirectory>

  <fileSets>
    <fileSet>
      <directory>${project.basedir}</directory>
      <outputDirectory></outputDirectory>
      <includes>
        <include>README*</include>
        <include>LICENSE*</include>
      </includes>
    </fileSet>
    <fileSet>
      <directory>${project.build.directory}</directory>
      <outputDirectory />
      <includes>
        <include>${project.artifactId}-${project.version}.jar</include>
      </includes>
    </fileSet>
    <fileSet>
      <directory>${project.basedir}/src/main/package-res</directory>
      <outputDirectory />
    </fileSet>
  </fileSets>

</assembly>


text:
/*!
 * @file VideoCapture.cpp
 * @author a_hasimoto
 * @date Date Created: 2012/Jan/12
 * @date Last Change: 2013/Dec/02.
 */
#include "sklVideoCapture.h"
#include "VideoCaptureDefault.h"
#include "VideoCaptureImageList.h"
#include "VideoCaptureOptFlowImageList.h"
#include <sstream>
#include <fstream>

using namespace skl;

/*!
 * @brief デフォルトコンストラクタ
 */
VideoCapture::VideoCapture():VideoCaptureInterface<VideoCapture>(){
}

/*!
 * @brief デストラクタ
 */
VideoCapture::~VideoCapture(){
	release();
}

bool VideoCapture::isOpened()const{
	if(size()==0) return false;
	for(size_t i=0;i<size();i++){
		if(cam_interface[i].empty()) return false;
		if(!cam_interface[i]->isOpened()) return false;
	}
	return true;
}

void VideoCapture::release(){
	for(size_t i=0;i<size();i++){
		if(!cam_interface[i].empty()){
			cam_interface[i]->release();
		}
	}
	cam_interface.clear();
}

class ParallelGrab{
	public:
		ParallelGrab(VideoCapture* _cam,std::vector<bool>* _flags):cam(_cam),flags(_flags){}
		~ParallelGrab(){}
		void operator()(const cv::BlockedRange& r)const{
			for(int i=r.begin();i!=r.end();i++){
				(*flags)[i] = (*cam)[i].grab();
//				std::cerr << i << ": " << flags[i] << std::endl;
			}
		}
	protected:
		VideoCapture* cam;
		std::vector<bool>* flags;
};

bool VideoCapture::grab(){
	std::vector<bool> flags(size(),false);
	cv::parallel_for(cv::BlockedRange(0,size()),ParallelGrab(this,&flags));
	for(size_t i=0;i<size();i++){
		if(!flags[i]) return false;
	}

/*
	for(size_t i=0;i<size();i++){
		if(!cam_interface[i]->grab()) return false;
	}
	*/
	return true;
}

bool VideoCapture::retrieve(cv::Mat& image, int channel){
	if(cam_interface.empty()){
		image.release();
		return false;
	}
	return cam_interface[0]->retrieve(image,channel);
}

bool VideoCapture::set(capture_property_t prop_id,double val){
	for(size_t i=0;i<size();i++){
		if(!cam_interface[i]->set(prop_id,val)) return false;
	}
	return true;
}

double VideoCapture::get(capture_property_t prop_id){
	if(cam_interface.empty()) return 0.0;
	double val = cam_interface[0]->get(prop_id);
	for(size_t i=1;i<size();i++){
		if(val!=cam_interface[i]->get(prop_id)) return 0.0;
	}
	return val;
}

/*
 * @brief this is used to set/get parameters for cameras independently, when VideoCapture manage several cameras.
 * @param device derect camera device.
 * @return an instance which has the same get/set functions, but for camera[device].
 * */
_VideoCaptureInterface& VideoCapture::operator[](int device){
	assert(static_cast<unsigned int>(device) < size());
	return *(cam_interface[device]);
}

bool VideoCapture::push_back(const std::string& filename){
	std::string extname = ExtName(filename);
	cv::Ptr<_VideoCaptureInterface> capture;
	if(extname==".lst" || extname==".lst0"){
		capture = new VideoCaptureImageList();
	}
	else if(extname==".lstf" || extname==".lstflow"){
		capture = new VideoCaptureOptFlowImageList();
	}
	else{
		capture = new VideoCaptureDefault();
	}
	if(!capture->open(filename)){
		return false;
	}
	cam_interface.push_back(capture);

	return true;
}
bool VideoCapture::push_back(const std::string& filename, cv::Ptr<_VideoCaptureInterface> cam){
	if(!cam->open(filename)){
		return false;
	}
	cam_interface.push_back(cam);
	return true;
}

bool VideoCapture::push_back(int device){
	cv::Ptr<_VideoCaptureInterface> video_capture = new VideoCaptureDefault();
	if(!video_capture->open(device)){
		return false;
	}
	cam_interface.push_back(video_capture);
	return true;
}
bool VideoCapture::push_back(int device, cv::Ptr<_VideoCaptureInterface> cam){
	if(!cam->open(device)){
		return false;
	}
	cam_interface.push_back(cam);
	return true;
}

class VideoCapture_parallel_retrieve{
	public:
		VideoCapture_parallel_retrieve(
				std::vector<cv::Ptr<_VideoCaptureInterface> >& cam_interface,
				std::vector<cv::Mat>& mat_vec):cam_interface(cam_interface),mat_vec(mat_vec){}
		~VideoCapture_parallel_retrieve(){}
		void operator()(const cv::BlockedRange& range)const{
			for(int i = range.begin(); i != range.end(); i++){
				cam_interface[i]->retrieve(mat_vec[i],0);
			}
		}
	protected:
		std::vector<cv::Ptr<_VideoCaptureInterface> >& cam_interface;
		std::vector<cv::Mat>& mat_vec;
};

VideoCapture& VideoCapture::operator>>(std::vector<cv::Mat>& mat_vec){
	if(size()!=mat_vec.size()){
		mat_vec.resize(size());
	}
	if(!grab()){
		mat_vec.clear();
	}
	else{
		cv::parallel_for(
				cv::BlockedRange(0,(int)size()),
				VideoCapture_parallel_retrieve(cam_interface,mat_vec)
				);
	}
	return *this;
}


text:
'use strict';

const momenttz = require('moment-timezone');
const jm = require('moment-jalali');
const date = new Date();
const day = date.getDay();
const days = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'];

console.log("\n");
console.log("\t%s\n\t=========================", days[day].charAt(0).toUpperCase() + days[day].substr(1).toLowerCase());
console.log("\tJ: %s %s (%s)",
	jm().format('jYYYY/jMM/jDD'),
	String(momenttz(Date.now()).tz('Asia/Tehran').format("HH:mm")),
	"IRI");
console.log("\tG: %s %s (%s)",
	jm().format('YYYY/MM/DD'),
	String(momenttz(Date.now()).tz('Europe/London').format("HH:mm")),
	"GMT");
console.log("\n");


text:
"""OTT tools: A set of tools to use OT in differentiable ML pipelines."""

from . import plot
from . import sinkhorn_divergence
from . import soft_sort
from . import transport


text:
#include "board.h"

#ifdef BSP_USING_ONCHIP_RTC

//#define DRV_DEBUG
#define LOG_TAG             "drv.rtc"
#include <drv_log.h>

#define BKUP_REG_DATA 0xA5A5

static struct rt_device rtc;

static RTC_HandleTypeDef RTC_Handler;

static time_t get_rtc_timestamp(void)
{
    RTC_TimeTypeDef RTC_TimeStruct;
    RTC_DateTypeDef RTC_DateStruct;
    struct tm tm_new;

    HAL_RTC_GetTime(&RTC_Handler, &RTC_TimeStruct, RTC_FORMAT_BIN);
    HAL_RTC_GetDate(&RTC_Handler, &RTC_DateStruct, RTC_FORMAT_BIN);

    tm_new.tm_sec  = RTC_TimeStruct.Seconds;
    tm_new.tm_min  = RTC_TimeStruct.Minutes;
    tm_new.tm_hour = RTC_TimeStruct.Hours;
    tm_new.tm_mday = RTC_DateStruct.Date;
    tm_new.tm_mon  = RTC_DateStruct.Month - 1;
    tm_new.tm_year = RTC_DateStruct.Year + 100;

    LOG_D("get rtc time.");
    return mktime(&tm_new);
}

static rt_err_t set_rtc_time_stamp(time_t time_stamp)
{
    RCC_PeriphCLKInitTypeDef PeriphClkInitStruct;
    RTC_TimeTypeDef RTC_TimeStruct;
    RTC_DateTypeDef RTC_DateStruct;
    struct tm *p_tm;

    HAL_PWR_EnableBkUpAccess();
    PeriphClkInitStruct.PeriphClockSelection = RCC_PERIPHCLK_RTC;
    PeriphClkInitStruct.RTCClockSelection = RCC_RTCCLKSOURCE_LSE;
    HAL_RCCEx_PeriphCLKConfig(&PeriphClkInitStruct);

    p_tm = localtime(&time_stamp);
    if (p_tm->tm_year < 100)
    {
        return -RT_ERROR;
    }

    RTC_TimeStruct.Seconds = p_tm->tm_sec ;
    RTC_TimeStruct.Minutes = p_tm->tm_min ;
    RTC_TimeStruct.Hours   = p_tm->tm_hour;
    RTC_DateStruct.Date    = p_tm->tm_mday;
    RTC_DateStruct.Month   = p_tm->tm_mon + 1 ;
    RTC_DateStruct.Year    = p_tm->tm_year - 100;
    RTC_DateStruct.WeekDay = p_tm->tm_wday + 1;

    if(HAL_RTC_SetTime(&RTC_Handler, &RTC_TimeStruct, RTC_FORMAT_BIN) != HAL_OK)
    {
        return -RT_ERROR;
    }
    if(HAL_RTC_SetDate(&RTC_Handler, &RTC_DateStruct, RTC_FORMAT_BIN) != HAL_OK)
    {
        return -RT_ERROR;
    }

    LOG_D("set rtc time.");
    HAL_RTCEx_BKUPWrite(&RTC_Handler, RTC_BKP_DR1, BKUP_REG_DATA);
    return RT_EOK;
}

static void rt_rtc_init(void)
{
    __HAL_RCC_PWR_CLK_ENABLE();

    RCC_OscInitTypeDef RCC_OscInitStruct;
    RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_LSE;
    RCC_OscInitStruct.PLL.PLLState = RCC_PLL_NONE;
    RCC_OscInitStruct.LSEState = RCC_LSE_ON;
    HAL_RCC_OscConfig(&RCC_OscInitStruct);
}

static rt_err_t rt_rtc_config(struct rt_device *dev)
{
    RCC_PeriphCLKInitTypeDef PeriphClkInitStruct;

    HAL_PWR_EnableBkUpAccess();
    PeriphClkInitStruct.PeriphClockSelection = RCC_PERIPHCLK_RTC;
    PeriphClkInitStruct.RTCClockSelection = RCC_RTCCLKSOURCE_LSE;
    HAL_RCCEx_PeriphCLKConfig(&PeriphClkInitStruct);

    RTC_Handler.Instance = RTC;
    if (HAL_RTCEx_BKUPRead(&RTC_Handler, RTC_BKP_DR1) != BKUP_REG_DATA)
    {
        LOG_W("RTC hasn't been configured, please use <date> command to config.");

#if defined(SOC_SERIES_STM32F1)
        RTC_Handler.Init.OutPut = RTC_OUTPUTSOURCE_NONE;
        RTC_Handler.Init.AsynchPrediv = RTC_AUTO_1_SECOND;
#elif defined(SOC_SERIES_STM32F4) || defined(SOC_SERIES_STM32L4)
        RTC_Handler.Init.AsynchPrediv = 0X7F;
        RTC_Handler.Init.SynchPrediv = 0XFF;
        RTC_Handler.Init.HourFormat = RTC_HOURFORMAT_24;
        RTC_Handler.Init.OutPut = RTC_OUTPUT_DISABLE;
        RTC_Handler.Init.OutPutPolarity = RTC_OUTPUT_POLARITY_HIGH;
        RTC_Handler.Init.OutPutType = RTC_OUTPUT_TYPE_OPENDRAIN;
#endif
        if (HAL_RTC_Init(&RTC_Handler) != HAL_OK)
        {
            return -RT_ERROR;
        }
    }
    return RT_EOK;
}

static rt_err_t rt_rtc_control(rt_device_t dev, int cmd, void *args)
{
    rt_err_t result = RT_EOK;
    RT_ASSERT(dev != RT_NULL);
    switch (cmd)
    {
    case RT_DEVICE_CTRL_RTC_GET_TIME:
        *(rt_uint32_t *)args = get_rtc_timestamp();
        LOG_D("RTC: get rtc_time %x\n", *(rt_uint32_t *)args);
        break;

    case RT_DEVICE_CTRL_RTC_SET_TIME:
        if(set_rtc_time_stamp(*(rt_uint32_t *)args))
        {
            result = -RT_ERROR;
        }
        LOG_D("RTC: set rtc_time %x\n", *(rt_uint32_t *)args);
        break;
    }

    return result;
}

#ifdef RT_USING_DEVICE_OPS
const static struct rt_device_ops rtc_ops =
{
    RT_NULL,
    RT_NULL,
    RT_NULL,
    RT_NULL,
    RT_NULL,
    rt_rtc_control
};
#endif

static rt_err_t rt_hw_rtc_register(rt_device_t device, const char *name, rt_uint32_t flag)
{
    RT_ASSERT(device != RT_NULL);

    rt_rtc_init();
    if (rt_rtc_config(device) != RT_EOK)
    {
        return -RT_ERROR;
    }
#ifdef RT_USING_DEVICE_OPS
    device->ops         = &rtc_ops;
#else
    device->init        = RT_NULL;
    device->open        = RT_NULL;
    device->close       = RT_NULL;
    device->read        = RT_NULL;
    device->write       = RT_NULL;
    device->control     = rt_rtc_control;
#endif
    device->type        = RT_Device_Class_RTC;
    device->rx_indicate = RT_NULL;
    device->tx_complete = RT_NULL;
    device->user_data   = RT_NULL;

    /* register a character device */
    return rt_device_register(device, name, flag);
}

int rt_hw_rtc_init(void)
{
    rt_err_t result;
    result = rt_hw_rtc_register(&rtc, "rtc", RT_DEVICE_FLAG_RDWR);
    if (result != RT_EOK)
    {
        LOG_E("rtc register err code: %d", result);
        return result;
    }
    LOG_D("rtc init success");
    return RT_EOK;
}
INIT_DEVICE_EXPORT(rt_hw_rtc_init);

#endif /* BSP_USING_ONCHIP_RTC */


text:
<?xml version="1.0" encoding="UTF-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android" android:shape="rectangle">
    <padding android:left="10dp" android:right="10dp" android:top="10dp" android:bottom="10dp"/>
    <stroke android:width="1dip" android:color="#d8d8d8" />
</shape>


text:
**Table of Contents**

- [Folders](#folders)
  - [auth](#auth)
  - [client](#client)
  - [data](#data)
  - [feeders](#feeders)
  - [idas](#idas)
  - [misc](#misc)
  - [public](#public)
  - [routes](#routes)
  - [schema](#schema)
  - [spec](#spec)
- [Files](#files)
  - [.jscsrc](#jscsrc)
  - [.jshintrc](#jshintrc)
  - [app.js](#appjs)
  - [config.js](#configjs)
  - [Gruntfile.js](#gruntfilejs)
  - [package.json](#packagejson)
  - [setup-test-env.sh](#setup-test-envsh)
  - [utils.js](#utilsjs)

## Folders

### auth

Contains the files with the functions needed to interact with [Keyrock](http://catalogue.fiware.org/enablers/identity-management-keyrock) and its [Oauth2 Authentication flow](http://fiware-idm.readthedocs.io/en/latest/oauth2.html#oauth2-authentication). See more [inside](auth/README.md).

### client

Symlink to the client side.

### data

Contains the Open Data restaurants file provided by the [Basque Government](http://opendata.euskadi.eus/catalogo/-/restaurantes-asadores-sidrerias-bodegas-y-bares-de-pintxos-de-euskadi/), to use with the [restaurant feeder](feeders/README.md) to fill the restaurant-data [image](https://hub.docker.com/r/fiware/tutorials.tourguide-app.restaurant-data/).

### feeders

Set of tools to generate restaurants, reviews and reservations using the dataset mentioned below. You can see further instructions regarding it's usage [inside the folder](feeders/README.md).

### idas

`ul20.js` contains the [ultralight 2.0](https://github.com/telefonicaid/fiware-IoTAgent-Cplusplus/blob/release/1.0.2/doc/UL20_protocol.md) functions needed to generate new Sensors and update them in [IDAS](http://catalogue.fiware.org/enablers/backend-device-management-idas).

### misc

Contains a simple node.js script that allows to retrieve a Oauth `token` using the [Resource Owner Credentials Grant](http://fiware-idm.readthedocs.io/en/latest/oauth2.html#resource-owner-password-credentials-grant) from [Keyrock](http://catalogue.fiware.org/enablers/identity-management-keyrock). See it's [usage](misc/README.md).

### public

This folder is usually intended for static files only, but in our case, the application is served by [Apache server](https://httpd.apache.org/), as it is much better than Node in serving static files.

### routes

Defines the application routes and the logic to interact with [Orion](http://catalogue.fiware.org/enablers/publishsubscribe-context-broker-orion-context-broker). `orion.js` is the file containing the functions that interacts with the Context Broker. Also, the methods and functions will handle the data received through the server API. This includes not only the methods for creating, reading, updating and deleting items, but also any additional logic.

### schema

`schema.js` contains the schema types definitions to validate restaurants, reviews and reservations entities.

### spec

[jasmine](http://jasmine.github.io/) based end to end tests. See more [inside](spec/README.md).

## Files

### .jscsrc

[JSCS](http://jscs.info/) is a code style linter and formatter for your style guide. This file contains the preset and configuration selected for this project.

### .jshintrc

Another [code style linter](http://jshint.com/). Includes ES6 support.

### app.js

`app.js` is the starting point of the application. It loads everything and begins serving user requests.

### config.js

Simple template that allow us to configure several parameters for the GEris integration.

### Gruntfile.js

[Grunt task runner](http://gruntjs.com/) configuration file. It contains a set of tasks to automate and perform repetitive tasks like linting or testing.

### package.json

Provides information to [npm](https://www.npmjs.com/) that allows to identify the project as well as handle the project's dependencies.

### setup-test-env.sh

Script that launches a testing scenario and run the tests. In case of failure, it dumps the docker logs to the standard output.

### utils.js

`utils.js` contains a large set of functions to handle input and output data, as well as the logic needed.


text:
/**
 *
 */
package org.commcare.util;

/**
 * @author ctsims
 *
 */
public interface MultimediaListener {
    public void registerAudioTrigger(int index, String audioURI);
}


text:
namespace chromecast {
namespace metrics {

class ExternalMetrics;

class CastBrowserMetrics {
 public:
  explicit CastBrowserMetrics(
      std::unique_ptr<CastMetricsServiceClient> metrics_service_client);

  CastBrowserMetrics(const CastBrowserMetrics&) = delete;
  CastBrowserMetrics& operator=(const CastBrowserMetrics&) = delete;

  ~CastBrowserMetrics();
  void Initialize();
  void Finalize();

  // Processes all events from shared file. This should be used to consume all
  // events in the file before shutdown. This function is safe to call from any
  // thread.
  void ProcessExternalEvents(base::OnceClosure cb);
  void CollectFinalMetricsForLog(base::OnceClosure done_callback);

  metrics::CastMetricsServiceClient* metrics_service_client() const {
    return metrics_service_client_.get();
  }

 private:
  std::unique_ptr<CastMetricsServiceClient> metrics_service_client_;

#if defined(OS_LINUX) || defined(OS_CHROMEOS)
  ExternalMetrics* external_metrics_ = nullptr;
  ExternalMetrics* platform_metrics_ = nullptr;
#endif  // defined(OS_LINUX) || defined(OS_CHROMEOS)
};

}  // namespace metrics
}  // namespace chromecast

#endif  // CHROMECAST_BROWSER_METRICS_CAST_BROWSER_METRICS_H_


text:
<!--

Copyright (C) 2012-2014 Open Solutions for Java

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>org.oss4j.samples.groovy</groupId>
  <artifactId>all</artifactId>
  <version>1.0.0-SNAPSHOT</version>
  <packaging>pom</packaging>
  <name>OSS4J :: Samples :: Groovy ::  All</name>

  <inceptionYear>2014</inceptionYear>

  <url>https://github.com/oss4j/oss4j-groovy-samples</url>

  <scm>
    <url>scm:git:git@github.com:oss4j/oss4j-groovy-samples.git</url>
    <connection>scm:git:git@github.com:oss4j/oss4j-groovy-samples.git</connection>
    <developerConnection>scm:git:git@github.com:oss4j/oss4j-groovy-samples.git</developerConnection>
    <tag>HEAD</tag>
  </scm>

  <licenses>
    <license>
      <name>The Apache Software License, Version 2.0</name>
      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
      <distribution>repo</distribution>
    </license>
  </licenses>

  <organization>
    <name>Open Solutions for Java</name>
    <url>http://www.oss4j.org</url>
  </organization>

  <developers>
    <developer>
      <id>sobkowiak</id>
      <name>Krzysztof Sobkowiak</name>
      <email>krzys.sobkowiak@gmail.com</email>
      <organization>Capgemini</organization>
      <roles>
        <role>Developer</role>
        <role>Architect</role>
        <role>Team Lead</role>
      </roles>
      <timezone>UTC+1</timezone>
    </developer>
  </developers>

  <modules>
    <module>parent</module>
    <module>groovy-java-mix-sample</module>
    <module>groovy-java-mix-gmaven-sample</module>
  </modules>
</project>


text:
package com.mapswithme.maps.settings;

import android.os.Bundle;
import android.preference.PreferenceFragment;
import android.support.annotation.XmlRes;
import com.mapswithme.util.UiUtils;

abstract class BaseXmlSettingsFragment extends PreferenceFragment
{
  protected abstract @XmlRes int getXmlResources();

  @Override
  public void onCreate(Bundle savedInstanceState)
  {
    super.onCreate(savedInstanceState);
    addPreferencesFromResource(getXmlResources());
  }

  @Override
  public void onResume()
  {
    super.onResume();
    org.alohalytics.Statistics.logEvent("$onResume", getClass().getSimpleName() + ":" +
                                                     UiUtils.deviceOrientationAsString(getActivity()));
  }

  @Override
  public void onPause()
  {
    super.onPause();
    org.alohalytics.Statistics.logEvent("$onPause", getClass().getSimpleName() + ":" +
                                                    UiUtils.deviceOrientationAsString(getActivity()));
  }
}


text:
export default function($stateProvider: angular.ui.IStateProvider, $urlRouterProvider: angular.ui.IUrlRouterProvider) {
    'ngInject';
    $stateProvider.state('app.battery', {
        url: "/battery",
        views: {
            'content@app': {
                template: require("./index.html"),
                controller: "BatteryController as batteryCtrl"
            }
        }
    });
}


text:
package ec;
import ec.util.*;

/*
 * BreedingSource.java
 *
 * Created: Thu Nov 18 17:40:26 1999
 * By: Sean Luke
 */

/**
 * A BreedingSource is a Prototype which
 * provides Individuals to populate new populations based on
 * old ones.  The BreedingSource/BreedingPipeline/SelectionMethod mechanism
 * is inherently designed to work within single subpopulations, which is
 * by far the most common case.  If for some
 * reason you need to breed among different subpopulations to produce new ones
 * in a manner that can't be handled with exchanges, you will probably have to
 * write your own custom Breeder; you'd have to write your own custom breeding
 * pipelines anyway of course, though you can probably get away with reusing
 * the SelectionMethods.
 *
 *
A BreedingSource may have parent sources which feed it as well.
 * Some BreedingSources, SelectionMethods,
 * are meant solely to plug into other BreedingSources, BreedingPipelines.
 * BreedingPipelines can plug into other BreedingPipelines, and can also be
 * used to provide the final Individual meant to populate a new generation.
 *
 *
Think of BreedingSources as Streams of Individuals; at one end of the
 * stream is the provider, a SelectionMethod, which picks individuals from
 * the old population.  At the other end of the stream is a BreedingPipeline
 * which hands you the finished product, a small set of new Individuals
 * for you to use in populating your new population.

Parameters

base<tt>.prob</tt>

 0.0 <= float <= 1.0, or undefined
 (probability this BreedingSource gets chosen.  Undefined is only valid if the caller of this BreedingSource doesn't need a probability)

 * @author Sean Luke
 * @version 1.0
 */

public abstract class BreedingSource implements Prototype, RandomChoiceChooser
    {
    public static final String P_PROB = "prob";
    public static final float NO_PROBABILITY = -1.0f;

    /** The probability that this BreedingSource will be chosen
        to breed over other BreedingSources.  This may or may
        not be used, depending on what the caller to this BreedingSource is.
        It also might be modified by external sources owning this object,
        for their own purposes.  A BreedingSource should not use it for
        any purpose of its own, nor modify it except when setting it up.

The most common modification is to normalize it with some other
        set of probabilities, then set all of them up in increasing summation;
        this allows the use of the fast static BreedingSource-picking utility
        method, BreedingSource.pickRandom(...).  In order to use this method,
        for example, if four
        breeding source probabilities are {0.3, 0.2, 0.1, 0.4}, then
        they should get normalized and summed by the outside owners
        as: {0.3, 0.5, 0.6, 1.0}.
    */

    public float probability;

    /** Sets up the BreedingPipeline.  You can use state.output.error here
        because the top-level caller promises to call exitIfErrors() after calling
        setup.  Note that probability might get modified again by
        an external source if it doesn't normalize right.

The most common modification is to normalize it with some other
        set of probabilities, then set all of them up in increasing summation;
        this allows the use of the fast static BreedingSource-picking utility
        method, BreedingSource.pickRandom(...).  In order to use this method,
        for example, if four
        breeding source probabilities are {0.3, 0.2, 0.1, 0.4}, then
        they should get normalized and summed by the outside owners
        as: {0.3, 0.5, 0.6, 1.0}.

        @see Prototype#setup(EvolutionState,Parameter)
    */
    public void setup(final EvolutionState state, final Parameter base)
        {
        Parameter def = defaultBase();

        if (!state.parameters.exists(base.push(P_PROB),def.push(P_PROB)))
            probability = NO_PROBABILITY;
        else
            {
            probability = state.parameters.getFloat(base.push(P_PROB),def.push(P_PROB),0.0);
            if (probability<0.0) state.output.error("Breeding Source's probability must be a floating point value >= 0.0, or empty, which represents NO_PROBABILITY.",base.push(P_PROB),def.push(P_PROB));
            }
        }

    public final float getProbability(final Object obj)
        {
        return ((BreedingSource)obj).probability;
        }

    public final void setProbability(final Object obj, final float prob)
        {
        ((BreedingSource)obj).probability = prob;
        }

    /** Picks a random source from an array of sources, with their
        probabilities normalized and summed as follows:  For example,
        if four
        breeding source probabilities are {0.3, 0.2, 0.1, 0.4}, then
        they should get normalized and summed by the outside owners
        as: {0.3, 0.5, 0.6, 1.0}. */

    public static int pickRandom(final BreedingSource[] sources,final float prob)
        {
        return RandomChoice.pickFromDistribution(sources,sources[0], prob);
        }

    /** Normalizes and arranges the probabilities in sources so that they
        are usable by pickRandom(...).  If the sources have all zero probabilities,
        then a uniform selection is used.  Negative probabilities will
        generate an ArithmeticException, as will an empty source array. */
    public static void setupProbabilities(final BreedingSource[] sources)
        {
        RandomChoice.organizeDistribution(sources,sources[0],true);
        }

    /** Returns the "typical" number of individuals
        generated with one call of produce(...). */
    public abstract int typicalIndsProduced();

    /** Returns true if this BreedingSource, when attached to the given
        subpopulation, will produce individuals of the subpopulation's species.
        SelectionMethods should additionally make sure that their Fitnesses are
        of a valid type, if necessary. newpop *may* be the same as state.population
    */

    public abstract boolean produces(final EvolutionState state,
        final Population newpop,
        final int subpopulation,
        int thread);

    /** Called before produce(...), usually once a generation, or maybe only
        once if you're doing steady-state evolution, to let the breeding source
        "warm up" prior to producing.  Individuals should be produced from
        old individuals in positions [start...start+length] in the subpopulation
        only.  May be called again to reset the BreedingSource for a whole
        'nuther subpopulation. */

    public abstract void prepareToProduce(final EvolutionState state,
        final int subpopulation,
        final int thread);

    /** Called after produce(...), usually once a generation, or maybe only
        once if you're doing steady-state evolution (at the end of the run). */

    public abstract void finishProducing(final EvolutionState s,
        final int subpopulation,
        final int thread);

    /** Produces n individuals from the given subpopulation
        and puts them into inds[start...start+n-1],
        where n = Min(Max(q,min),max), where q is the "typical" number of
        individuals the BreedingSource produces in one shot, and returns
        n.  max must be >= min, and min must be >= 1. For example, crossover
        might typically produce two individuals, tournament selection might typically
        produce a single individual, etc. */
    public abstract int produce(final int min,
        final int max,
        final int start,
        final int subpopulation,
        final Individual[] inds,
        final EvolutionState state,
        final int thread) ;

    public Object clone()
        {
        try { return super.clone(); }
        catch (CloneNotSupportedException e)
            { throw new InternalError(); } // never happens
        }

    /** A hook which should be passed to all your subsidiary breeding
        sources.  The default does this for you already, so ordinarily
        you don't need to change anything.  If you are a BreedingPipeline and you
        implement your sources in a way different
        than using the sources[] array, be sure to override this method
        so that it calls preparePipeline(hook) on all of your sources.

ECJ at present does not custom-implement or call this method:
        it's available for you. Becuase it has custom functionality,
        this method might get called more than once, and by various objects
        as needed.  If you use it, you should determine somehow how to use
        it to send information under the assumption that it might be sent
        by nested items in the pipeline; you don't want to scribble over
        each other's calls! Note that this method should travel *all*
        breeding source paths regardless of whether or not it's redundant to
        do so. */
    public void preparePipeline(final Object hook)
        {
        // the default method does nothing, though BreedingPipelines override this
        // to guarantee that it's called on all their sources as well.
        }
    }


text:
namespace fs = std::experimental::filesystem;

int main() {
  using namespace fs;

  const file_status st(file_type::regular, perms::owner_read);

  // type test
  {
    static_assert(noexcept(st.type()),
                  "operation must be noexcept");
    static_assert(std::is_same<decltype(st.type()), file_type>::value,
                 "operation must return file_type");
    assert(st.type() == file_type::regular);
  }
  // permissions test
  {
    static_assert(noexcept(st.permissions()),
                  "operation must be noexcept");
    static_assert(std::is_same<decltype(st.permissions()), perms>::value,
                 "operation must return perms");
    assert(st.permissions() == perms::owner_read);
  }
}


text:
import stream = require('stream');

export interface DecoderOptions extends stream.TransformOptions {
    max_depth?: number;
    tags?: { [tag: number]: (v: any) => any };
    bigint?: boolean;
}

interface DecodeOptions {
    encoding?: string;
}

type decodeCallback = (error?: Error, value?: any) => void;
type decodeAllCallback = (error?: Error, value?: any[]) => void;

export class Decoder extends stream.Transform {
    constructor(options?: DecoderOptions);

    static nullcheck(val: any): any;
    static decodeFirstSync(input: string | Buffer, options?: DecodeOptions | string): any;
    static decodeAllSync(input: string | Buffer, options?: DecodeOptions | string): any[];

    static decodeFirst(input: string | Buffer, cb: decodeCallback): void;
    static decodeFirst(input: string | Buffer, options: DecodeOptions | string, cb: decodeCallback): void;
    static decodeFirst(input: string | Buffer, options?: DecodeOptions | string): Promise<any>;

    static decodeAll(input: string | Buffer, cb: decodeAllCallback): void;
    static decodeAll(input: string | Buffer, options: DecodeOptions | string, cb: decodeAllCallback): void;
    static decodeAll(input: string | Buffer, options?: DecodeOptions | string): Promise<any>;
}

export interface EncoderOptions extends stream.TransformOptions {
    genTypes?: any[];
    canonical?: boolean;
    detectLoops?: boolean | symbol;
    dateType?: 'number' | 'float' | 'int' | 'string';
    encodeUndefined?: any;
    disallowUndefinedKeys?: boolean;
}

export class Encoder extends stream.Transform {
    constructor(options?: EncoderOptions);
    addSemanticType<T>(
        type: new (...args: any[]) => T,
        encodeFunction: (encoder: Encoder, t: T) => boolean,
    ): (encoder: Encoder, t: T) => boolean | null;
    pushAny(input: any): boolean;
    removeLoopDetectors(obj: any): boolean;

    static encode(...objs: any[]): Buffer;
    static encodeCanonical(...objs: any[]): Buffer;
    static encodeOne(obj: any, options?: EncoderOptions): Buffer;
    static encodeAsync(obj: any, options?: EncoderOptions): Promise<Buffer>;
}

export class Simple {
    constructor(value: number);

    toString(): string;
    encodeCBOR(gen: Encoder): boolean;

    static isSimple(obj: any): boolean;
    static decode(
        val: number,
        has_parent?: boolean,
        parent_indefinite?: boolean,
    ): null | undefined | boolean | symbol | Simple;
}

export interface CommentedOptions extends stream.TransformOptions {
    max_depth?: number;
}

interface CommentOptions {
    max_depth?: number;
    encoding?: string;
}

type commentCallback = (error?: Error, commented?: string) => void;

export class Commented extends stream.Transform {
    constructor(options?: CommentedOptions);

    static comment(input: string | Buffer, cb: commentCallback): void;
    static comment(input: string | Buffer, options: CommentOptions | string, cb: commentCallback): void;
    static comment(input: string | Buffer, options?: CommentOptions | string): Promise<string>;
}

declare class CborMap extends Map {
    constructor(iterable: Iterable<[any, any]>);
    encodeCBOR(gen: Encoder): boolean;
}

export { CborMap as Map };

export class Tagged {
    constructor(tag: number, value?: any, err?: Error);

    toString(): string;
    encodeCBOR(gen: Encoder): boolean;
    convert(converters?: { [tag: number]: (val: any) => any }): any;
}

export interface DiagnoseOptions extends stream.TransformOptions {
    separator?: string;
    stream_errors?: boolean;
    max_depth?: number;
}

type diagnoseCallback = (error?: Error, str?: string) => void;
export class Diagnose extends stream.Transform {
    constructor(options?: DiagnoseOptions);

    static diagnose(input: string | Buffer, encoding?: string): Promise<string>;
    static diagnose(input: string | Buffer, cb: diagnoseCallback): void;
}

export const decode: typeof Decoder.decodeFirstSync;
export const decodeFirstSync: typeof Decoder.decodeFirstSync;
export const decodeAllSync: typeof Decoder.decodeAllSync;
export const decodeFirst: typeof Decoder.decodeFirst;
export const decodeAll: typeof Decoder.decodeAll;

export const encode: typeof Encoder.encode;
export const encodeCanonical: typeof Encoder.encodeCanonical;
export const encodeOne: typeof Encoder.encodeOne;
export const encodeAsync: typeof Encoder.encodeAsync;

export const comment: typeof Commented.comment;

export namespace leveldb {
    const decode: typeof Decoder.decodeAllSync;
    const encode: typeof Encoder.encode;
    const buffer: true;
    const name: 'cbor';
}


text:
package party.lemons.anima.content.block;

import net.minecraft.block.material.Material;
import net.minecraft.tileentity.TileEntity;
import net.minecraft.world.World;
import party.lemons.anima.content.block.tileentity.TileEntitySucker;

import javax.annotation.Nullable;

/**
 * Created by Sam on 21/06/2017.
 */
public class BlockSucker extends BlockAnima implements ILinkable
{
	protected BlockSucker()
	{
		super("sucker", Material.WOOD);
	}

	@Nullable
	@Override
	public TileEntity createNewTileEntity(World worldIn, int meta)
	{
		return new TileEntitySucker();
	}
}


text:
package org.apache.reef.io.storage;

import org.apache.reef.exception.evaluator.ServiceException;
import org.apache.reef.exception.evaluator.ServiceRuntimeException;
import org.apache.reef.io.Tuple;
import org.apache.reef.io.serialization.Deserializer;

import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.NoSuchElementException;

public class FramingTupleDeserializer<K, V> implements
    Deserializer<Tuple<K, V>, InputStream> {

  private final Deserializer<K, InputStream> keyDeserializer;
  private final Deserializer<V, InputStream> valDeserializer;

  public FramingTupleDeserializer(Deserializer<K, InputStream> keyDeserializer,
                                  Deserializer<V, InputStream> valDeserializer) {
    this.keyDeserializer = keyDeserializer;
    this.valDeserializer = valDeserializer;
  }

  @Override
  public Iterable<Tuple<K, V>> create(InputStream ins) {
    final DataInputStream in = new DataInputStream(ins);
    final Iterable<K> keyItbl = keyDeserializer.create(in);
    final Iterable<V> valItbl = valDeserializer.create(in);
    return new Iterable<Tuple<K, V>>() {
      @Override
      public Iterator<Tuple<K, V>> iterator() {
        final Iterator<K> keyIt = keyItbl.iterator();
        final Iterator<V> valIt = valItbl.iterator();
        try {
          return new Iterator<Tuple<K, V>>() {

            private int readFrameLength() throws ServiceException {
              try {
                return in.readInt();
              } catch (IOException e) {
                throw new ServiceRuntimeException(e);
              }
            }

            int nextFrameLength = readFrameLength();

            @Override
            public boolean hasNext() {
              return nextFrameLength != -1;
            }

            @Override
            public Tuple<K, V> next() {
              try {
                if (nextFrameLength == -1) {
                  throw new NoSuchElementException();
                }
                K k = keyIt.next();
                readFrameLength();
                V v = valIt.next();
                nextFrameLength = readFrameLength();
                return new Tuple<>(k, v);
              } catch (ServiceException e) {
                throw new ServiceRuntimeException(e);
              }
            }

            @Override
            public void remove() {
              throw new UnsupportedOperationException();
            }
          };
        } catch (ServiceException e) {
          throw new ServiceRuntimeException(e);
        }
      }
    };
  }

}


text:
package org.sell2me.application.configuration;

import org.sell2me.application.utils.AppConstants;
import org.sell2me.application.utils.TokenUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
import org.springframework.security.web.authentication.WebAuthenticationDetailsSource;
import org.springframework.web.context.support.WebApplicationContextUtils;

import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;

public class AuthenticationTokenFilter extends UsernamePasswordAuthenticationFilter {

    @Autowired
    private TokenUtils tokenUtils;

    @Autowired
    private UserDetailsService userDetailsService;

    @Override
    public void doFilter(ServletRequest request,
                         ServletResponse response,
                         FilterChain chain) throws IOException, ServletException
    {
        tokenUtils = WebApplicationContextUtils
                .getRequiredWebApplicationContext(this.getServletContext())
                .getBean(TokenUtils.class);
        userDetailsService = WebApplicationContextUtils
                .getRequiredWebApplicationContext(this.getServletContext())
                .getBean(UserDetailsService.class);

        HttpServletResponse resp = (HttpServletResponse) response;
        resp.setHeader("Access-Control-Allow-Origin", "*");
        resp.setHeader("Access-Control-Allow-Methods", "POST, GET, PUT, OPTIONS, DELETE, PATCH");
        resp.setHeader("Access-Control-Max-Age", "3600");
        resp.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, " + AppConstants.tokenHeader);

        HttpServletRequest httpRequest = (HttpServletRequest) request;
        String authToken = httpRequest.getHeader(AppConstants.tokenHeader);
        String username = this.tokenUtils.getUsernameFromToken(authToken);

        if (username != null && SecurityContextHolder.getContext().getAuthentication() == null) {
            UserDetails userDetails = this.userDetailsService.loadUserByUsername(username);
            if (this.tokenUtils.validateToken(authToken, userDetails)) {
                UsernamePasswordAuthenticationToken authentication =
                        new UsernamePasswordAuthenticationToken(userDetails, null, userDetails.getAuthorities());
                authentication.setDetails(new WebAuthenticationDetailsSource().buildDetails(httpRequest));
                SecurityContextHolder.getContext().setAuthentication(authentication);
            }
        }

        chain.doFilter(request, response);
    }
}


text:
@interface LoginViewController () <UITextFieldDelegate>
{
    UITextField *_userNameText;
    UITextField *_passWordText;
    UIButton    *_loginBtn;

    LoginedViewController *_loginedVC;
}

@end

@implementation LoginViewController

- (void)viewDidLoad {

    [super viewDidLoad];

    self.title = @"首页";

    _userNameText = [self addTextField:100 leftViewName:@"账号:"];
    _passWordText = [self addTextField:CGRectGetMaxY(_userNameText.frame) + 20 leftViewName:@"密码:"];

    _loginBtn = [self addButtonWithRect:CGRectMake((SWidth - 100) * 0.5, CGRectGetMaxY(_passWordText.frame) + 50, 100, 40) title:@"登录"];
}

- (UITextField *)addTextField:(CGFloat)y leftViewName:(NSString *)name {

    UITextField *textField=[[UITextField alloc]initWithFrame:CGRectMake(20, y, SWidth - 40, 40)];
    textField.layer.borderWidth = 1.0f;
    [self.view addSubview:textField];
    UILabel *label = [[UILabel alloc] initWithFrame:CGRectMake(0, 0, 50, 40)];
    label.text = name;
    label.textAlignment = NSTextAlignmentCenter;
    label.font = [UIFont systemFontOfSize:14];
    label.adjustsFontSizeToFitWidth = YES;
    textField.leftView = label;
    textField.delegate = self;
    textField.leftViewMode = UITextFieldViewModeAlways;
    return textField;
}

- (UIButton *)addButtonWithRect:(CGRect)frame title:(NSString *)name {

    UIButton *button = [UIButton buttonWithType:UIButtonTypeRoundedRect];
    [button setFrame:frame];
    [button setTitle:name forState:UIControlStateNormal];
    [button setTitleColor:[UIColor blackColor] forState:UIControlStateNormal];
    [button addTarget:self action:@selector(buttonAction:) forControlEvents:UIControlEventTouchUpInside];
    [self.view addSubview:button];
    return button;
}

- (void)buttonAction:(UIButton *)button {

    [self textResignFirstResponder];

    if (!_loginedVC) {
        _loginedVC = [[LoginedViewController alloc] init];
        [_loginedVC addObserver:self forKeyPath:@"userName" options:NSKeyValueObservingOptionNew context:nil];
        [_loginedVC addObserver:self forKeyPath:@"passWord" options:NSKeyValueObservingOptionNew context:nil];
    }
    [self.navigationController pushViewController:_loginedVC animated:YES];
}

- (void)textResignFirstResponder {

    [_userNameText resignFirstResponder];
    [_passWordText resignFirstResponder];
}

- (void)touchesBegan:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {

    [self textResignFirstResponder];
}

- (void)observeValueForKeyPath:(NSString *)keyPath ofObject:(id)object change:(NSDictionary<NSKeyValueChangeKey,id> *)change context:(void *)context {

    if ([keyPath isEqualToString:@"userName"]) {
        _userNameText.text = change[NSKeyValueChangeNewKey];
    }
    if ([keyPath isEqualToString:@"passWord"]) {
        _passWordText.text = change[NSKeyValueChangeNewKey];
    }
}

@end