text:
.class public Landroid/app/ComposedIconInfo;
.super Ljava/lang/Object;
.source "ComposedIconInfo.java"

# interfaces
.implements Landroid/os/Parcelable;

# annotations
.annotation system Ldalvik/annotation/MemberClasses;
    value = {
        Landroid/app/ComposedIconInfo$SwatchType;
    }
.end annotation

# static fields
.field public static final CREATOR:Landroid/os/Parcelable$Creator;
    .annotation system Ldalvik/annotation/Signature;
        value = {
            "Landroid/os/Parcelable$Creator",
            "<",
            "Landroid/app/ComposedIconInfo;",
            ">;"
        }
    .end annotation
.end field

# instance fields
.field public colorFilter:[F

.field public defaultSwatchColors:[I

.field public iconBacks:[I

.field public iconDensity:I

.field public iconMask:I

.field public iconPaletteBack:I

.field public iconRotation:F

.field public iconRotationVariance:F

.field public iconScale:F

.field public iconSize:I

.field public iconTranslationX:F

.field public iconTranslationY:F

.field public iconUpon:I

.field public swatchType:Landroid/app/ComposedIconInfo$SwatchType;

# direct methods
.method static constructor <clinit>()V
    .locals 1

    .prologue
    .line 127
    new-instance v0, Landroid/app/ComposedIconInfo$1;

    invoke-direct {v0}, Landroid/app/ComposedIconInfo$1;-><init>()V

    sput-object v0, Landroid/app/ComposedIconInfo;->CREATOR:Landroid/os/Parcelable$Creator;

    return-void
.end method

.method public constructor <init>()V
    .locals 2

    .prologue
    const/4 v1, 0x0

    .line 41
    invoke-direct {p0}, Ljava/lang/Object;-><init>()V

    .line 42
    const/4 v0, 0x0

    iput v0, p0, Landroid/app/ComposedIconInfo;->iconPaletteBack:I

    .line 43
    sget-object v0, Landroid/app/ComposedIconInfo$SwatchType;->None:Landroid/app/ComposedIconInfo$SwatchType;

    iput-object v0, p0, Landroid/app/ComposedIconInfo;->swatchType:Landroid/app/ComposedIconInfo$SwatchType;

    .line 44
    iput v1, p0, Landroid/app/ComposedIconInfo;->iconRotation:F

    .line 45
    iput v1, p0, Landroid/app/ComposedIconInfo;->iconTranslationX:F

    .line 46
    iput v1, p0, Landroid/app/ComposedIconInfo;->iconTranslationY:F

    .line 47
    const/high16 v0, 0x3f800000    # 1.0f

    iput v0, p0, Landroid/app/ComposedIconInfo;->iconScale:F

    .line 48
    return-void
.end method

.method private constructor <init>(Landroid/os/Parcel;)V
    .locals 6
    .param p1, "source"    # Landroid/os/Parcel;

    .prologue
    .line 50
    invoke-direct {p0}, Ljava/lang/Object;-><init>()V

    .line 51
    invoke-virtual {p1}, Landroid/os/Parcel;->readFloat()F

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconScale:F

    .line 52
    invoke-virtual {p1}, Landroid/os/Parcel;->readFloat()F

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconRotation:F

    .line 53
    invoke-virtual {p1}, Landroid/os/Parcel;->readFloat()F

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconRotationVariance:F

    .line 54
    invoke-virtual {p1}, Landroid/os/Parcel;->readFloat()F

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconTranslationX:F

    .line 55
    invoke-virtual {p1}, Landroid/os/Parcel;->readFloat()F

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconTranslationY:F

    .line 56
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconDensity:I

    .line 57
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconSize:I

    .line 58
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v0

    .line 59
    .local v0, "backCount":I
    if-lez v0, :cond_0

    .line 60
    new-array v4, v0, [I

    iput-object v4, p0, Landroid/app/ComposedIconInfo;->iconBacks:[I

    .line 61
    const/4 v2, 0x0

    .local v2, "i":I
    :goto_0
    if-ge v2, v0, :cond_0

    .line 62
    iget-object v4, p0, Landroid/app/ComposedIconInfo;->iconBacks:[I

    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v5

    aput v5, v4, v2

    .line 61
    add-int/lit8 v2, v2, 0x1

    goto :goto_0

    .line 65
    .end local v2    # "i":I
    :cond_0
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconMask:I

    .line 66
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconUpon:I

    .line 67
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v1

    .line 68
    .local v1, "colorFilterSize":I
    if-lez v1, :cond_1

    .line 69
    new-array v4, v1, [F

    iput-object v4, p0, Landroid/app/ComposedIconInfo;->colorFilter:[F

    .line 70
    const/4 v2, 0x0

    .restart local v2    # "i":I
    :goto_1
    if-ge v2, v1, :cond_1

    .line 71
    iget-object v4, p0, Landroid/app/ComposedIconInfo;->colorFilter:[F

    invoke-virtual {p1}, Landroid/os/Parcel;->readFloat()F

    move-result v5

    aput v5, v4, v2

    .line 70
    add-int/lit8 v2, v2, 0x1

    goto :goto_1

    .line 74
    .end local v2    # "i":I
    :cond_1
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v4

    iput v4, p0, Landroid/app/ComposedIconInfo;->iconPaletteBack:I

    .line 75
    invoke-static {}, Landroid/app/ComposedIconInfo$SwatchType;->values()[Landroid/app/ComposedIconInfo$SwatchType;

    move-result-object v4

    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v5

    aget-object v4, v4, v5

    iput-object v4, p0, Landroid/app/ComposedIconInfo;->swatchType:Landroid/app/ComposedIconInfo$SwatchType;

    .line 76
    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v3

    .line 77
    .local v3, "numDefaultColors":I
    if-lez v3, :cond_2

    .line 78
    new-array v4, v3, [I

    iput-object v4, p0, Landroid/app/ComposedIconInfo;->defaultSwatchColors:[I

    .line 79
    const/4 v2, 0x0

    .restart local v2    # "i":I
    :goto_2
    if-ge v2, v3, :cond_2

    .line 80
    iget-object v4, p0, Landroid/app/ComposedIconInfo;->defaultSwatchColors:[I

    invoke-virtual {p1}, Landroid/os/Parcel;->readInt()I

    move-result v5

    aput v5, v4, v2

    .line 79
    add-int/lit8 v2, v2, 0x1

    goto :goto_2

    .line 83
    .end local v2    # "i":I
    :cond_2
    return-void
.end method

.method synthetic constructor <init>(Landroid/os/Parcel;Landroid/app/ComposedIconInfo$1;)V
    .locals 0
    .param p1, "x0"    # Landroid/os/Parcel;
    .param p2, "x1"    # Landroid/app/ComposedIconInfo$1;

    .prologue
    .line 22
    invoke-direct {p0, p1}, Landroid/app/ComposedIconInfo;-><init>(Landroid/os/Parcel;)V

    return-void
.end method

# virtual methods
.method public describeContents()I
    .locals 1

    .prologue
    .line 87
    const/4 v0, 0x0

    return v0
.end method

.method public writeToParcel(Landroid/os/Parcel;I)V
    .locals 8
    .param p1, "dest"    # Landroid/os/Parcel;
    .param p2, "flags"    # I

    .prologue
    const/4 v7, 0x0

    .line 92
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconScale:F

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeFloat(F)V

    .line 93
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconRotation:F

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeFloat(F)V

    .line 94
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconRotationVariance:F

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeFloat(F)V

    .line 95
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconTranslationX:F

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeFloat(F)V

    .line 96
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconTranslationY:F

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeFloat(F)V

    .line 97
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconDensity:I

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 98
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconSize:I

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 99
    iget-object v6, p0, Landroid/app/ComposedIconInfo;->iconBacks:[I

    if-eqz v6, :cond_0

    iget-object v6, p0, Landroid/app/ComposedIconInfo;->iconBacks:[I

    array-length v6, v6

    :goto_0
    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 100
    iget-object v6, p0, Landroid/app/ComposedIconInfo;->iconBacks:[I

    if-eqz v6, :cond_1

    .line 101
    iget-object v0, p0, Landroid/app/ComposedIconInfo;->iconBacks:[I

    .local v0, "arr$":[I
    array-length v3, v0

    .local v3, "len$":I
    const/4 v2, 0x0

    .local v2, "i$":I
    :goto_1
    if-ge v2, v3, :cond_1

    aget v4, v0, v2

    .line 102
    .local v4, "resId":I
    invoke-virtual {p1, v4}, Landroid/os/Parcel;->writeInt(I)V

    .line 101
    add-int/lit8 v2, v2, 0x1

    goto :goto_1

    .end local v0    # "arr$":[I
    .end local v2    # "i$":I
    .end local v3    # "len$":I
    .end local v4    # "resId":I
    :cond_0
    move v6, v7

    .line 99
    goto :goto_0

    .line 105
    :cond_1
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconMask:I

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 106
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconUpon:I

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 107
    iget-object v6, p0, Landroid/app/ComposedIconInfo;->colorFilter:[F

    if-eqz v6, :cond_2

    .line 108
    iget-object v6, p0, Landroid/app/ComposedIconInfo;->colorFilter:[F

    array-length v6, v6

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 109
    iget-object v0, p0, Landroid/app/ComposedIconInfo;->colorFilter:[F

    .local v0, "arr$":[F
    array-length v3, v0

    .restart local v3    # "len$":I
    const/4 v2, 0x0

    .restart local v2    # "i$":I
    :goto_2
    if-ge v2, v3, :cond_3

    aget v5, v0, v2

    .line 110
    .local v5, "val":F
    invoke-virtual {p1, v5}, Landroid/os/Parcel;->writeFloat(F)V

    .line 109
    add-int/lit8 v2, v2, 0x1

    goto :goto_2

    .line 113
    .end local v0    # "arr$":[F
    .end local v2    # "i$":I
    .end local v3    # "len$":I
    .end local v5    # "val":F
    :cond_2
    invoke-virtual {p1, v7}, Landroid/os/Parcel;->writeInt(I)V

    .line 115
    :cond_3
    iget v6, p0, Landroid/app/ComposedIconInfo;->iconPaletteBack:I

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 116
    iget-object v6, p0, Landroid/app/ComposedIconInfo;->swatchType:Landroid/app/ComposedIconInfo$SwatchType;

    invoke-virtual {v6}, Landroid/app/ComposedIconInfo$SwatchType;->ordinal()I

    move-result v6

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 117
    iget-object v6, p0, Landroid/app/ComposedIconInfo;->defaultSwatchColors:[I

    if-eqz v6, :cond_4

    .line 118
    iget-object v6, p0, Landroid/app/ComposedIconInfo;->defaultSwatchColors:[I

    array-length v6, v6

    invoke-virtual {p1, v6}, Landroid/os/Parcel;->writeInt(I)V

    .line 119
    iget-object v0, p0, Landroid/app/ComposedIconInfo;->defaultSwatchColors:[I

    .local v0, "arr$":[I
    array-length v3, v0

    .restart local v3    # "len$":I
    const/4 v2, 0x0

    .restart local v2    # "i$":I
    :goto_3
    if-ge v2, v3, :cond_5

    aget v1, v0, v2

    .line 120
    .local v1, "color":I
    invoke-virtual {p1, v1}, Landroid/os/Parcel;->writeInt(I)V

    .line 119
    add-int/lit8 v2, v2, 0x1

    goto :goto_3

    .line 123
    .end local v0    # "arr$":[I
    .end local v1    # "color":I
    .end local v2    # "i$":I
    .end local v3    # "len$":I
    :cond_4
    invoke-virtual {p1, v7}, Landroid/os/Parcel;->writeInt(I)V

    .line 125
    :cond_5
    return-void
.end method


text:
from collections import OrderedDict
import argparse
import os
import json
import ray
import time
import timeit

import torch.optim as optim
import numpy as np
import torch
import horovod.torch as hvd
from horovod.ray import RayExecutor

from ray_shuffling_data_loader.data_generation import DATA_SPEC
from ray_shuffling_data_loader.embedding_model import MyModel, annotation, huber_loss
from ray.data.dataset_pipeline import DatasetPipeline

# Training settings
parser = argparse.ArgumentParser(description="Dataset ingestion Example")
parser.add_argument(
    "--batch-size",
    type=int,
    default=250000,
    metavar="N",
    help="input batch size for training (default: 64)",
)
parser.add_argument(
    "--epochs",
    type=int,
    default=10,
    metavar="N",
    help="number of epochs to train (default: 10)",
)
parser.add_argument("--debug", action="store_true", default=False, help="disables hvd")
parser.add_argument(
    "--seed", type=int, default=42, metavar="S", help="random seed (default: 42)"
)
parser.add_argument(
    "--log-interval",
    type=int,
    default=10,
    metavar="N",
    help=("how many batches to wait before logging training " "status"),
)
parser.add_argument("--num-workers", type=int, default=16)
parser.add_argument("--num-files", type=int, default=30)
parser.add_argument("--num-windows", type=int, default=1)
parser.add_argument("--manual-windows", type=bool, default=False)
parser.add_argument("--parallelism", type=int, default=400)

SIZE_50_G = 30  # 49.17GB
SIZE_100_G = 62  # 101.62GB
SIZE_500_G = 305  # 499.93GB
SIZE_1500_G = 915

def construct_optimizers(model):
    sparse_params = []
    dense_params = []
    for k, v in model.named_parameters():
        if "input.embeddings.embeddings" in k:
            sparse_params.append((k, v))
        else:
            dense_params.append((k, v))

    optimizers = []
    if len(dense_params) > 0:
        opt = optim.Adam([v for _, v in dense_params], lr=0.001)
        opt = hvd.DistributedOptimizer(opt, dense_params)
        optimizers.append(opt)
    if len(sparse_params) > 0:
        opt = optim.SparseAdam([v for _, v in sparse_params], lr=0.001)
        opt = hvd.DistributedOptimizer(opt, sparse_params)
        optimizers.append(opt)

    if hvd.rank() == 0:
        print(optimizers)

    return optimizers

def train_main(args, splits):
    # Horovod: initialize library.
    hvd.init()
    torch.manual_seed(args.seed)

    if torch.cuda.is_available():
        # Horovod: pin GPU to local rank.
        torch.cuda.set_device(hvd.local_rank())
        torch.cuda.manual_seed(args.seed)

    # Horovod: limit # of CPU threads to be used per worker.
    torch.set_num_threads(1)
    rank = hvd.rank()

    model = MyModel(annotation, use_bn=False)
    # By default, Adasum doesn"t need scaling up learning rate.
    if torch.cuda.is_available():
        # Move model to GPU.
        model.cuda()

    optimizers = construct_optimizers(model)
    loss_function = huber_loss
    # Horovod: broadcast parameters & optimizer state.
    hvd.broadcast_parameters(model.state_dict(), root_rank=0)
    for opt in optimizers:
        hvd.broadcast_optimizer_state(opt, root_rank=0)

    def _train(epoch, train_dataset):
        model.train()
        # Horovod: set epoch to sampler for shuffling.
        # train_dataset.set_epoch(epoch)
        start_epoch = timeit.default_timer()
        last_batch_time = start_epoch
        batch_wait_times = []
        for batch_idx, (data, target) in enumerate(train_dataset):
            batch_wait_times.append(timeit.default_timer() - last_batch_time)
            if torch.cuda.is_available():
                data = data.cuda()
                target = target.cuda()
            for opt in optimizers:
                opt.zero_grad()
            batch = OrderedDict()
            batch["embeddings"] = OrderedDict()
            batch["one_hot"] = OrderedDict()
            for i, name in enumerate(annotation["embeddings"]):
                batch["embeddings"][name] = data[:, i : i + 1]
            batch["one_hot"]["hot0"] = data[:, -2:-1]
            batch["one_hot"]["hot1"] = data[:, -1:]

            batch_pred = model(batch)

            if batch_idx % args.log_interval == 0:
                print(
                    f"Processing batch {batch_idx} in epoch {epoch} on worker "
                    f"{rank}."
                )
            loss = loss_function(batch_pred, target, delta=60)
            loss.mean().backward()
            for opt in optimizers:
                opt.step()

            last_batch_time = timeit.default_timer()
        epoch_duration = timeit.default_timer() - start_epoch
        avg_batch_wait_time = np.mean(batch_wait_times)
        std_batch_wait_time = np.std(batch_wait_times)
        max_batch_wait_time = np.max(batch_wait_times)
        min_batch_wait_time = np.min(batch_wait_times)
        print(
            f"\nEpoch {epoch}, worker {rank} stats over "
            f"{len(batch_wait_times)} steps: {epoch_duration:.3f}"
        )
        print(
            f"Mean batch wait time: {avg_batch_wait_time:.3f}s +- "
            f"{std_batch_wait_time}"
        )
        print(f"Max batch wait time: {max_batch_wait_time:.3f}s")
        print(f"Min batch wait time: {min_batch_wait_time:.3f}s")
        return batch_wait_times

    print(f"Starting training on worker {rank}.")
    batch_wait_times = []
    for epoch, split_ds in enumerate(splits[rank].iter_epochs()):
        train_dataset = create_torch_iterator(split_ds, args.batch_size, rank)
        new_batch_times = _train(epoch, train_dataset)
        new_batch_times.pop(0)
        batch_wait_times.extend(new_batch_times)
    print(f"Done training on worker {rank}.")
    avg_batch_wait_time = np.mean(batch_wait_times)
    std_batch_wait_time = np.std(batch_wait_times)
    max_batch_wait_time = np.max(batch_wait_times)
    min_batch_wait_time = np.min(batch_wait_times)
    print(f"\nWorker {rank} training stats over {args.epochs} epochs:")
    print(
        f"Mean batch wait time: {avg_batch_wait_time:.3f}s +- " f"{std_batch_wait_time}"
    )
    print(f"Max batch wait time: {max_batch_wait_time:.3f}s")
    print(f"Min batch wait time: {min_batch_wait_time:.3f}s")

######################################################

numpy_to_torch_dtype = {
    np.bool: torch.bool,
    np.uint8: torch.uint8,
    np.int8: torch.int8,
    np.int16: torch.int16,
    np.int32: torch.int32,
    np.int64: torch.int64,
    np.float16: torch.float16,
    np.float32: torch.float32,
    np.float64: torch.float64,
    np.complex64: torch.complex64,
    np.complex128: torch.complex128,
}

def create_torch_iterator(split, batch_size, rank=None):
    print(
        f"Creating Torch shuffling dataset for worker {rank} with "
        f"{batch_size} batch size."
    )
    feature_columns = list(DATA_SPEC.keys())
    feature_types = [numpy_to_torch_dtype[dtype] for _, _, dtype in DATA_SPEC.values()]
    label_column = feature_columns.pop()
    label_type = feature_types.pop()

    torch_iterator = split.to_torch(
        label_column=label_column,
        feature_columns=feature_columns,
        label_column_dtype=label_type,
        feature_column_dtypes=feature_types[0],
        batch_size=batch_size,
        # prefetch_blocks: int = 0,
        # drop_last: bool = False
    )
    return torch_iterator

def create_dataset(
    files,
    num_workers=4,
    epochs=50,
    num_windows=1,
    manual_windowing=False,
    parallelism=400,
):
    if num_windows > 1 and manual_windowing:
        num_rows = ray.data.read_parquet(
            files
        ).count()  # This should only read Parquet metadata.
        file_splits = np.array_split(files, num_windows)

        class Windower:
            def __init__(self):
                self.i = 0
                self.iterations = epochs * num_windows

            def __iter__(self):
                return self

            def __next__(self):
                if self.i >= self.iterations:
                    raise StopIteration()
                split = file_splits[self.i % num_windows]
                self.i += 1
                return lambda: ray.data.read_parquet(list(split))

        pipe = DatasetPipeline.from_iterable(Windower())
        split_indices = [
            i * num_rows // num_windows // num_workers for i in range(1, num_workers)
        ]
        pipe = pipe.random_shuffle_each_window()
        pipe_shards = pipe.split_at_indices(split_indices)
    else:
        ds = ray.data.read_parquet(files, parallelism=parallelism)
        if num_windows > 1:
            window_size = max(ds.num_blocks() // num_windows, 1)
            ds = ds.window(blocks_per_window=window_size)
        pipe = ds.repeat(epochs)
        pipe = pipe.random_shuffle_each_window()
        pipe_shards = pipe.split(num_workers, equal=True)
    return pipe_shards

@ray.remote
def consume(split, rank=None, batch_size=None):
    torch_iterator = create_torch_iterator(split, batch_size=batch_size, rank=rank)
    start = time.perf_counter()
    batch_start = start
    batch_wait_time = []
    num_batches = 0
    for i, (x, y) in enumerate(torch_iterator):
        num_batches += 1
        batch_wait = time.perf_counter() - batch_start
        batch_wait_time.append(batch_wait)
        if i % 10 == 0:
            print(f"Consumer #{rank} finishes batch #{i}")
        batch_start = time.perf_counter()

    duration = time.perf_counter() - start
    t50 = np.quantile(batch_wait_time, 0.5)
    t95 = np.quantile(batch_wait_time, 0.95)
    tmax = np.max(batch_wait_time)
    print(
        f"Consumer #{rank} total time: {duration}, total batches: {num_batches}, "
        f"P50/P95/Max batch wait time (s): {t50}/{t95}/{tmax}."
    )

    return

if __name__ == "__main__":
    args = parser.parse_args()
    import ray

    print("Connecting to Ray cluster...")
    ray.init(address="auto")

    num = args.num_files

    files = [
        f"s3://shuffling-data-loader-benchmarks/data/r10_000_000_000-f1000"
        f"/input_data_{i}.parquet.snappy"
        for i in range(args.num_files)
    ]

    start = time.time()

    splits = create_dataset(
        files,
        num_workers=args.num_workers,
        epochs=args.epochs,
        num_windows=args.num_windows,
        manual_windowing=args.manual_windows,
        parallelism=args.parallelism,
    )

    if args.debug:
        tasks = [
            consume.options(num_gpus=1, num_cpus=0).remote(
                split, rank=idx, batch_size=args.batch_size
            )
            for idx, split in enumerate(splits)
        ]
        ray.get(tasks)
    else:
        print("Create Ray executor")
        settings = RayExecutor.create_settings(timeout_s=30)
        executor = RayExecutor(settings, num_workers=args.num_workers, use_gpu=True)
        executor.start()
        executor.run(train_main, args=[args, splits])
        executor.shutdown()

    delta = time.time() - start
    print(f"success! total time {delta}")
    with open(os.environ["TEST_OUTPUT_JSON"], "w") as f:
        f.write(json.dumps({"ingest_time": delta, "success": 1}))

redpajama_set_name: RedPajamaGithub


text:
package se.sitic.megatron.core;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;

import se.sitic.megatron.db.DbManager;
import se.sitic.megatron.entity.LogEntry;
import se.sitic.megatron.entity.MailJob;
import se.sitic.megatron.entity.Organization;
import se.sitic.megatron.mail.MailSender;
import se.sitic.megatron.parser.LogEntryMapper;
import se.sitic.megatron.util.AppUtil;
import se.sitic.megatron.util.Constants;
import se.sitic.megatron.util.IpAddressUtil;
import se.sitic.megatron.util.SqlUtil;
import se.sitic.megatron.util.StringUtil;

/**
 * Sends mail from log entries in the database.
 */
public class MailExporter extends AbstractExporter {
    private static final Logger log = Logger.getLogger(MailExporter.class);

    private static final String JOB_INFO_HEADER = "---- Job Info ----" + Constants.LINE_BREAK;
    private static final String MESSAGE_HEADER = "---- Sample Message ----" + Constants.LINE_BREAK;
    private static final String RECIPIENTS_HEADER = "---- Recipients ----" + Constants.LINE_BREAK;
    private static final String SUMMARY_SUBJECT_TEMPLATE = "Megatron: Mail job #$mailJobId with RTIR ID #$rtirId finished";
    private static final String JOB_INFO_TEMPLATE =
        JOB_INFO_HEADER +
        "No. of Sent Emails: $noOfSentMails" + Constants.LINE_BREAK +
        "No. of Sent Log Entries: $noOfSentLogEntries" + Constants.LINE_BREAK +
        "No. of Quarantine Log Entries: $noOfQuarantineLogEntries" + Constants.LINE_BREAK +
        "Mail Job ID: $mailJobId" + Constants.LINE_BREAK +
        "Mail Job Started: $mailJobStarted" + Constants.LINE_BREAK +
        "Job Name: $jobName" + Constants.LINE_BREAK +
        "Filename: $jobFilename" + Constants.LINE_BREAK +
        "RTIR ID: $rtirId" + Constants.LINE_BREAK +
        "Use Secondary Org.: $useOrg2" + Constants.LINE_BREAK +
        "Comment:" + Constants.LINE_BREAK + "$mailJobComment" + Constants.LINE_BREAK + Constants.LINE_BREAK;

    // Variables in subject and body
    private static final String MAIL_JOB_ID = "mailJobId";
    private static final String MAIL_JOB_STARTED = "mailJobStarted";
    private static final String MAIL_JOB_COMMENT = "mailJobComment";
    private static final String JOB_STARTED = "jobStarted";
    private static final String JOB_NAME = "jobName";
    private static final String JOB_FILENAME = "jobFilename";
    private static final String RTIR_ID = "rtirId";
    private static final String ORGANIZATION_NAME = "organizationName";
    private static final String EMAIL_ADDRESSES = "emailAddresses";

    private MailJobContext mailJobContext;
    private MailJob mailJob;
    private DbManager dbManager;
    private MailSender mailSender;
    private StringBuilder summaryBody;
    private Map<String, String> globalAttributeMap;

    /**
     * Constructor.
     */
    public MailExporter(MailJobContext mailJobContext) {
        super(mailJobContext);
        this.mailJobContext = mailJobContext;
        this.props = mailJobContext.getProps();
        this.mailJob = mailJobContext.getMailJob();
        init();
    }

    /**
     * Sends one mail with specified log entries. Note: All log entries must
     * belong to the same organization.
     */
    public void sendMail(List<LogEntry> logEntries) throws MegatronException {
        // -- check params
        if (logEntries.size() == 0) {
            String msg = "Assertion error: List of log entries is empty.";
            throw new MegatronException(msg);
        }
        Organization organization = null;
        Organization prevOrganization = null;
        for (Iterator<LogEntry> iterator = logEntries.iterator(); iterator.hasNext(); ) {
            organization = getOrganization(iterator.next());
            if ((prevOrganization != null) && !prevOrganization.equals(organization)) {
                String msg = "Assertion error: All log entries in the same mail must belong to the same organization.";
                throw new MegatronException(msg);
            }
        }

        // -- exists email addresses? no addresses == organization in kill list
        organization = getOrganization(logEntries.get(0));
        if (StringUtil.isNullOrEmpty(organization.getEmailAddresses())) {
            String msg = "Email address is missing; will not send email to organization#" +  organization.getId() + ": " + organization.getName();
            log.info(msg);
            return;
        }

        // -- valid email addresses?
        if (!AppUtil.isEmailAddressesValid(organization.getEmailAddresses())) {
            String msg = "Mail not sent; invalid email addresses (" + organization.getEmailAddresses() + ") for organization (" + organization.getName() + ").";
            log.error(msg);
            mailJobContext.writeToConsole(msg);
            return;
        }

        // -- remove quarantined log entries
        List<LogEntry> logEntriesToSend = new ArrayList<LogEntry>();
        int periodInSecs = props.getInt(AppProperties.MAIL_IP_QUARANTINE_PERIOD_KEY, 7*24*60*60);
        for (Iterator<LogEntry> iterator = logEntries.iterator(); iterator.hasNext(); ) {
            LogEntry logEntry = iterator.next();
            Long ip = mailJob.isUsePrimaryOrg() ? logEntry.getIpAddress() : logEntry.getIpAddress2();
            boolean exists = (ip != null) && dbManager.existsMailForIp(ip.longValue(), organization, periodInSecs, mailJob.isUsePrimaryOrg());
            if (exists) {
                String msg = "Ignoring log entry#" + logEntry.getId() + ", because IP is quarantined. An email have already been sent for this IP: " +
                    IpAddressUtil.convertIpAddress(ip, false);
                log.info(msg);
                mailJobContext.writeToConsole(msg);
                mailJobContext.incNoOfQuarantineLogEntries(1);
            } else {
                logEntriesToSend.add(logEntry);
            }
        }

        // -- all log entries removed?
        if (logEntriesToSend.isEmpty()) {
            String msg = "All log entries quarantined; will not send email to organization#" +  organization.getId() + ": " + organization.getName();
            log.info(msg);
            mailJobContext.writeToConsole(msg);
            return;
        }

        // -- send mail (or write to the console if --dry-run2)
        MessageData messageData = createMessageData(logEntriesToSend);
        if (props.isMailDryRun2()) {
            mailJobContext.writeToConsole("To: " + organization.getEmailAddresses());
            mailJobContext.writeToConsole("Subject: " + messageData.getSubject());
            String body = messageData.getBody();
            mailJobContext.writeToConsole("---- begin body (" + body.length() + " characters) ----");
            mailJobContext.writeToConsole(body);
            mailJobContext.writeToConsole("---- end body ----");
            mailJobContext.writeToConsole("");
            mailJobContext.incNoOfMailsTobeSent(1);
        } else if (props.isMailDryRun()) {
            addToSummary(organization, messageData);
            mailJobContext.incNoOfMailsTobeSent(1);
        } else {
            addToSummary(organization, messageData);
            log.info("Sending abuse mail:" + organization.getEmailAddresses());
            mailSender.clear();
            mailSender.setToAddresses(organization.getEmailAddresses());
            mailSender.setBccAddresses(props.getString(AppProperties.MAIL_ARCHIVE_BCC_ADDRESSES_KEY, null));
            mailSender.setSubject(messageData.getSubject());
            mailSender.setBody(messageData.getBody());
            mailSender.send(props);
            mailJobContext.incNoOfSentMails(1);
            mailJobContext.incNoOfSentLogEntries(logEntriesToSend.size());
            for (Iterator<LogEntry> iterator = logEntriesToSend.iterator(); iterator.hasNext(); ) {
                LogEntry logEntry = iterator.next();
                log.debug("Adding log entry to mail job. LogEntry#" + logEntry.getId());
                mailJob.addToLogEntries(logEntry);
            }
            // Removing the line below due to deprication of the update method in DbManager
            // The method is depricated since there have been problems with Hibernate persisting the MailJob object
            // properly after a LogEntry has been added to the LogEntry collection.
            // dbManager.updateMailJob(mailJob);
        }
    }

    /**
     * Sends summary email, or writes summary to the console if --dry-runX is enabled.
     */
    public void sendSummaryMail() throws MegatronException {
        if (props.isMailDryRun2()) {
            return;
        }
        if (summaryBody == null) {
            summaryBody = new StringBuilder(512);
            summaryBody.append("Mail Job is empty. It contains no emails.").append(Constants.LINE_BREAK).append(Constants.LINE_BREAK);
            // init globalAttributeMap
            createAttributeMap(null);
        }

        String jobInfo = JOB_INFO_TEMPLATE;
        jobInfo = StringUtil.replace(jobInfo, "$noOfSentMails", mailJobContext.getNoOfSentMails() + "");
        jobInfo = StringUtil.replace(jobInfo, "$noOfSentLogEntries", mailJobContext.getNoOfSentLogEntries() + "");
        jobInfo = StringUtil.replace(jobInfo, "$noOfQuarantineLogEntries", mailJobContext.getNoOfQuarantineLogEntries() + "");
        jobInfo = StringUtil.replace(jobInfo, "$useOrg2", !mailJob.isUsePrimaryOrg() + "");
        jobInfo = AppUtil.replaceVariables(jobInfo, globalAttributeMap, false, "JOB_INFO_TEMPLATE (hardcoded)");
        summaryBody.insert(0, jobInfo);

        if (props.isMailDryRun()) {
            mailJobContext.writeToConsole(summaryBody.toString());
        } else {
            String toAddresses = props.getString(AppProperties.MAIL_SUMMARY_TO_ADDRESSES_KEY, "");
            if (toAddresses.trim().length() == 0) {
                String msg = "Sending mail job summary email skipped; no to-addresses defined in config.";
                log.info(msg);
            } else {
                log.info("Sending summary mail:" + toAddresses);
                String subject = props.getString(AppProperties.MAIL_JOB_SUMMARY_SUBJECT_TEMPLATE_KEY, SUMMARY_SUBJECT_TEMPLATE);
                subject = AppUtil.replaceVariables(subject, globalAttributeMap, false, AppProperties.MAIL_JOB_SUMMARY_SUBJECT_TEMPLATE_KEY);
                mailSender.clear();
                mailSender.setToAddresses(toAddresses);
                mailSender.setSubject(subject);
                mailSender.setBody(summaryBody.toString());
                mailSender.send(props);
            }
        }
    }

    public void close() throws MegatronException {
        dbManager.close();
    }

    @Override
    protected String getTemplateDir() {
        return props.getString(AppProperties.MAIL_TEMPLATE_DIR_KEY, "conf/template/mail");
    }

    @Override
    protected String getTimestampFormat() {
        return props.getString(AppProperties.MAIL_TIMESTAMP_FORMAT_KEY, "yyyy-MM-dd HH:mm:ss z");
    }

    private void init() {
        this.dbManager = mailJobContext.getDbManager();
        this.mailSender = new MailSender();
    }

    private MessageData createMessageData(List<LogEntry> logEntries) throws MegatronException {
        String templateName = null;
        String template = null;
        Organization organization = getOrganization(logEntries.get(0));
        Map<String, String> attributeMap = createAttributeMap(organization);
        String langCode = null;
        String defaultLangCode = props.getString(AppProperties.MAIL_DEFAULT_LANGUAGE_CODE_KEY, null);
        if ((organization != null) && (organization.getLanguageCode() != null) && (defaultLangCode != null) && !organization.getLanguageCode().equalsIgnoreCase(defaultLangCode)) {
            langCode = organization.getLanguageCode();
        }

        // -- subject
        if (langCode != null) {
            templateName = AppProperties.MAIL_SUBJECT_TEMPLATE_KEY + "." + langCode.toLowerCase();
            template = props.getString(templateName, null);
        }
        if (template == null) {
            templateName = AppProperties.MAIL_SUBJECT_TEMPLATE_KEY;
            template = props.getString(templateName, "S\u00e4kerhetsmeddelande fr\u00e5n SITIC [RTIR ID #$rtirId]");
        }
        String subject = AppUtil.replaceVariables(template, attributeMap, false, templateName);

        // -- body
        StringBuilder body = new StringBuilder(512);
        // header
        templateName = props.getString(AppProperties.MAIL_HEADER_FILE_KEY, null);
        template = readTemplate(AppProperties.MAIL_HEADER_FILE_KEY, langCode, false);
        if (template != null) {
            body.append(AppUtil.replaceVariables(template, attributeMap, false, templateName));
        }
        // rows
        templateName = props.getString(AppProperties.MAIL_ROW_FILE_KEY, null);
        boolean raiseErrorForDebugTemplate = props.getBoolean(AppProperties.MAIL_RAISE_ERROR_FOR_DEBUG_TEMPLATE_KEY, true);
        if (raiseErrorForDebugTemplate && ((templateName == null) || templateName.equals("debug_row.txt"))) {
            String msg = "Mail templates are not specified or debug templates are used. Probably, no templates exists for this job type.";
            throw new MegatronException(msg);
        }
        template = readTemplate(AppProperties.MAIL_ROW_FILE_KEY, langCode, true);
        for (Iterator<LogEntry> iterator = logEntries.iterator(); iterator.hasNext(); ) {
            LogEntry logEntry = iterator.next();
            LogEntryMapper mapper = new LogEntryMapper(props, logEntry);
            body.append(mapper.replaceVariables(template, false, templateName));
        }
        // footer
        templateName = props.getString(AppProperties.MAIL_FOOTER_FILE_KEY, null);
        template = readTemplate(AppProperties.MAIL_FOOTER_FILE_KEY, langCode, false);
        if (template != null) {
            body.append(AppUtil.replaceVariables(template, attributeMap, false, templateName));
        }

        return new MessageData(subject, body.toString());
    }

    private Map<String, String> createAttributeMap(Organization organization) {
        if (globalAttributeMap == null) {
            globalAttributeMap = new HashMap<String, String>();
            String mailJobId = (mailJob.getId() != null) ? mailJob.getId().toString() : "-";
            addString(globalAttributeMap, MAIL_JOB_ID, mailJobId);
            addTimestamp(globalAttributeMap, MAIL_JOB_STARTED, SqlUtil.convertTimestampToSec(mailJobContext.getStartedTimestamp()));
            addString(globalAttributeMap, MAIL_JOB_COMMENT, mailJob.getComment());
            addTimestamp(globalAttributeMap, JOB_STARTED, SqlUtil.convertTimestampToSec(mailJobContext.getJob().getStarted()));
            addString(globalAttributeMap, JOB_NAME, mailJob.getJob().getName());
            addString(globalAttributeMap, JOB_FILENAME, mailJob.getJob().getFilename());
            String rtirId = StringUtil.isNullOrEmpty(props.getId()) ? "-" : props.getId();
            addString(globalAttributeMap, RTIR_ID, rtirId);
        }

        Map<String, String> result = new HashMap<String, String>();
        result.putAll(globalAttributeMap);
        if (organization != null) {
            addString(result, ORGANIZATION_NAME, organization.getName());
            addString(result, EMAIL_ADDRESSES, organization.getEmailAddresses());
        }

        return result;
    }

    private Organization getOrganization(LogEntry logEntry) {
        return mailJob.isUsePrimaryOrg() ? logEntry.getOrganization() : logEntry.getOrganization2();
    }

    private void addToSummary(Organization organization, MessageData messageData) {
        if (summaryBody == null) {
            summaryBody = new StringBuilder(1024);

            summaryBody.append(MESSAGE_HEADER);
            summaryBody.append("Subject: " + messageData.getSubject()).append(Constants.LINE_BREAK);
            summaryBody.append(messageData.getBody()).append(Constants.LINE_BREAK);
            summaryBody.append(RECIPIENTS_HEADER);
        }

        String str = "@organizationName@: @emailAddresses@" + Constants.LINE_BREAK;
        str = StringUtil.replace(str, "@organizationName@", organization.getName());
        str = StringUtil.replace(str, "@emailAddresses@", organization.getEmailAddresses());
        summaryBody.append(str);
    }

    /**
     * Entiry class for the content in an email.
     */
    private class MessageData {
        private String subject;
        private String body;

        public MessageData(String subject, String body) {
            this.subject = subject;
            this.body = body;
        }

        public String getSubject() {
            return subject;
        }

        public String getBody() {
            return body;
        }
    }

}


text:
"""
An object to handle a list of valid Minecraft version files.
"""
import os
import json
from .mcversion import MCVersion

class MCVersionsList(object):
    """
    An object to handle a list of valid Minecraft version files.
    """
    def __init__(self, mc_dir):
        """Initialize a dict of MCVersion objects according to the
        '.minecraft' folder in mc_dir."""
        self._dict = {}
        os.chdir(os.path.join(mc_dir, 'versions'))
        for version in os.listdir():
            if os.path.isdir(version):
                json_file = os.path.join(version, version + '.json')
                if os.path.exists(json_file):
                    with open(json_file) as fp:
                        self._dict[version] = MCVersion(json.load(fp))
        os.chdir('..')

        self.list = sorted(self._dict)

    def get(self, version_id):
        """Attempt to get a MCVersion object according to its id.
        Returns None if not in the list."""
        this = self._dict.get(version_id)
        if this is None:
            raise ValueError('Invalid version id {0}. Please check if {0}.json exists.'.format(version_id))
        parent_id = this.inherits_from
        if parent_id:
            parent = self.get(parent_id) # recursively do parent's inheritance by self.get instead of _dict.get
            this.inherit(parent)
        return this


text:
"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.DragSource = DragSource;

var _invariant = require("@react-dnd/invariant");

var _internals = require("../internals");

var _utils = require("./utils");

var _decorateHandler = require("./decorateHandler");

var _createSourceFactory = require("./createSourceFactory");

/**
 * Decorates a component as a dragsource
 * @param type The dragsource type
 * @param spec The drag source specification
 * @param collect The props collector function
 * @param options DnD options
 */
function DragSource(type, spec, collect) {
  var options = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : {};
  (0, _utils.checkDecoratorArguments)('DragSource', 'type, spec, collect[, options]', type, spec, collect, options);
  var getType = type;

  if (typeof type !== 'function') {
    (0, _invariant.invariant)((0, _utils.isValidType)(type), 'Expected "type" provided as the first argument to DragSource to be ' + 'a string, or a function that returns a string given the current props. ' + 'Instead, received %s. ' + 'Read more: http://react-dnd.github.io/react-dnd/docs/api/drag-source', type);

    getType = function getType() {
      return type;
    };
  }

  (0, _invariant.invariant)((0, _utils.isPlainObject)(spec), 'Expected "spec" provided as the second argument to DragSource to be ' + 'a plain object. Instead, received %s. ' + 'Read more: http://react-dnd.github.io/react-dnd/docs/api/drag-source', spec);
  var createSource = (0, _createSourceFactory.createSourceFactory)(spec);
  (0, _invariant.invariant)(typeof collect === 'function', 'Expected "collect" provided as the third argument to DragSource to be ' + 'a function that returns a plain object of props to inject. ' + 'Instead, received %s. ' + 'Read more: http://react-dnd.github.io/react-dnd/docs/api/drag-source', collect);
  (0, _invariant.invariant)((0, _utils.isPlainObject)(options), 'Expected "options" provided as the fourth argument to DragSource to be ' + 'a plain object when specified. ' + 'Instead, received %s. ' + 'Read more: http://react-dnd.github.io/react-dnd/docs/api/drag-source', collect);
  return function decorateSource(DecoratedComponent) {
    return (0, _decorateHandler.decorateHandler)({
      containerDisplayName: 'DragSource',
      createHandler: createSource,
      registerHandler: _internals.registerSource,
      createConnector: function createConnector(backend) {
        return new _internals.SourceConnector(backend);
      },
      createMonitor: function createMonitor(manager) {
        return new _internals.DragSourceMonitorImpl(manager);
      },
      DecoratedComponent: DecoratedComponent,
      getType: getType,
      collect: collect,
      options: options
    });
  };
}


text:
<?php

namespace Ulff\BehatRestApiExtension\Exception;

class CollectionExpectedException extends \Exception
{
    public function __construct()
    {
        $this->message = 'Expected response JSON with collection';
    }
}


text:
using System.Threading;

namespace System.IO.Pipelines
{
    /// <summary>
    /// Default <see cref="PipeWriter"/> and <see cref="PipeReader"/> implementation.
    /// </summary>
    public sealed partial class Pipe
    {
        private sealed class DefaultPipeWriter : PipeWriter, IPipeAwaiter<FlushResult>
        {
            private readonly Pipe _pipe;

            public DefaultPipeWriter(Pipe pipe)
            {
                _pipe = pipe;
            }

            public override void Complete(Exception exception = null) => _pipe.CompleteWriter(exception);

            public override void CancelPendingFlush() => _pipe.CancelPendingFlush();

            public override void OnReaderCompleted(Action<Exception, object> callback, object state) => _pipe.OnReaderCompleted(callback, state);

            public override PipeAwaiter<FlushResult> FlushAsync(CancellationToken cancellationToken = default) => _pipe.FlushAsync(cancellationToken);

            public override void Advance(int bytes) => _pipe.Advance(bytes);

            public override Memory<byte> GetMemory(int minimumLength = 0) => _pipe.GetMemory(minimumLength);

            public override Span<byte> GetSpan(int minimumLength = 0) => _pipe.GetMemory(minimumLength).Span;

            public override int MaxBufferSize => _pipe._pool.MaxBufferSize;

            public bool IsCompleted => _pipe.IsFlushAsyncCompleted;

            public FlushResult GetResult() => _pipe.GetFlushAsyncResult();

            public void OnCompleted(Action continuation) => _pipe.OnFlushAsyncCompleted(continuation);
        }
    }
}


text:
package com.camnter.newlife.ui.activity.jsbridge;

import android.os.Handler;
import android.os.Looper;
import android.webkit.WebView;
import java.lang.ref.WeakReference;
import org.json.JSONObject;

/**
 * Description:JsCallback
 *
 * native 层执行完代码,回调到 Js 层
 *
 * Created by:CaMnter
 */

public class JsCallback {

    private static Handler mHandler = new Handler(Looper.getMainLooper());
    private static final String JS_CALLBACK_FORMAT = "javascript:JsBridge.onFinish('%s', %s);";
    private String mPort;
    private WeakReference<WebView> mWebViewRef;

    public JsCallback(WebView view, String port) {
        this.mWebViewRef = new WeakReference<>(view);
        this.mPort = port;
    }

    public void apply(JSONObject jsonObject) {
        final String execJs = String.format(JS_CALLBACK_FORMAT, this.mPort,
            String.valueOf(jsonObject));
        if (this.mWebViewRef != null && this.mWebViewRef.get() != null) {
            mHandler.post(() -> mWebViewRef.get().loadUrl(execJs));
        }
    }

}


text:
/**
 * Initialise changelog table for releases and prereleases
 * @param {string} selector - The element selector to initialise the app.
 * @param {string} repo - Repository to load from, in the format user/repo.
 */
export default function(selector, repo) {
    new Vue({
        el: selector,
        data: {
            url: `https://api.github.com/repos/${repo}/releases`,
            releases: [],
            prereleases: [],
            error: false
        },
        beforeMount() {
            fetch(this.url)
                .then(res => res.json())
                .then(json => this.$_update(json))
                .catch(err => { this.error = true });
        },
        updated() {
            window.dispatchEvent(new Event('resize'));  // scroll position for progress
        },
        methods: {
            $_update(json) {
                const allReleases = Object.values(json)
                    .filter(release => release.name)
                    .map(release => ({
                        title: (release.name.split(': ').length == 2) ? release.name.split(': ')[1] : release.name,
                        url: release.html_url,
                        date: release.published_at.split('T')[0],
                        tag: release.tag_name,
                        pre: release.prerelease
                    }));
                this.releases = allReleases.filter(release => !release.pre);
                this.prereleases = allReleases.filter(release => release.pre);
            }
        }
    });
}


text:
ActiveRecord::Schema.define(:version => 20131014062229) do

  create_table "categories", :force => true do |t|
    t.integer  "user_id"
    t.integer  "parent_category_id"
    t.string   "name"
    t.boolean  "is_creditor"
    t.integer  "category_type"
    t.string   "shortcut"
    t.integer  "display_order"
    t.datetime "created_at",         :null => false
    t.datetime "updated_at",         :null => false
  end

  add_index "categories", ["user_id", "display_order"], :name => "index_categories_on_user_id_and_display_order"

  create_table "kakeibo_entries", :force => true do |t|
    t.integer  "user_id"
    t.integer  "debtor_id"
    t.integer  "debtor_sub_id"
    t.integer  "creditor_id"
    t.integer  "creditor_sub_id"
    t.integer  "amount"
    t.string   "memo"
    t.datetime "transaction_date"
    t.datetime "created_at",                      :null => false
    t.datetime "updated_at",                      :null => false
    t.integer  "is_template",      :default => 0, :null => false
  end

  add_index "kakeibo_entries", ["user_id", "creditor_id"], :name => "index_kakeibo_entries_on_user_id_and_creditor_id"
  add_index "kakeibo_entries", ["user_id", "debtor_id"], :name => "index_kakeibo_entries_on_user_id_and_debtor_id"
  add_index "kakeibo_entries", ["user_id", "transaction_date"], :name => "index_kakeibo_entries_on_user_id_and_transaction_date"

  create_table "settings", :force => true do |t|
    t.integer  "user_id"
    t.string   "key"
    t.string   "value"
    t.datetime "created_at", :null => false
    t.datetime "updated_at", :null => false
  end

  add_index "settings", ["user_id", "key"], :name => "index_settings_on_user_id_and_key", :unique => true
  add_index "settings", ["user_id"], :name => "index_settings_on_user_id"

  create_table "summaries", :force => true do |t|
    t.integer  "user_id"
    t.integer  "category_id"
    t.integer  "year"
    t.integer  "month"
    t.integer  "sum_amount"
    t.integer  "balance"
    t.integer  "budget"
    t.datetime "created_at",  :null => false
    t.datetime "updated_at",  :null => false
  end

  add_index "summaries", ["user_id", "year", "month", "category_id"], :name => "index_summaries_on_user_id_and_year_and_month_and_category_id", :unique => true

  create_table "users", :force => true do |t|
    t.string   "email",                  :default => "", :null => false
    t.string   "encrypted_password",     :default => "", :null => false
    t.string   "reset_password_token"
    t.datetime "reset_password_sent_at"
    t.datetime "remember_created_at"
    t.integer  "sign_in_count",          :default => 0
    t.datetime "current_sign_in_at"
    t.datetime "last_sign_in_at"
    t.string   "current_sign_in_ip"
    t.string   "last_sign_in_ip"
    t.string   "confirmation_token"
    t.datetime "confirmed_at"
    t.datetime "confirmation_sent_at"
    t.string   "unconfirmed_email"
    t.datetime "created_at",                             :null => false
    t.datetime "updated_at",                             :null => false
  end

  add_index "users", ["email"], :name => "index_users_on_email", :unique => true
  add_index "users", ["reset_password_token"], :name => "index_users_on_reset_password_token", :unique => true

end


text:
import { enableProdMode } from '@angular/core';
import { platformBrowserDynamic } from '@angular/platform-browser-dynamic';
import { AppModule } from './app/app.module';
import { environment } from './environments/environment';

const providers = [];

if (environment.production) {
    enableProdMode();
}

platformBrowserDynamic(providers).bootstrapModule(AppModule)
    .catch(err => console.log(err));


text:
@class AHReach;

typedef void (^AHReachChangedBlock)(AHReach *);

@interface AHReach : NSObject

+ (AHReach *)reachForHost:(NSString *)host;
+ (AHReach *)reachForAddress:(const struct sockaddr_in *)addr;
+ (AHReach *)reachForDefaultHost;

- (BOOL)isReachable;
- (BOOL)isReachableViaWWAN;
- (BOOL)isReachableViaWiFi;

- (void)startUpdatingWithBlock:(AHReachChangedBlock)changedBlock;
- (void)stopUpdating;

@end


text:
//
//  UIWindow+MAREX.h
//  MARExtensionDemo
//
//  Created by Martin.liu on 17/5/18.
//  Copyright © 2017年 MAR. All rights reserved.
//

#import <UIKit/UIKit.h>

@interface UIWindow (MAREX)

/**
*  Take a screenshot of current window, without saving it
*
*  @return Returns the screenshot as an UIImage
*/
- (UIImage * _Nonnull)mar_takeScreenshot;

/**
 *  Take a screenshot of current window and choose if save it or not
 *
 *  @param save YES to save, NO to don't save
 *
 *  @return Returns the screenshot as an UIImage
 */
- (UIImage * _Nonnull)mar_takeScreenshotAndSave:(BOOL)save;

/**
 *  Take a screenshot of current window, choose if save it or not after a delay
 *
 *  @param delay      The delay, in seconds
 *  @param save       YES to save, NO to don't save
 *  @param completion Completion handler with the UIImage
 */
- (void)mar_takeScreenshotWithDelay:(CGFloat)delay save:(BOOL)save completion:(void (^ _Nullable)(UIImage * _Nonnull screenshot))completion;

/**
 *  Show touch on screen. (Use BFShowTouchOnScreen macro)
 */
- (void)mar_activateTouch;

/**
 *  Hide touch on screen. (Use BFHideTouchOnScreen macro)
 */
- (void)mar_deactivateTouch;

- (UIImage * _Nonnull)mar_touchImage;

@end


text:
package io.github.interestinglab.waterdrop.input.batch

import org.apache.spark.sql.{Dataset, Row, SparkSession}

class Mysql extends Jdbc {

  override def checkConfig(): (Boolean, String) = {

    val requiredOptions = List("url", "table", "user", "password");

    val nonExistsOptions = requiredOptions.map(optionName => (optionName, config.hasPath(optionName))).filter { p =>
      val (optionName, exists) = p
      !exists
    }

    if (nonExistsOptions.isEmpty) {
      (true, "")
    } else {
      (
        false,
        "please specify " + nonExistsOptions
          .map { case (field, _) => "[" + field + "]" }
          .mkString(", ") + " as non-empty string")
    }
  }

  override def getDataset(spark: SparkSession): Dataset[Row] = {
    jdbcReader(spark, "com.mysql.jdbc.Driver").load()
  }
}


text:
/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

  $(document).ready(function(){

    $("#CargaAcademica").click(function(event)
    {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Cargas/gestionCargas.php");
    });

    $("#Asignaturas").click(function(event) {
      event.preventDefault();

      $("#contenedor" ).load( //Pantalla para ingresar un estudiante
                            );
      });
    $("#Edificios").click(function(event) {
      event.preventDefault();

      $("#contenedor" ).load( "pages/CargaAcademica/ca_crearEdificios.php"
                            );
      });

    $("#Aulas").click(function(event) {
      event.preventDefault();
      $("#contenedor").load(// Pantalla para ingresar una nueva solicitud
                            );
      });

    $("#Clases").click(function(event) {
      event.preventDefault();
      $("#contenedor").load(// Pantalla para ingresar una solicitud por estudiante
                            );
      });

    $("#seccionesMenu").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Secciones/gestionSeccion.php"
                            );
      });

    $("#estadosCargas").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Estados/Estados.php"
                            );
      });

    $("#AsignacionClases").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Clases/gestionClases.php"
                            );
      });

      $("#Cargas").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Cargas/gestionCargas.php"
                            );
      });

      $("#Acondicionamientos").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Acondicionamientos/ca_index_Acondicionamiento.php"
                            );
      });

    $("#RegistroDocentes").click(function(event) {
      event.preventDefault();
      $("#contenedor").load( "pages/CargaAcademica/RegistroDocentes/RegristroDocentes.php"
                            );
      });
    $("#Facultad").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/facultad/ca_indexFacultad.php"
                            );
      });
    $("#AreasProyecto").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Areas_de_Proyectos/ca_index_Areas_de_Proyecto.php"
                            );
      });
    $("#AreasVinculacion").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/ca_crearAreaVinculacion.php"
                            );
      });
    $("#Proyectos").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/Proyecto/Proyectos.php"
                            );
      });
    $("#AsigancionProyecto").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/AsignacionProyectos/asignacionProyectos.php"
                            );
      });
    $("#busquedaCargaAcademica").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/BusquedaAvanzada/reporte.php"
                            );
      });

    $("#BusquedaAvanzadaProyectos").click(function(event) {
      event.preventDefault();
      $("#contenedor").load("pages/CargaAcademica/BusquedaAvanzada/BusquedaAvanzadaProyectos.php"
                            );
      });

});


text:
package io.hydrosphere.mist.master.execution.status

import akka.actor.ActorSystem
import io.hydrosphere.mist.core.logging.LogEvent
import io.hydrosphere.mist.master.{EventsStreamer, JobDetails}
import io.hydrosphere.mist.master.Messages.StatusMessages._
import io.hydrosphere.mist.master.logging.LogService
import io.hydrosphere.mist.master.store.JobRepository

import scala.concurrent.Future

trait StatusReporter {

  def report(ev: ReportedEvent): Unit
  def reportPlain(e: UpdateStatusEvent): Unit = report(ReportedEvent.plain(e))
  def reportWithFlushCallback(e: UpdateStatusEvent): Future[JobDetails] = {
    val ev = ReportedEvent.withCallback(e)
    report(ev)
    ev.callback.future
  }
}

object StatusReporter {

  val NOOP = new StatusReporter {
    override def report(ev: ReportedEvent): Unit = ()
  }

  /**
    * Send status updates to store + async interfaces
    */
  def reporter(
    repo: JobRepository,
    streamer: EventsStreamer,
    logService: LogService
  )(implicit sys: ActorSystem): StatusReporter = {
    val flusher = sys.actorOf(StoreFlusher.props(repo, logService))
    new StatusReporter {
      override def report(ev: ReportedEvent): Unit = {
        flusher ! ev
        streamer.push(ev.e)
      }
    }
  }
}


text:
package io.wia;

import java.util.List;
import java.util.Date;

import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

/**
 * Represents a location.
 */
public class WiaLocation extends WiaResource {

  String id;

  Double latitude;

  Double longitude;

  Double altitude;

  // /// The timestamp of this location
  // public var timestamp: Date?

  /// The timestamp milliseconds of this Event
  @SerializedName("timestamp")
  private long timestampMs;

  /**
   * @return id of this location.
   */
  public String id() {
    return id;
  }

  /**
   * @return latitude of this location.
   */
  public Double latitude() {
    return latitude;
  }

  /**
   * @return longitude of this location.
   */
  public Double longitude() {
    return longitude;
  }

  /**
   * @return altitude of this location.
   */
  public Double altitude() {
    return altitude;
  }

  /**
   * @return timestamp of this event.
   */
  public Date timestamp() {
    return new Date(timestampMs);
  }

  /**
   * Create a String from this object.
   * @return a String containing the id and name of this location
   */
  @Override public String toString() {
    return "WiaLocation{"
        + "id='" + id() + '\''
        + ", latitude='" + Double.toString(latitude) + '\''
        + ", longitude='" + Double.toString(longitude) + '\''
        + ", timestamp='" + timestamp() + '\''
        + '}';
  }
}


text:
#region Copyright notice and license

// Copyright 2015 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#endregion

using System;
using System.Threading;
using Grpc.Core;
using Grpc.Core.Internal;
using System.Collections.Generic;
using System.Diagnostics;

namespace Grpc.Microbenchmarks
{
    public class ThreadedBenchmark
    {
        List<ThreadStart> runners;

        public ThreadedBenchmark(IEnumerable<ThreadStart> runners)
        {
            this.runners = new List<ThreadStart>(runners);
        }

        public ThreadedBenchmark(int threadCount, Action threadBody)
        {
            this.runners = new List<ThreadStart>();
            for (int i = 0; i < threadCount; i++)
            {
                this.runners.Add(new ThreadStart(() => threadBody()));
            }
        }

        public void Run()
        {
            Console.WriteLine("Running threads.");
            var threads = new List<Thread>();
            for (int i = 0; i < runners.Count; i++)
            {
                var thread = new Thread(runners[i]);
                thread.Start();
                threads.Add(thread);
            }

            foreach (var thread in threads)
            {
                thread.Join();
            }
            Console.WriteLine("All threads finished.");
        }
    }
}


text:
<?php
namespace TijsVerkoyen\DocDataPayments\Types;

use TijsVerkoyen\DocDataPayments\Base\PaymentInfo;

/**
 * DocDataPayments MaestroPaymentInfo class
 *
 * @author Tijs Verkoyen <php-docdatapayments@verkoyen.eu>
 */
class MaestroPaymentInfo extends PaymentInfo
{
}


text:
all:
	gcc -I../../src/kBuffer ../../src/kBuffer/kBuffer.c demo.c -lm -o kBufferDemo


text:
'''Class for cleaning phone numbers'''

import phonenumbers

def standardize_phone(number):
    '''Do the best we can in trying to standardize the given phone number'''

    try:
        parsed_number = phonenumbers.parse(number, 'US')
        return phonenumbers.format_number(parsed_number,
                                          phonenumbers.PhoneNumberFormat.NATIONAL)
    except phonenumbers.NumberParseException as npe:
        return ""


text:
#ifndef ossimGuiConnection_HEADER
#define ossimGuiConnection_HEADER
#include <ossimGui/Export.h>
#include <ossim/base/ossimConnectableObject.h>

namespace ossimGui {
   class OSSIMGUI_DLL ConnectableObject : public ossimConnectableObject
   {
   public:
      ConnectableObject():ossimConnectableObject(0,0,false,false),m_autoDelete(true){}
      ConnectableObject(ossimObject* owner,
                 ossim_int32 inputListSize,
                 ossim_int32 outputListSize,
                 bool inputListIsFixedFlag=true,
                 bool outputListIsFixedFlag=true):ossimConnectableObject(owner, inputListSize, outputListSize, inputListIsFixedFlag, outputListIsFixedFlag){}

      virtual bool saveState(ossimKeywordlist& kwl, const char* prefix=0)const;
      virtual bool loadState(const ossimKeywordlist& kwl, const char* prefix=0);
      /**
       * Specifies if the connectable display auto deletes on disconnect
       *
       * Note, if the display attribute is not delete on close it
       * will not delete but just hide and keep it alive in the datamanager
       */
      bool autoDelete()const{return m_autoDelete;}
      /**
       * Turns on and off the ability to auto delete when the node is disconnected
       * from all inputs.
       */
      void setAutoDelete(bool flag){m_autoDelete = flag;}

   protected:
      bool m_autoDelete;
      TYPE_DATA;
   };

}
#endif


text:
require 'rails_helper'

RSpec.describe ProjectObserver do
  let(:contribution){ create(:contribution, key: 'should be updated', payment_method: 'should be updated', state: 'confirmed', confirmed_at: nil) }
  let(:project) do
    project = create(:project, state: 'draft', goal: 3000)
    create(:reward, project: project)
    project.update_attribute :state, :online
    project
  end

  let(:admin_user) do
    CatarseSettings[:email_payments] = 'admin@foo.com'
    create(:user, email: CatarseSettings[:email_payments])
  end

  let(:redbooth_user) do
    CatarseSettings[:email_redbooth] = 'foo@foo.com'
    create(:user, email: CatarseSettings[:email_redbooth])
  end

  let(:zendesk_user_atendimento) do
    CatarseSettings[:email_contact] = 'foo_contato@foo.com'
    create(:user, email: CatarseSettings[:email_contact])
  end

  subject{ contribution }

  before do
    CatarseSettings[:support_forum] = 'http://support.com'
    CatarseSettings[:email_projects] = 'foo@foo.com'
    CatarseSettings[:facebook_url] = 'http://facebook.com/foo'
    CatarseSettings[:blog_url] = 'http://blog.com/foo'
    CatarseSettings[:company_name] = 'Catarse'
  end

  describe "#before_save" do
    context "when project is new" do
      before do
        expect(project).to receive(:update_expires_at)
      end

      let(:project) { build(:project, state: 'approved') }

      it "should call update_expires_at" do
        project.save(validate: false)
      end
    end

    context "when project is being updated and online_days does not change" do
      before do
        expect(project).to_not receive(:update_expires_at)
      end

      let!(:project) { create(:project, state: 'approved', expires_at: Date.tomorrow) }

      it "should not call update_expires_at" do
        project.save(validate: false)
      end
    end

    context "when expires_at is nil and we have both online_at and online_days" do
      before do
        expect(project).to receive(:update_expires_at)
      end

      let!(:project) { create(:project, state: 'approved', online_days: 60) }

      it "should not call update_expires_at" do
        project.save(validate: false)
      end
    end
  end

  describe "#after_save" do
    let(:project) { build(:project, state: 'approved') }
    context "when we change the video_url" do
      let(:project){ create(:project, video_url: 'http://vimeo.com/11198435', state: 'draft')}
      before do
        expect(ProjectDownloaderWorker).to receive(:perform_async).with(project.id).never
      end
      it "should call project downloader" do
        project.save(validate: false)
      end
    end
  end

  describe "#from_draft_to_in_analysis" do
    before do
      @user = create(:user, email: ::CatarseSettings[:email_projects])
      @project = project = create(:project, state: 'draft')
      create(:reward, project: project)
      project.notify_observers(:from_draft_to_in_analysis)
    end

    it "should create notification for catarse admin" do
      expect(ProjectNotification.where(user_id: @user.id, template_name: :new_draft_project, project_id: @project.id).count).to eq 1
    end
  end

  describe "#from_online_to_waiting_funds" do
    before do
      zendesk_user_atendimento
      project.notify_observers(:from_online_to_waiting_funds)
    end

    context "when project has not reached goal" do
      let(:project) do
        project = create(:project, state: 'draft', goal: 3000)
        create(:reward, project: project)
        project.update_attribute :state, :online
        allow(project).to receive(:reached_goal?).and_return(false)
        project
      end

      it "should not send project_will_succeed notification" do
        expect(ProjectNotification.where(template_name: 'project_will_succeed', user: zendesk_user_atendimento, project: project).count).to eq 0
      end
    end

    context "when project has reached goal" do
      let(:project) do
        project = create(:project, state: 'draft', goal: 3000)
        create(:reward, project: project)
        project.update_attribute :state, :online
        allow(project).to receive(:reached_goal?).and_return(true)
        project
      end

      it "should send project_will_succeed notification" do
        expect(ProjectNotification.where(template_name: 'project_will_succeed', user: zendesk_user_atendimento, project: project).count).to eq 1
      end
    end

  end

  describe "#from_online_to_failed" do
    let(:project) do
      create_project({
        goal: 100,
        online_days: 30,
        state: 'online'
      }, {
        to_state: 'online',
        created_at: 3.days.ago,
      })
    end

    let(:contribution_invalid) do
      create(:confirmed_contribution, value: 10, project: project)
    end

    let(:contribution_valid) do
      create(:confirmed_contribution, value: 10, project: project)
    end

    context "not request refund to invalid bank_account slip payment" do
      let(:payment_valid) do
        contribution_valid.payments.first
      end

      let(:payment_invalid) do
        p = contribution_invalid.payments.first
        p.update_column(:payment_method, 'BoletoBancario')
        p
      end

      before do
        Sidekiq::Testing.inline!
        payment_valid
        payment_invalid
        contribution_invalid.user.bank_account.destroy
        project.update_attribute :online_days, 1
        expect(DirectRefundWorker).to receive(:perform_async).with(payment_valid.id)
        expect(DirectRefundWorker).to_not receive(:perform_async).with(payment_invalid.id)
      end

      it { project.finish }
    end
  end

  describe '#from_waiting_funds_to_successful' do
    before do
      admin_user
      redbooth_user
      project.notify_observers(:from_waiting_funds_to_successful)
    end

    it "should create notification for admin" do
      expect(ProjectNotification.where(template_name: 'redbooth_task', user: redbooth_user, project_id: project.id).count).not_to be_nil
    end
  end
end


text:
#include "config.h"

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <termios.h>
#include <fcntl.h>
#include <time.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>

#ifdef WITH_OUTB
#ifdef HAVE_SYS_IO_H
#include <sys/io.h>
#else
#ifdef HAVE_ASM_IO_H
#include <asm/io.h>
#else
#warning neither sys/io.h nor asm/io.h exists.
#warning raw port I/O will be disabled.
#undef WITH_OUTB
#endif
#endif
#endif

#if defined (HAVE_LINUX_PARPORT_H) && defined (HAVE_LINUX_PPDEV_H)
#define WITH_PPDEV
#include <linux/parport.h>
#include <linux/ppdev.h>
#else
#define PARPORT_CONTROL_STROBE    0x1
#define PARPORT_CONTROL_AUTOFD    0x2
#define PARPORT_CONTROL_INIT      0x4
#define PARPORT_CONTROL_SELECT    0x8
#define PARPORT_STATUS_ERROR      0x8
#define PARPORT_STATUS_SELECT     0x10
#define PARPORT_STATUS_PAPEROUT   0x20
#define PARPORT_STATUS_ACK        0x40
#define PARPORT_STATUS_BUSY       0x80
#endif

/* these signals are inverted by hardware on the parallel port */
#define PARPORT_CONTROL_INVERTED (PARPORT_CONTROL_STROBE | PARPORT_CONTROL_SELECT | PARPORT_CONTROL_AUTOFD)

#if !defined(WITH_OUTB) && !defined(WITH_PPDEV)
#error neither outb() nor ppdev() possible
#error cannot compile parallel port driver
#endif

#include "debug.h"
#include "qprintf.h"
#include "cfg.h"
#include "udelay.h"
#include "drv_generic_parport.h"

static char *Driver = "";
static char *Section = "";
static unsigned short Port = 0;
static char *PPdev = NULL;

/* Any bits set here will have their logic inverted at the parallel port */
static unsigned char inverted_control_bits = 0;

#ifdef WITH_OUTB
/* initial value taken from linux/parport_pc.c */
static unsigned char ctr = 0xc;
#endif

#ifdef WITH_PPDEV
static int PPfd = -1;
#endif

int drv_generic_parport_open(const char *section, const char *driver)
{
    char *s, *e;

    Section = (char *) section;
    Driver = (char *) driver;

    udelay_init();

#ifndef WITH_PPDEV
    error("The files include/linux/parport.h and/or include/linux/ppdev.h");
    error("were missing at compile time. Even if your system supports");
    error("ppdev, it will not be used.");
    error("You *really* should install these files and recompile LCD4linux!");
#endif

    s = cfg_get(Section, "Port", NULL);
    if (s == NULL || *s == '\0') {
	error("%s: no '%s.Port' entry from %s", Driver, Section, cfg_source());
	return -1;
    }

    PPdev = NULL;
    if ((Port = strtol(s, &e, 0)) == 0 || *e != '\0') {
#ifdef WITH_PPDEV
	Port = 0;
	PPdev = s;
#else
	error("%s: bad %s.Port '%s' from %s", Driver, Section, s, cfg_source());
	free(s);
	return -1;
#endif
    }
#ifdef WITH_PPDEV
    if (PPdev) {
	info("%s: using ppdev %s", Driver, PPdev);
	PPfd = open(PPdev, O_RDWR);
	if (PPfd == -1) {
	    error("%s: open(%s) failed: %s", Driver, PPdev, strerror(errno));
	    return -1;
	}
#if 0
	/* PPEXCL fails if someone else uses the port (e.g. lp.ko) */
	if (ioctl(PPfd, PPEXCL)) {
	    info("%s: ioctl(%s, PPEXCL) failed: %s", PPdev, Driver, strerror(errno));
	    info("%s: could not get exclusive access to %s.", Driver, PPdev);
	} else {
	    info("%s: got exclusive access to %s.", Driver, PPdev);
	}
#endif
	if (ioctl(PPfd, PPCLAIM)) {
	    error("%s: ioctl(%s, PPCLAIM) failed: %d %s", Driver, PPdev, errno, strerror(errno));
	    return -1;
	}
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	error("using raw port 0x%x (deprecated!)", Port);
	error("You *really* should change your setup and use ppdev!");
	if ((Port + 3) <= 0x3ff) {
	    if (ioperm(Port, 3, 1) != 0) {
		error("%s: ioperm(0x%x) failed: %s", Driver, Port, strerror(errno));
		return -1;
	    }
	} else {
	    if (iopl(3) != 0) {
		error("%s: iopl(1) failed: %s", Driver, strerror(errno));
		return -1;
	    }
	}
    }
#endif
    return 0;
}

int drv_generic_parport_close(void)
{
#ifdef WITH_PPDEV
    if (PPdev) {
	debug("closing ppdev %s", PPdev);
	if (ioctl(PPfd, PPRELEASE)) {
	    error("%s: ioctl(%s, PPRELEASE) failed: %s", Driver, PPdev, strerror(errno));
	}
	if (close(PPfd) == -1) {
	    error("%s: close(%s) failed: %s", Driver, PPdev, strerror(errno));
	    return -1;
	}
	free(PPdev);
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	debug("closing raw port 0x%x", Port);
	if ((Port + 3) <= 0x3ff) {
	    if (ioperm(Port, 3, 0) != 0) {
		error("%s: ioperm(0x%x) failed: %s", Driver, Port, strerror(errno));
		return -1;
	    }
	} else {
	    if (iopl(0) != 0) {
		error("%s: iopl(0) failed: %s", Driver, strerror(errno));
		return -1;
	    }
	}
    }
#endif

    return 0;
}

static unsigned char drv_generic_parport_signal_ctrl(const char *name, const char *signal)
{
    unsigned char wire;
    int invert = 0;

    /* Prefixing a signal name with '-' inverts the logic */
    if (signal[0] == '-') {
	invert = 1;
	signal++;
    }

    if (strcasecmp(signal, "STROBE") == 0) {
	wire = PARPORT_CONTROL_STROBE;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:STROBE (Pin  1)%s", Driver, name, invert ? " [inverted]" : "");
    } else if (strcasecmp(signal, "AUTOFD") == 0) {
	wire = PARPORT_CONTROL_AUTOFD;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:AUTOFD (Pin 14)%s", Driver, name, invert ? " [inverted]" : "");
    } else if (strcasecmp(signal, "INIT") == 0) {
	wire = PARPORT_CONTROL_INIT;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:INIT   (Pin 16)%s", Driver, name, invert ? " [inverted]" : "");
    } else if (strcasecmp(signal, "SLCTIN") == 0) {
	wire = PARPORT_CONTROL_SELECT;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:SLCTIN (Pin 17)%s", Driver, name, invert ? " [inverted]" : "");
    } else if (strcasecmp(signal, "SELECT") == 0) {
	wire = PARPORT_CONTROL_SELECT;
	error("%s: SELECT is deprecated. Please use SLCTIN instead!", Driver);
	info("%s: wiring: DISPLAY:%-9s - PARPORT:SLCTIN (Pin 17)%s", Driver, name, invert ? " [inverted]" : "");
    } else if (strcasecmp(signal, "GND") == 0) {
	wire = 0;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:GND", Driver, name);
    } else {
	error("%s: unknown signal <%s> for control line <%s>", Driver, signal, name);
	error("%s: should be STROBE, AUTOFD, INIT, SLCTIN or GND", Driver);
	return 0xff;
    }

    if (invert) {
	inverted_control_bits |= wire;
    }

    return wire;
}

unsigned char drv_generic_parport_wire_ctrl(const char *name, const char *deflt)
{
    unsigned char wire;
    char key[256];
    char *val;

    qprintf(key, sizeof(key), "Wire.%s", name);
    val = cfg_get(Section, key, deflt);

    wire = drv_generic_parport_signal_ctrl(name, val);

    free(val);

    return wire;
}

unsigned char drv_generic_parport_hardwire_ctrl(const char *name, const char *signal)
{
    unsigned char wire;
    char key[256];
    char *val;

    qprintf(key, sizeof(key), "Wire.%s", name);
    val = cfg_get(Section, key, "");

    /* maybe warn the user */
    if (*val != '\0' && strcasecmp(signal, val) != 0) {
	error("%s: ignoring configured signal <%s> for control line <%s>", Driver, val, name);
    }
    free(val);

    wire = drv_generic_parport_signal_ctrl(name, signal);

    return wire;
}

static unsigned char drv_generic_parport_signal_status(const char *name, const char *signal)
{
    unsigned char wire;

    if (strcasecmp(signal, "ERROR") == 0) {
	wire = PARPORT_STATUS_ERROR;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:ERROR    (Pin 15)", Driver, name);
    } else if (strcasecmp(signal, "SELECT") == 0) {
	wire = PARPORT_STATUS_SELECT;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:SELECT   (Pin 13)", Driver, name);
    } else if (strcasecmp(signal, "PAPEROUT") == 0) {
	wire = PARPORT_STATUS_PAPEROUT;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:PAPEROUT (Pin 12)", Driver, name);
    } else if (strcasecmp(signal, "ACK") == 0) {
	wire = PARPORT_STATUS_ACK;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:ACK      (Pin 10)", Driver, name);
    } else if (strcasecmp(signal, "BUSY") == 0) {
	wire = PARPORT_STATUS_BUSY;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:BUSY     (Pin 11)", Driver, name);
    } else if (strcasecmp(signal, "GND") == 0) {
	wire = 0;
	info("%s: wiring: DISPLAY:%-9s - PARPORT:GND", Driver, name);
    } else {
	error("%s: unknown signal <%s> for status line <%s>", Driver, signal, name);
	error("%s: should be ERROR, SELECT, PAPEROUT, ACK, BUSY or GND", Driver);
	return 0xff;
    }

    return wire;
}

unsigned char drv_generic_parport_wire_status(const char *name, const char *deflt)
{
    unsigned char wire;
    char key[256];
    char *val;

    qprintf(key, sizeof(key), "Wire.%s", name);
    val = cfg_get(Section, key, deflt);

    wire = drv_generic_parport_signal_status(name, val);

    free(val);

    return wire;
}

unsigned char drv_generic_parport_wire_data(const char *name, const char *deflt)
{
    unsigned char w;
    char wire[256];
    char *s;

    qprintf(wire, sizeof(wire), "Wire.%s", name);
    s = cfg_get(Section, wire, deflt);
    if (strlen(s) == 3 && strncasecmp(s, "DB", 2) == 0 && s[2] >= '0' && s[2] <= '7') {
	w = s[2] - '0';
    } else if (strcasecmp(s, "GND") == 0) {
	w = 0;
    } else {
	error("%s: unknown signal <%s> for data line <%s>", Driver, s, name);
	error("%s: should be DB0..7 or GND", Driver);
	return 0xff;
    }
    free(s);
    if (w == 0) {
	info("%s: wiring: DISPLAY:%-9s - PARPORT:GND", Driver, name);
    } else {
	info("%s: wiring: DISPLAY:%-9s - PARPORT:DB%d (Pin %2d)", Driver, name, w, w + 2);
    }

    w = 1 << w;

    return w;
}

void drv_generic_parport_direction(const int direction)
{
#ifdef WITH_PPDEV
    if (PPdev) {
	ioctl(PPfd, PPDATADIR, &direction);
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	/* code stolen from linux/parport_pc.h */
	ctr = (ctr & ~0x20) ^ (direction ? 0x20 : 0x00);
	outb(ctr, Port + 2);
    }
#endif
}

unsigned char drv_generic_parport_status(void)
{
    unsigned char mask =
	PARPORT_STATUS_ERROR | PARPORT_STATUS_SELECT | PARPORT_STATUS_PAPEROUT | PARPORT_STATUS_ACK |
	PARPORT_STATUS_BUSY;

    unsigned char data;

#ifdef WITH_PPDEV
    if (PPdev) {
	ioctl(PPfd, PPRSTATUS, &data);
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	data = inb(Port + 1);
    }
#endif

    /* clear unused bits */
    data &= mask;

    return data;
}

void drv_generic_parport_control(const unsigned char mask, const unsigned char value)
{
    unsigned char val;

    /* any signal affected? */
    /* Note: this may happen in case a signal is hardwired to GND */
    if (mask == 0)
	return;

    /* Strobe, Select and AutoFeed are inverted! */
    val = mask & (value ^ PARPORT_CONTROL_INVERTED ^ inverted_control_bits);

#ifdef WITH_PPDEV
    if (PPdev) {
	struct ppdev_frob_struct frob;
	frob.mask = mask;
	frob.val = val;
	ioctl(PPfd, PPFCONTROL, &frob);
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	/* code stolen from linux/parport_pc.h */
	ctr = (ctr & ~mask) ^ val;
	outb(ctr, Port + 2);
    }
#endif
}

void drv_generic_parport_toggle(const unsigned char bits, const int level, const unsigned long delay)
{
    unsigned char value1, value2;

    /* any signal affected? */
    /* Note: this may happen in case a signal is hardwired to GND */
    if (bits == 0)
	return;

    /* prepare value */
    value1 = level ? bits : 0;
    value2 = level ? 0 : bits;

    /* Strobe, Select and AutoFeed are inverted! */
    value1 = bits & (value1 ^ PARPORT_CONTROL_INVERTED ^ inverted_control_bits);
    value2 = bits & (value2 ^ PARPORT_CONTROL_INVERTED ^ inverted_control_bits);

#ifdef WITH_PPDEV
    if (PPdev) {
	struct ppdev_frob_struct frob;
	frob.mask = bits;

	/* rise */
	frob.val = value1;
	ioctl(PPfd, PPFCONTROL, &frob);

	/* pulse width */
	ndelay(delay);

	/* lower */
	frob.val = value2;
	ioctl(PPfd, PPFCONTROL, &frob);

    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	/* rise */
	ctr = (ctr & ~bits) ^ value1;
	outb(ctr, Port + 2);

	/* pulse width */
	ndelay(delay);

	/* lower */
	ctr = (ctr & ~bits) ^ value2;
	outb(ctr, Port + 2);
    }
#endif
}

void drv_generic_parport_data(const unsigned char data)
{
#ifdef WITH_PPDEV
    if (PPdev) {
	ioctl(PPfd, PPWDATA, &data);
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	outb(data, Port);
    }
#endif
}

unsigned char drv_generic_parport_read(void)
{
    unsigned char data = 0;

#ifdef WITH_PPDEV
    if (PPdev) {
	ioctl(PPfd, PPRDATA, &data);
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	data = inb(Port);
    }
#endif

    return data;
}

void drv_generic_parport_debug(void)
{
    unsigned char control = 0;

#ifdef WITH_PPDEV
    if (PPdev) {
	ioctl(PPfd, PPRCONTROL, &control);
    }
#endif

#ifdef WITH_OUTB
    if (Port) {
	control = ctr;
    }
#endif

    debug("%cSTROBE %cAUTOFD %cINIT %cSLCTIN",
	  control & PARPORT_CONTROL_STROBE ? '-' : '+',
	  control & PARPORT_CONTROL_AUTOFD ? '-' : '+', control & PARPORT_CONTROL_INIT ? '+' : '-',
	  control & PARPORT_CONTROL_SELECT ? '-' : '+');

}


text:
"use strict";

var fs = require("fs");
var path = require("path");

var rot13 = require("rot13-transform");

var Hapi = require("hapi");
var server = new Hapi.Server();

server.connection({
    host: "localhost",
    port: Number(process.argv[2] || 8080)
});

var stream = fs.createReadStream(path.join(__dirname, "assets/for-encryption.txt"));

function streaming(request, reply) {
    return reply(stream.pipe(rot13()));
}

server.route({ path: "/", method: "GET", handler: streaming });
server.start();


text:
{% extends "venue/base.html" %}

{% load boxes_tags %}

{% block body_id %}directions{% endblock %}

{% block subpage_title %}Directions to the Venue{% endblock %}
{% block content %}
    {% box "directions" %}
{% endblock %}


text:
using System;
using System.Collections;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Reflection;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading;

using Mono.Security.Protocol.Tls;

public class State {

	static ArrayList handleList = new ArrayList ();

	private int id;
	private HttpWebRequest request;
	private ManualResetEvent handle;
	private Stream stream;
	private byte[] buffer;
	private FileStream file;

	public State (int id, HttpWebRequest req)
	{
		this.id = id;
		request = req;
		handle = new ManualResetEvent (false);
		handleList.Add (handle);
	}

	public int Id {
		get { return id; }
	}

	public HttpWebRequest Request {
		get { return request; }
	}

	public Stream Stream {
		get { return stream; }
		set { stream = value; }
	}

	public byte[] Buffer {
		get {
			if (buffer == null)
				buffer = new byte [256]; // really small on purpose
			return buffer;
		}
	}

	public Stream File {
		get {
			if (file == null)
				file = new FileStream (id.ToString (), FileMode.Create);
			return file;
		}
	}

	public void Complete ()
	{
		if (file != null)
			file.Close ();
		handle.Set ();
	}

	static public void WaitAll ()
	{
		if (handleList.Count > 0) {
			WaitHandle[] handles = (WaitHandle[]) handleList.ToArray (typeof (WaitHandle));
			WaitHandle.WaitAll (handles);
			handleList.Clear ();
		}
	}
}

public class SaveTest {

	public static void Main (string[] args)
	{
		if (args.Length == 0) {
			Console.WriteLine ("usage: mono tlssave.exe url1 [url ...]");
			return;
		} else if (args.Length > 64) {
			Console.WriteLine ("WaitHandle has a limit of 64 handles so you cannot process {0} URLs.", args.Length);
			return;
		}

		ServicePointManager.CertificatePolicy = new TestCertificatePolicy ();

		int id = 1;
		foreach (string url in args) {
			Console.WriteLine ("GET #{0} at {1}", id, url);
			HttpWebRequest wreq = (HttpWebRequest) WebRequest.Create (url);
			State s = new State (id++, wreq);
			wreq.BeginGetResponse (new AsyncCallback (ResponseCallback), s);
		}

		State.WaitAll ();
	}

	private static void ResponseCallback (IAsyncResult result)
	{
		State state = ((State) result.AsyncState);
		HttpWebResponse response = (HttpWebResponse) state.Request.EndGetResponse (result);
		state.Stream = response.GetResponseStream ();
		state.Stream.BeginRead (state.Buffer, 0, state.Buffer.Length, new AsyncCallback (StreamCallBack), state);
	}

	private static void StreamCallBack (IAsyncResult result)
	{
		State state = ((State) result.AsyncState);
		int length = state.Stream.EndRead (result);
		if (length > 0) {
			state.File.Write (state.Buffer, 0, length);
			state.Stream.BeginRead (state.Buffer, 0, state.Buffer.Length, new AsyncCallback (StreamCallBack), state);
		} else {
			state.Stream.Close ();
			Console.WriteLine ("END #{0}", state.Id);
			state.Complete ();
		}
	}

	public class TestCertificatePolicy : ICertificatePolicy {

		public bool CheckValidationResult (ServicePoint sp, X509Certificate certificate, WebRequest request, int error)
		{
			// whatever the reason we do not stop the SSL connection
			return true;
		}
	}
}


text:
from gherkin.token_scanner import TokenScanner
from gherkin.token_matcher import TokenMatcher
from gherkin.parser import Parser
from gherkin.errors import ParserError
from nose.tools import assert_equals, assert_raises

def test_parser():
    parser = Parser()
    feature_file = parser.parse(TokenScanner("Feature: Foo"))
    expected = {
        'comments': [],
        'feature': {
            'keyword': u'Feature',
            'language': 'en',
            'location': {'column': 1, 'line': 1},
            'name': u'Foo',
            'children': [],
            'tags': [],
            'type': 'Feature'},
        'type': 'GherkinDocument'}

    assert_equals(expected, feature_file)

def test_parse_multiple_features():
    parser = Parser()
    ff1 = parser.parse(TokenScanner("Feature: 1"))
    ff2 = parser.parse(TokenScanner("Feature: 2"))

    assert_equals("1", ff1['feature']['name'])
    assert_equals("2", ff2['feature']['name'])

def test_parse_feature_after_parser_error():
    parser = Parser()
    with assert_raises(ParserError):
        parser.parse(TokenScanner('# a comment\n' +
                                  'Feature: Foo\n' +
                                  '  Scenario: Bar\n' +
                                  '    Given x\n' +
                                  '      ```\n' +
                                  '      unclosed docstring\n'))
    feature_file = parser.parse(TokenScanner('Feature: Foo\n' +
                                             '  Scenario: Bar\n' +
                                             '    Given x\n'
                                             '      """\n'
                                             '      closed docstring\n'
                                             '      """\n'))
    expected = [{
        'name': u'Bar',
        'keyword': u'Scenario',
        'tags': [],
        'steps': [{
            'text': u'x',
            'type': 'Step',
            'location': {'column': 5, 'line': 3},
            'keyword': u'Given ',
            'argument': {
                'content': u'closed docstring',
                'type': 'DocString',
                'location': {'column': 7, 'line': 4}}}],
        'location': {'column': 3, 'line': 2},
        'type': 'Scenario'}]

    assert_equals(expected, feature_file['feature']['children'])

def test_change_the_default_language():
    parser = Parser()
    matcher = TokenMatcher('no')
    feature_file = parser.parse(TokenScanner("Egenskap: i18n support - åæø"), matcher)
    expected = {
        'comments': [],
        'feature': {
            'keyword': u'Egenskap',
            'language': 'no',
            'location': {'column': 1, 'line': 1},
            'name': u'i18n support - åæø',
            'children': [],
            'tags': [],
            'type': 'Feature'},
        'type': 'GherkinDocument'}

    assert_equals(expected, feature_file)