text:
class SkAndroidFrameworkTraceUtil
{
public:
  SkAndroidFrameworkTraceUtil(const char* name)
  {
    if (CC_UNLIKELY(gEnableAndroidTracing))
    {
      ATRACE_BEGIN(name);
    }
  }
  SkAndroidFrameworkTraceUtil(bool, const char* fmt, ...)
  {
    if (CC_LIKELY((!gEnableAndroidTracing) || (!ATRACE_ENABLED())))
    {
      return ;
    }
    const int BUFFER_SIZE = 256;
    va_list ap;
    char buf[BUFFER_SIZE];
    va_start(ap, fmt);
    vsnprintf(buf, BUFFER_SIZE, fmt, ap);
    va_end(ap);
    ATRACE_BEGIN(buf);
  }
  ~SkAndroidFrameworkTraceUtil()
  {
    if (CC_UNLIKELY(gEnableAndroidTracing))
    {
      ATRACE_END();
    }
  }
  static void setEnableTracing(bool enableAndroidTracing)
  {
    gEnableAndroidTracing = enableAndroidTracing;
  }
  static bool getEnableTracing()
  {
    return gEnableAndroidTracing;
  }
private:
  static bool gEnableAndroidTracing;
};
#    define ATRACE_ANDROID_FRAMEWORK(fmt, ...) SkAndroidFrameworkTraceUtil __trace(true, fmt, ##__VA_ARGS__)
// Records a pair of begin and end events called "name" for the current scope, with 0, 1 or 2
// associated arguments. In the framework, the arguments are ignored.
#    define TRACE_EVENT0(category_group, name)	 \
    SkAndroidFrameworkTraceUtil __trace(name)
#    define TRACE_EVENT1(category_group, name, arg1_name, arg1_val)	 \
    SkAndroidFrameworkTraceUtil __trace(name)
#    define TRACE_EVENT2(category_group, name, arg1_name, arg1_val, arg2_name, arg2_val)	 \
    SkAndroidFrameworkTraceUtil __trace(name)
// Records a single event called "name" immediately, with 0, 1 or 2 associated arguments. If the
// category is not enabled, then this does nothing.
#    define TRACE_EVENT_INSTANT0(category_group, name, scope)	 \
    do { SkAndroidFrameworkTraceUtil __trace(name); } while(0)
#    define TRACE_EVENT_INSTANT1(category_group, name, scope, arg1_name, arg1_val)	 \
    do { SkAndroidFrameworkTraceUtil __trace(name); } while(0)
#    define TRACE_EVENT_INSTANT2(category_group, name, scope, arg1_name, arg1_val, \
                             arg2_name, arg2_val)                              \
    do { SkAndroidFrameworkTraceUtil __trace(name); } while(0)
// Records the value of a counter called "name" immediately. Value
// must be representable as a 32 bit integer.
#    define TRACE_COUNTER1(category_group, name, value)	 \
    if (CC_UNLIKELY(SkAndroidFrameworkTraceUtil::getEnableTracing())) { \
        ATRACE_INT(name, value); \
    }
// Records the values of a multi-parted counter called "name" immediately.
// In Chrome, this macro produces a stacked bar chart. ATrace doesn't support
// that, so this just produces two separate counters.
#    define TRACE_COUNTER2(category_group, name, value1_name, value1_val, value2_name, value2_val)	 \
    do { \
        if (CC_UNLIKELY(SkAndroidFrameworkTraceUtil::getEnableTracing())) { \
            ATRACE_INT(name "-" value1_name, value1_val); \
            ATRACE_INT(name "-" value2_name, value2_val); \
        } \
    } while (0)
// ATrace has no object tracking
#    define TRACE_EVENT_OBJECT_CREATED_WITH_ID(category_group, name, id)	 TRACE_EMPTY
#    define TRACE_EVENT_OBJECT_SNAPSHOT_WITH_ID(category_group, name, id, snapshot)	 TRACE_EMPTY
#    define TRACE_EVENT_OBJECT_DELETED_WITH_ID(category_group, name, id)	 TRACE_EMPTY
// Macro to efficiently determine if a given category group is enabled.
// This is only used for some shader text logging that isn't supported in ATrace anyway.
#    define TRACE_EVENT_CATEGORY_GROUP_ENABLED(category_group, ret)	             \
  do { *ret = false; } while (0)
#  else
#    define ATRACE_ANDROID_FRAMEWORK(fmt, ...)	 TRACE_EMPTY
// Records a pair of begin and end events called "name" for the current scope, with 0, 1 or 2
// associated arguments. If the category is not enabled, then this does nothing.
#    define TRACE_EVENT0(category_group, name)	 \
  INTERNAL_TRACE_EVENT_ADD_SCOPED(category_group, name)
#    define TRACE_EVENT1(category_group, name, arg1_name, arg1_val)	 \
  INTERNAL_TRACE_EVENT_ADD_SCOPED(category_group, name, arg1_name, arg1_val)
#    define TRACE_EVENT2(category_group, name, arg1_name, arg1_val, arg2_name, arg2_val)	 \
  INTERNAL_TRACE_EVENT_ADD_SCOPED(category_group, name, arg1_name, arg1_val, arg2_name, arg2_val)
// Records a single event called "name" immediately, with 0, 1 or 2 associated arguments. If the
// category is not enabled, then this does nothing.
#    define TRACE_EVENT_INSTANT0(category_group, name, scope)	                   \
  INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category_group, name, \
                           TRACE_EVENT_FLAG_NONE | scope)
#    define TRACE_EVENT_INSTANT1(category_group, name, scope, arg1_name, arg1_val)	 \
  INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category_group, name,    \
                           TRACE_EVENT_FLAG_NONE | scope, arg1_name, arg1_val)
#    define TRACE_EVENT_INSTANT2(category_group, name, scope, arg1_name, arg1_val, \
                             arg2_name, arg2_val)                              \
  INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category_group, name,    \
                           TRACE_EVENT_FLAG_NONE | scope, arg1_name, arg1_val, \
                           arg2_name, arg2_val)
// Records the value of a counter called "name" immediately. Value
// must be representable as a 32 bit integer.
#    define TRACE_COUNTER1(category_group, name, value)	                         \
  INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category_group, name, \
                           TRACE_EVENT_FLAG_NONE, "value",                  \
                           static_cast<int>(value))
// Records the values of a multi-parted counter called "name" immediately.
// The UI will treat value1 and value2 as parts of a whole, displaying their
// values as a stacked-bar chart.
#    define TRACE_COUNTER2(category_group, name, value1_name, value1_val,       \
                       value2_name, value2_val)                             \
  INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category_group, name, \
                           TRACE_EVENT_FLAG_NONE, value1_name,              \
                           static_cast<int>(value1_val), value2_name,       \
                           static_cast<int>(value2_val))
#    define TRACE_EVENT_ASYNC_BEGIN0(category, name, id)	                                           \
    INTERNAL_TRACE_EVENT_ADD_WITH_ID(                                                          \
        TRACE_EVENT_PHASE_ASYNC_BEGIN, category, name, id, TRACE_EVENT_FLAG_NONE)
#    define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val)	                      \
    INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN,                            \
        category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val)
#    define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, arg2_name, arg2_val)	 \
    INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN,                            \
        category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, arg2_name, arg2_val)
#    define TRACE_EVENT_ASYNC_END0(category, name, id)	                                           \
    INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END,                            \
        category, name, id, TRACE_EVENT_FLAG_NONE)
#    define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val)	                      \
    INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END,                            \
        category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val)
#    define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, arg2_name, arg2_val)	 \
    INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END,                            \
        category, name, id, TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, arg2_name, arg2_val)
// Macros to track the life time and value of arbitrary client objects.
#    define TRACE_EVENT_OBJECT_CREATED_WITH_ID(category_group, name, id)	 \
  INTERNAL_TRACE_EVENT_ADD_WITH_ID(                                  \
      TRACE_EVENT_PHASE_CREATE_OBJECT, category_group, name, id,     \
      TRACE_EVENT_FLAG_NONE)
#    define TRACE_EVENT_OBJECT_SNAPSHOT_WITH_ID(category_group, name, id, \
                                            snapshot)                 \
  INTERNAL_TRACE_EVENT_ADD_WITH_ID(                                   \
      TRACE_EVENT_PHASE_SNAPSHOT_OBJECT, category_group, name,        \
      id, TRACE_EVENT_FLAG_NONE, "snapshot", snapshot)
#    define TRACE_EVENT_OBJECT_DELETED_WITH_ID(category_group, name, id)	 \
  INTERNAL_TRACE_EVENT_ADD_WITH_ID(                                  \
      TRACE_EVENT_PHASE_DELETE_OBJECT, category_group, name, id,     \
      TRACE_EVENT_FLAG_NONE)
// Macro to efficiently determine if a given category group is enabled.
#    define TRACE_EVENT_CATEGORY_GROUP_ENABLED(category_group, ret)	             \
  do {                                                                      \
    INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category_group);                 \
    if (INTERNAL_TRACE_EVENT_CATEGORY_GROUP_ENABLED_FOR_RECORDING_MODE()) { \
      *ret = true;                                                          \
    } else {                                                                \
      *ret = false;                                                         \
    }                                                                       \
  } while (0)
#  endif
// Flags for changing the behavior of TRACE_EVENT_API_ADD_TRACE_EVENT.
#  define TRACE_EVENT_FLAG_NONE(static_cast<unsigned int>(0))
#  define TRACE_EVENT_FLAG_COPY(static_cast<unsigned int>(1 << 0))
#  define TRACE_EVENT_FLAG_HAS_ID(static_cast<unsigned int>(1 << 1))
#  define TRACE_EVENT_FLAG_MANGLE_ID(static_cast<unsigned int>(1 << 2))
#  define TRACE_EVENT_FLAG_SCOPE_OFFSET(static_cast<unsigned int>(1 << 3))
#  define TRACE_EVENT_FLAG_SCOPE_EXTRA(static_cast<unsigned int>(1 << 4))
#  define TRACE_EVENT_FLAG_EXPLICIT_TIMESTAMP(static_cast<unsigned int>(1 << 5))
#  define TRACE_EVENT_FLAG_ASYNC_TTS(static_cast<unsigned int>(1 << 6))
#  define TRACE_EVENT_FLAG_BIND_TO_ENCLOSING(static_cast<unsigned int>(1 << 7))
#  define TRACE_EVENT_FLAG_FLOW_IN(static_cast<unsigned int>(1 << 8))
#  define TRACE_EVENT_FLAG_FLOW_OUT(static_cast<unsigned int>(1 << 9))
#  define TRACE_EVENT_FLAG_HAS_CONTEXT_ID(static_cast<unsigned int>(1 << 10))
#  define TRACE_EVENT_FLAG_SCOPE_MASK(static_cast<unsigned int>(TRACE_EVENT_FLAG_SCOPE_OFFSET | \
                             TRACE_EVENT_FLAG_SCOPE_EXTRA))
// Type values for identifying types in the TraceValue union.
#  define TRACE_VALUE_TYPE_BOOL(static_cast<unsigned char>(1))
#  define TRACE_VALUE_TYPE_UINT(static_cast<unsigned char>(2))
#  define TRACE_VALUE_TYPE_INT(static_cast<unsigned char>(3))
#  define TRACE_VALUE_TYPE_DOUBLE(static_cast<unsigned char>(4))
#  define TRACE_VALUE_TYPE_POINTER(static_cast<unsigned char>(5))
#  define TRACE_VALUE_TYPE_STRING(static_cast<unsigned char>(6))
#  define TRACE_VALUE_TYPE_COPY_STRING(static_cast<unsigned char>(7))
#  define TRACE_VALUE_TYPE_CONVERTABLE(static_cast<unsigned char>(8))
// Enum reflecting the scope of an INSTANT event. Must fit within TRACE_EVENT_FLAG_SCOPE_MASK.
#  define TRACE_EVENT_SCOPE_GLOBAL(static_cast<unsigned char>(0 << 3))
#  define TRACE_EVENT_SCOPE_PROCESS(static_cast<unsigned char>(1 << 3))
#  define TRACE_EVENT_SCOPE_THREAD(static_cast<unsigned char>(2 << 3))
#  define TRACE_EVENT_SCOPE_NAME_GLOBAL('g')
#  define TRACE_EVENT_SCOPE_NAME_PROCESS('p')
#  define TRACE_EVENT_SCOPE_NAME_THREAD('t')
#endif


text: Powered by Django

redpajama_set_name: RedPajamaGithub


text: service mysql start


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
	"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

	<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
	<title>Jack Tripper</title>

	<link rel="stylesheet" href="http://devfiles.myopera.com/articles/658/style1.css" type="text/css" media="screen" title="no title" charset="utf-8"/>

Tripper, Jack

Jack Tripper

Jack's Bistro

834 Ocean Vista Ave.

          Santa Monica,
          CA,
          90405

(310) 444-8444

jackthetripper@example.com


text:
<?php
declare(strict_types=1);

require __DIR__ . '/../../app/iot/Init.php';
$app->bootstrap()->run();


text:
Changelog
=========

1.2.2 (27-7-2015)
-----------------
- msm estimation: new fast transition matrix sampler
- msm estimation: new feature "auto-sparse": automatically decide which datatype
  to use for transition matrix estimation.
- coordinates package: kinetic map feature for TICA (arXiv:1506.06259 [physics.comp-ph])
- coordinates package: better examples for API functions.
- coordinates package: cluster assignment bugfix in parallel environments (OpenMP).
- coordinates package: added cos/sin transformations for angle based features to
  featurizer. This is recommended for PCA/TICA transformations.
- coordinates package: added minimum RMSD feature to featurizer.
- coordinates package: Regular space clustering terminates early now, when it reaches
  max_clusters cutoff.
- plots package: use Fruchterman Reingold spring algorithm to calculate positions
  in network plots.
- ipython notebooks: new real-world examples, which show the complete workflow
- general: made all example codes in documentation work.

1.2.1 (28-5-2015)
-----------------
- general: Time consuming algorithms now display progressbars (optional).
- general: removed scikit-learn dependency (due to new kmeans impl. Thanks @clonker)
- coordinates package: new and faster implementation of Kmeans (10x faster than scikit-learn).
- coordinates package: allow metrics to be passed to cluster algorithms.
- coordinates package: cache trajectory lengths by default -
                       (uncached led to 1 pass of reading for non indexed (XTC) formats).
  This avoids re-reading e.g XTC files to determine their lengths.
- coordinates package: enable passing chunk size to readers and pipelines in API.
- coordinates package: assign_to_centers now allows all supported file formats as centers input.
- coordinates package: save_traj(s) now handles stride parameter.
- coordinates package: save_traj    now accepts also lists of files as an input
  In this case, an extra parameter topfile has to be parsed as well.
- plots package: added functions to plot flux and msm models.
- Bugfixes:
    - [msm.MSM.pcca]: coarse-grained transition matrix corrected
    - [msm.generation]: stopping states option fixed
    - [coordinates.NumPyReader]: during gathering of shapes of all files, none
                                 of them were closed. Thanks @gph82

1.2 (4-14-2015)
---------------
1.2 is a major new release which offers a load of new and useful functionalities
for coordinate loading, data processing and Markov model estimation and analysis.
In a few places we had to change existing API functions, but we encourage
everyone to update to 1.2.

- coordinate package: featurizer can be constructed separately
- coordinate package: new functions for loading data and creating file readers
  for large trajectories
- coordinate package: all clustering functions were renamed
  (e.g.: kmeans -> cluster_kmeans). Old function names do still work, but are deprecated
- coordinate package: new pipeline() function for generic data processing pipelines.
  Using pipelines you can go from data loading, over transformation via TICA or PCA,
  to clustered data all via stream processing. This avoids having to load large
  datasets into memory.
- msm package: markov_model() function creates a MSM object that offers a lot
  of analysis functions such as spectral analysis, mean first passage times,
  pcca, calculation of experimental observables, etc.
- msm package: estimate_markov_model() function creates a EstimatedMSM object
  from data. Offers all functionalities of MSM plus additional functions related
  to trajectories, such as drawing representative smaples for MSM states
- msm package: Chapman-Kolmogorow test and implied timescales calculation are more robust
- msm package: cktest() and tpt() functions now accept MSM objects as inputs
- various bug fixes

1.1.2 (3-18-2015)
-----------------

- PCCA++ now produces correct memberships (fixes a problem from nonorthonormal eigenvectors)
- Improved Coordinates API documentation (Examples, examples, EXAMPLES)


text:
import argparse
import gzip
import hashlib
import logging
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
from typing import Any, IO, Iterable, List, Optional, Tuple
import urllib.request
import zipfile

# Make sure sockeye is on the system path
try:
    from sockeye import constants as C
    from sockeye import utils
except ImportError:
    SOCKEYE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    raise RuntimeError("Please install the sockeye module or add the sockeye root directory to your Python path. Ex: export PYTHONPATH=%s"
                       % SOCKEYE_ROOT)

from sockeye_contrib.autopilot.tasks import ARCHIVE_NONE, ARCHIVE_TAR, ARCHIVE_ZIP
from sockeye_contrib.autopilot.tasks import TEXT_UTF8_RAW, TEXT_UTF8_RAW_SGML, TEXT_UTF8_RAW_BITEXT
from sockeye_contrib.autopilot.tasks import TEXT_UTF8_RAW_BITEXT_REVERSE, TEXT_REQUIRES_TOKENIZATION
from sockeye_contrib.autopilot.tasks import TEXT_UTF8_TOKENIZED
from sockeye_contrib.autopilot.tasks import RAW_FILES
from sockeye_contrib.autopilot.tasks import Task, TASKS
from sockeye_contrib.autopilot.models import MODELS, MODEL_NONE, MODEL_GNMT, MODEL_TEST_ARGS
from sockeye_contrib.autopilot.models import DECODE_ARGS, DECODE_STANDARD, DECODE_GNMT
from sockeye_contrib.autopilot import third_party

# Formats for custom files
CUSTOM_UTF8_RAW = "raw"
CUSTOM_UTF8_TOK = "tok"
CUSTOM_UTF8_BPE = "bpe"
CUSTOM_TEXT_TYPES = [CUSTOM_UTF8_RAW, CUSTOM_UTF8_TOK, CUSTOM_UTF8_BPE]

# Special file names
DIR_SOCKEYE_AUTOPILOT = "sockeye_autopilot"
FILE_WORKSPACE = ".workspace"
FILE_COMPLETE = ".complete"

# Sub-task directory and file names
DIR_CACHE = "cache"
DIR_LOGS = "logs"
DIR_SYSTEMS = "systems"
DIR_DATA = "data"
DIR_RAW = "raw"
DIR_TOK = "tok"
DIR_BPE = "bpe"
PREFIX_TRAIN = "train."
PREFIX_DEV = "dev."
PREFIX_TEST = "test."
DATA_SRC = "src"
DATA_TRG = "trg"
SUFFIX_SRC_GZ = DATA_SRC + ".gz"
SUFFIX_TRG_GZ = DATA_TRG + ".gz"
DIR_BPE_MODEL = "model.bpe"
FILE_BPE_CODES = "codes"
DIR_PREFIX_MODEL = "model."
DIR_RESULTS = "results"
FILE_COMMAND = "command.{}.sh"
SUFFIX_COMMAND = "command.sh"
SUFFIX_BPE = "bpe"
SUFFIX_TOK = "tok"
SUFFIX_DETOK = "detok"
SUFFIX_BLEU = "bleu"
SUFFIX_SACREBLEU = "sacrebleu"
SUFFIX_TEST = ".test"

# Reasonable defaults for model averaging
AVERAGE_NUM_CHECKPOINTS = 8
AVERAGE_METRIC = "perplexity"
AVERAGE_STRATEGY = "best"
PARAMS_BEST_SINGLE = "params.best.single"
PARAMS_AVERAGE = "params.average"

# Scaled down settings for test mode
TEST_BPE_OPS = 1024

def identify_raw_files(task: Task, test_mode: bool = False) -> List[str]:
    """
    Identify raw files that need to be downloaded for a given task.

    :param task: Sequence-to-sequence task.
    :param test_mode: Run in test mode, only downloading test data.
    :return: List of raw file names.
    """
    raw_files = set()
    all_sets = [task.test,] if test_mode else [task.train, task.dev, task.test]
    for file_sets in all_sets:
        for file_set in file_sets:
            for fname in file_set[:2]:
                raw_file = fname.split("/", 1)[0]
                if raw_file not in RAW_FILES:
                    raise RuntimeError("Unknown raw file %s found in path %s" % (raw_file, fname))
                raw_files.add(raw_file)
    return sorted(raw_files)

def download_extract_raw_files(names: List[str], cache_dir: str, dest_dir: str):
    """
    Download and extract raw files, making use of a cache directory.
    - Downloaded files are verified by MD5 sum.
    - Extraction overwrites existing files.

    :param names: List of raw file names in RAW_FILES.
    :param cache_dir: Cache directory for downloading raw files.
    :param dest_dir: Destination directory for extracting raw files.
    """

    for name in names:
        raw_file = RAW_FILES[name]
        local_dir = os.path.join(cache_dir, name)
        local_fname = os.path.join(local_dir, os.path.basename(raw_file.url))

        # Download file if not present
        if not os.path.exists(local_dir):
            logging.info("Create: %s", local_dir)
            os.makedirs(local_dir)
        if not os.path.exists(local_fname):
            logging.info("Download: %s -> %s", raw_file.url, local_fname)
            urllib.request.urlretrieve(raw_file.url, local_fname)

        # Check MD5 sum, attempt one re-download on mismatch
        md5 = md5sum(local_fname)
        if not md5 == raw_file.md5:
            logging.info("MD5 mismatch for %s, attempt re-download %s", local_fname, raw_file.url)
            urllib.request.urlretrieve(raw_file.url, local_fname)
            md5 = md5sum(local_fname)
            if not md5 == raw_file.md5:
                raise RuntimeError("MD5 mismatch for %s after re-download.  Check validity of %s"
                                   % (local_fname, raw_file.url))
        logging.info("Confirmed MD5: %s (%s)", local_fname, md5)

        # Extract file(s), overwriting directory if exists
        extract_path = os.path.join(dest_dir, name)
        if os.path.exists(extract_path):
            shutil.rmtree(extract_path)
        os.makedirs(extract_path)
        logging.info("Extract: %s -> %s", local_fname, extract_path)
        if raw_file.archive_type == ARCHIVE_NONE:
            os.symlink(local_fname, os.path.join(extract_path, os.path.basename(local_fname)))
        elif raw_file.archive_type == ARCHIVE_TAR:
            tar = tarfile.open(local_fname)
            tar.extractall(path=extract_path)
        elif raw_file.archive_type == ARCHIVE_ZIP:
            zipf = zipfile.ZipFile(local_fname, "r")
            zipf.extractall(path=extract_path)
        else:
            raise RuntimeError("Unknown archive type: %s" % raw_file.archive_type)

def md5sum(fname: str) -> str:
    """Compute MD5 sum of file."""
    with open(fname, "rb") as inp:
        md5 = hashlib.md5(inp.read()).hexdigest()
    return md5

def populate_parallel_text(extract_dir: str,
                           file_sets: List[Tuple[str, str, str]],
                           dest_prefix: str,
                           keep_separate: bool,
                           head_n: int = 0):
    """
    Create raw parallel train, dev, or test files with a given prefix.

    :param extract_dir: Directory where raw files (inputs) are extracted.
    :param file_sets: Sets of files to use.
    :param dest_prefix: Prefix for output files.
    :param keep_separate: True if each file set (source-target pair) should have
                          its own file (used for test sets).
    :param head_n: If N>0, use only the first N lines (used in test mode).
    """
    source_out = None  # type: IO[Any]
    target_out = None  # type: IO[Any]
    lines_written = 0
    # Single output file for each side
    if not keep_separate:
        source_dest = dest_prefix + SUFFIX_SRC_GZ
        target_dest = dest_prefix + SUFFIX_TRG_GZ
        logging.info("Populate: %s %s", source_dest, target_dest)
        source_out = gzip.open(source_dest, "wt", encoding="utf-8")
        target_out = gzip.open(target_dest, "wt", encoding="utf-8")
    for i, (source_fname, target_fname, text_type) in enumerate(file_sets):
        # One output file per input file for each side
        if keep_separate:
            if source_out:
                source_out.close()
            if target_out:
                target_out.close()
            source_dest = dest_prefix + str(i) + "." + SUFFIX_SRC_GZ
            target_dest = dest_prefix + str(i) + "." + SUFFIX_TRG_GZ
            logging.info("Populate: %s %s", source_dest, target_dest)
            source_out = gzip.open(source_dest, "wt", encoding="utf-8")
            target_out = gzip.open(target_dest, "wt", encoding="utf-8")
        for source_line, target_line in zip(
                plain_text_iter(os.path.join(extract_dir, source_fname), text_type, DATA_SRC),
                plain_text_iter(os.path.join(extract_dir, target_fname), text_type, DATA_TRG)):
            # Only write N lines total if requested, but reset per file when
            # keeping files separate
            if head_n > 0 and lines_written >= head_n:
                if keep_separate:
                    lines_written = 0
                break
            source_out.write("{}\n".format(source_line))
            target_out.write("{}\n".format(target_line))
            lines_written += 1
    source_out.close()
    target_out.close()

def copy_parallel_text(file_list: List[str], dest_prefix: str):
    """
    Copy pre-compiled raw parallel files with a given prefix.  Perform
    whitespace character normalization to ensure that only ASCII newlines are
    considered line breaks.

    :param file_list: List of file pairs to use.
    :param dest_prefix: Prefix for output files.
    """
    # Group files into source-target pairs
    file_sets = []
    for i in range(0, len(file_list), 2):
        file_sets.append((file_list[i], file_list[i + 1]))
    multiple_sets = len(file_sets) > 1
    for i, (source_fname, target_fname) in enumerate(file_sets):
        if multiple_sets:
            source_dest = dest_prefix + str(i) + "." + SUFFIX_SRC_GZ
            target_dest = dest_prefix + str(i) + "." + SUFFIX_TRG_GZ
        else:
            source_dest = dest_prefix + SUFFIX_SRC_GZ
            target_dest = dest_prefix + SUFFIX_TRG_GZ
        logging.info("Populate: %s %s", source_dest, target_dest)
        with gzip.open(source_dest, "wb") as source_out, gzip.open(target_dest, "wb") as target_out:
            with third_party.bin_open(source_fname) as inp:
                for line in inp:
                    line = (re.sub(r"\s", " ", line.decode("utf-8"))).encode("utf-8") + b"\n"
                    source_out.write(line)
            with third_party.bin_open(target_fname) as inp:
                for line in inp:
                    line = (re.sub(r"\s", " ", line.decode("utf-8"))).encode("utf-8") + b"\n"
                    target_out.write(line)

def plain_text_iter(fname: str, text_type: str, data_side: str) -> Iterable[str]:
    """
    Extract plain text from file as iterable.  Also take steps to ensure that
    whitespace characters (including unicode newlines) are normalized and
    outputs are line-parallel with inputs considering ASCII newlines only.

    :param fname: Path of possibly gzipped input file.
    :param text_type: One of TEXT_*, indicating data format.
    :param data_side: DATA_SRC or DATA_TRG.
    """
    if text_type in (TEXT_UTF8_RAW, TEXT_UTF8_TOKENIZED):
        with third_party.bin_open(fname) as inp:
            for line in inp:
                line = re.sub(r"\s", " ", line.decode("utf-8"))
                yield line.strip()
    elif text_type == TEXT_UTF8_RAW_SGML:
        with third_party.bin_open(fname) as inp:
            for line in inp:
                line = re.sub(r"\s", " ", line.decode("utf-8"))
                if line.startswith("<seg "):
                    # Extract segment text
                    text = re.sub(r"<seg.*?>(.*)</seg>.*?", "\\1", line)
                    text = re.sub(r"\s+", " ", text.strip())
                    # Unescape XML entities
                    text = text.replace(""", "\"")
                    text = text.replace("'", "'")
                    text = text.replace("<", "<")
                    text = text.replace(">", ">")
                    text = text.replace("&", "&")
                    yield text
    elif text_type in (TEXT_UTF8_RAW_BITEXT, TEXT_UTF8_RAW_BITEXT_REVERSE):
        # Select source or target field, reversing if needed
        if text_type == TEXT_UTF8_RAW_BITEXT:
            field_id = 0 if data_side == DATA_SRC else 1
        else:
            field_id = 1 if data_side == DATA_SRC else 0
        with third_party.bin_open(fname) as inp:
            for line in inp:
                line = re.sub(r"\s", " ", line.decode("utf-8"))
                fields = line.split("|||")
                yield fields[field_id].strip()
    else:
        raise RuntimeError("Unknown text type: %s" % text_type)

def touch_file(fname: str):
    """Create a file if not present, update access time."""
    # Reference not needed since there will be no reads or writes
    with open(fname, "a"):
        os.utime(fname, None)

def renew_step_dir(step_dir: str):
    """Delete step directory if exists and create, reporting actions."""
    if os.path.exists(step_dir):
        logging.info("Remove unfinished step %s", step_dir)
        shutil.rmtree(step_dir)
    logging.info("Create: %s", step_dir)
    os.makedirs(step_dir)

def call_sockeye_train(model: str,
                       bpe_dir: str,
                       model_dir: str,
                       log_fname: str,
                       num_gpus: int,
                       test_mode: bool = False):
    """
    Call sockeye.train with specified arguments on prepared inputs.  Will resume
    partial training or skip training if model is already finished.  Record
    command for future use.

    :param model: Type of translation model to train.
    :param bpe_dir: Directory of BPE-encoded input data.
    :param model_dir: Model output directory.
    :param log_fname: Location to write log file.
    :param num_gpus: Number of GPUs to use for training (0 for CPU).
    :param test_mode: Run in test mode, stopping after a small number of
                      updates.
    """
    # Inputs and outputs
    fnames = ["--source={}".format(os.path.join(bpe_dir, PREFIX_TRAIN + SUFFIX_SRC_GZ)),
              "--target={}".format(os.path.join(bpe_dir, PREFIX_TRAIN + SUFFIX_TRG_GZ)),
              "--validation-source={}".format(os.path.join(bpe_dir, PREFIX_DEV + SUFFIX_SRC_GZ)),
              "--validation-target={}".format(os.path.join(bpe_dir, PREFIX_DEV + SUFFIX_TRG_GZ)),
              "--output={}".format(model_dir)]
    # Assemble command
    command = [sys.executable, "-m", "sockeye.train"] + fnames + MODELS[model]
    # Request GPUs or specify CPU
    if num_gpus > 0:
        command.append("--device-ids=-{}".format(num_gpus))
    else:
        command.append("--use-cpu")
    # Test mode trains a smaller model for a small number of steps
    if test_mode:
        command += MODEL_TEST_ARGS[model]
    command_fname = os.path.join(model_dir, FILE_COMMAND.format("sockeye.train"))
    # Run unless training already finished
    if not os.path.exists(command_fname):
        # Call Sockeye training
        with open(log_fname, "wb") as log:
            logging.info("sockeye.train: %s", model_dir)
            logging.info("Log: %s", log_fname)
            logging.info("(This step can take several days. See log file or TensorBoard for progress)")
            subprocess.check_call(command, stderr=log)
        # Record successful command
        logging.info("Command: %s", command_fname)
        print_command(command, command_fname)

def call_sockeye_average(model_dir: str, log_fname: str):
    """
    Call sockeye.average with reasonable defaults.

    :param model_dir: Trained model directory.
    :param log_fname: Location to write log file.
    """
    params_best_fname = os.path.join(model_dir, C.PARAMS_BEST_NAME)
    params_best_single_fname = os.path.join(model_dir, PARAMS_BEST_SINGLE)
    params_average_fname = os.path.join(model_dir, PARAMS_AVERAGE)
    command = [sys.executable,
               "-m",
               "sockeye.average",
               "--metric={}".format(AVERAGE_METRIC),
               "-n",
               str(AVERAGE_NUM_CHECKPOINTS),
               "--output={}".format(params_average_fname),
               "--strategy={}".format(AVERAGE_STRATEGY),
               model_dir]
    command_fname = os.path.join(model_dir, FILE_COMMAND.format("sockeye.average"))
    # Run average if not previously run
    if not os.path.exists(command_fname):
        # Re-link best point to best single point
        os.symlink(os.path.basename(os.path.realpath(params_best_fname)), params_best_single_fname)
        os.remove(params_best_fname)
        # Call Sockeye average
        with open(log_fname, "wb") as log:
            logging.info("sockeye.average: %s", os.path.join(model_dir, params_best_fname))
            logging.info("Log: %s", log_fname)
            subprocess.check_call(command, stderr=log)
        # Link averaged point as new best
        os.symlink(PARAMS_AVERAGE, params_best_fname)
        # Record successful command
        logging.info("Command: %s", command_fname)
        print_command(command, command_fname)

def call_sockeye_translate(args: List[str],
                           input_fname: str,
                           output_fname: str,
                           model_dir: str,
                           log_fname: str,
                           use_cpu: bool):
    """
    Call sockeye.translate with specified arguments using a trained model.

    :param args: Command line arguments for sockeye.translate.
    :param input_fname: Input file (byte-pair encoded).
    :param output_fname: Raw decoder output file.
    :param model_dir: Model output directory.
    :param log_fname: Location to write log file.
    :param use_cpu: Use CPU instead of GPU for decoding.
    """
    # Inputs and outputs
    fnames = ["--input={}".format(input_fname),
              "--output={}".format(output_fname),
              "--models={}".format(model_dir)]
    # Assemble command
    command = [sys.executable, "-m", "sockeye.translate"] + fnames + args
    # Request GPUs or specify CPU
    if use_cpu:
        command.append("--use-cpu")
    command_fname = output_fname + "." + SUFFIX_COMMAND
    # Run unless translate already finished
    if not os.path.exists(command_fname):
        # Call Sockeye translate
        with open(log_fname, "wb") as log:
            logging.info("sockeye.translate: %s -> %s", input_fname, output_fname)
            logging.info("Log: %s", log_fname)
            subprocess.check_call(command, stderr=log)
        # Cleanup redundant log file
        try:
            os.remove(output_fname + ".log")
        except FileNotFoundError:
            pass

        # Record successful command
        logging.info("Command: %s", command_fname)
        print_command(command, command_fname)

def call_sacrebleu(input_fname: str, ref_fname: str, output_fname: str, log_fname: str, tokenized: bool = False):
    """
    Call sockeye_contrib.sacrebleu.sacrebleu on tokenized or detokenized inputs.

    :param input_fname: Input translation file.
    :param ref_fname: Reference translation file.
    :param output_fname: Output score file.
    :param log_fname: Location to write log file.
    :param tokenized: Whether inputs are tokenized (or byte-pair encoded).
    """
    # Assemble command
    command = [sys.executable,
               "-m",
               "sockeye_contrib.sacrebleu.sacrebleu",
               "--score-only",
               "--input={}".format(input_fname),
               ref_fname]
    # Already tokenized?
    if tokenized:
        command.append("--tokenize=none")
    # Call sacrebleu
    with open(log_fname, "wb") as log:
        logging.info("sockeye_contrib.sacrebleu.sacrebleu: %s -> %s", input_fname, output_fname)
        logging.info("Log: %s", log_fname)
        score = subprocess.check_output(command, stderr=log)
    # Record successful score
    with open(output_fname, "wb") as out:
        out.write(score)

def print_command(command: List[str], fname: str):
    """
    Format and print command to file.

    :param command: Command in args list form.
    :param fname: File name to write out.
    """
    with open(fname, "w", encoding="utf-8") as out:
        print(" \\\n".join(command), file=out)

def run_steps(args: argparse.Namespace):
    """Run all steps required to complete task.  Called directly from main."""

    logging.basicConfig(level=logging.INFO, format="sockeye.autopilot: %(message)s")

    # (1) Establish task

    logging.info("=== Start Autopilot ===")
    # Listed task
    if args.task:
        task = TASKS[args.task]
        logging.info("Task: %s", task.description)
        logging.info("URL: %s", task.url)

        def report_data(file_sets):
            for file_set in file_sets:
                for fname in file_set[:2]:
                    logging.info("    %s", fname)

        logging.info("  Train:")
        report_data(task.train)
        logging.info("  Dev:")
        report_data(task.dev)
        logging.info("  Test:")
        report_data(task.test)
    # Custom task
    else:
        logging.info("Task: custom")
    # Source and target language codes
    lang_codes = (task.src_lang, task.trg_lang) if args.task else args.custom_lang

    # (2) Establish workspace and task directories

    logging.info("=== Establish working directories ===")
    logging.info("Workspace: %s", args.workspace)
    special_fname = os.path.join(args.workspace, FILE_WORKSPACE)
    if not os.path.exists(args.workspace):
        logging.info("Create: %s", args.workspace)
        os.makedirs(args.workspace)
        touch_file(special_fname)
    else:
        if not os.path.exists(special_fname):
            raise RuntimeError("Directory %s exists but %s does not, stopping to avoid overwriting files in non-workspace directory"
                            % (args.workspace, special_fname))

    dir_third_party = os.path.join(args.workspace, third_party.DIR_THIRD_PARTY)
    dir_cache = os.path.join(args.workspace, DIR_CACHE)
    dir_logs = os.path.join(args.workspace, DIR_LOGS)
    dir_systems = os.path.join(args.workspace, DIR_SYSTEMS)
    task_name = args.task if args.task else args.custom_task
    if args.test:
        task_name += SUFFIX_TEST
    dir_task = os.path.join(dir_systems, task_name)
    for dirname in (dir_third_party, dir_cache, dir_logs, dir_systems, dir_task):
        if os.path.exists(dirname):
            logging.info("Exists: %s", dirname)
        else:
            logging.info("Create: %s", dirname)
            os.makedirs(dirname)

    # (3) Checkout necessary tools

    logging.info("=== Checkout third-party tools ===")
    # Requires tokenization?
    if args.task or args.custom_text_type == CUSTOM_UTF8_RAW:
        third_party.checkout_moses_tokenizer(args.workspace)
    # Requires byte-pair encoding?
    if args.task or args.custom_text_type in (CUSTOM_UTF8_RAW, CUSTOM_UTF8_TOK):
        third_party.checkout_subword_nmt(args.workspace)

    # (4) Populate train/dev/test data

    # This step also normalizes whitespace on data population or copy, ensuring
    # that for all input data, only ASCII newlines are considered line breaks.
    logging.info("=== Populate train/dev/test data ===")
    step_dir_raw = os.path.join(dir_task, DIR_DATA, DIR_RAW)
    complete_fname = os.path.join(step_dir_raw, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_raw)
    else:
        # Listed task
        if args.task:
            raw_files = identify_raw_files(task, test_mode=args.test)
            with tempfile.TemporaryDirectory(prefix="raw.", dir=dir_task) as raw_dir:
                # Download (or locate in cache) and extract raw files to temp directory
                logging.info("=== Download and extract raw files ===")
                download_extract_raw_files(raw_files, dir_cache, raw_dir)
                # Copy required files to train/dev/test
                logging.info("=== Create input data files ===")
                renew_step_dir(step_dir_raw)
                # Test mode uses the full test set as training data and the
                # first line of the test set as dev and test data
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.train,
                                       os.path.join(step_dir_raw, PREFIX_TRAIN),
                                       False)
                populate_parallel_text(raw_dir,
                                       task.test if args.test else task.dev,
                                       os.path.join(step_dir_raw, PREFIX_DEV),
                                       False,
                                       head_n=1 if args.test else 0)
                populate_parallel_text(raw_dir,
                                       task.test,
                                       os.path.join(step_dir_raw, PREFIX_TEST),
                                       True,
                                       head_n=1 if args.test else 0)
        # Custom task
        else:
            logging.info("=== Copy input data files ===")
            renew_step_dir(step_dir_raw)
            copy_parallel_text(args.custom_train, os.path.join(step_dir_raw, PREFIX_TRAIN))
            copy_parallel_text(args.custom_dev, os.path.join(step_dir_raw, PREFIX_DEV))
            copy_parallel_text(args.custom_test, os.path.join(step_dir_raw, PREFIX_TEST))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_raw)

    # (5) Tokenize train/dev/test data

    # Task requires tokenization if _any_ raw file is not already tokenized
    requires_tokenization = False
    if args.task:
        for file_sets in (task.train, task.dev, task.test):
            for _, _, text_type in file_sets:
                if text_type in TEXT_REQUIRES_TOKENIZATION:
                    requires_tokenization = True
    else:
        if args.custom_text_type == CUSTOM_UTF8_RAW:
            requires_tokenization = True
    logging.info("=== Tokenize train/dev/test data ===")
    step_dir_tok = os.path.join(dir_task, DIR_DATA, DIR_TOK)
    complete_fname = os.path.join(step_dir_tok, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_tok)
    else:
        renew_step_dir(step_dir_tok)

        # Tokenize each data file using the appropriate language code OR link
        # raw file if already tokenized.
        for fname in os.listdir(step_dir_raw):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_raw, fname)
            output_fname = os.path.join(step_dir_tok, fname)
            if requires_tokenization:
                lang_code = lang_codes[0] if fname.endswith(SUFFIX_SRC_GZ) else lang_codes[1]
                logging.info("Tokenize (%s): %s -> %s", lang_code, input_fname, output_fname)
                third_party.call_moses_tokenizer(workspace_dir=args.workspace,
                                                 input_fname=input_fname,
                                                 output_fname=output_fname,
                                                 lang_code=lang_code)
            else:
                logging.info("Link pre-tokenized: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_RAW, fname), output_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_tok)

    # (6) Learn byte-pair encoding model

    # Task requires byte-pair encoding unless using pre-encoded custom data
    skip_bpe = (not args.task) and args.custom_text_type == CUSTOM_UTF8_BPE
    logging.info("=== Learn byte-pair encoding model ===")
    step_dir_bpe_model = os.path.join(dir_task, DIR_BPE_MODEL)
    complete_fname = os.path.join(step_dir_bpe_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe_model)
    else:
        renew_step_dir(step_dir_bpe_model)
        if skip_bpe:
            logging.info("BPE model not required for pre-encoded data")
        else:
            source_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_SRC_GZ)
            target_fname = os.path.join(step_dir_tok, PREFIX_TRAIN + SUFFIX_TRG_GZ)
            codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
            num_ops = task.bpe_op if args.task else args.custom_bpe_op
            if args.test:
                num_ops = TEST_BPE_OPS
            logging.info("BPE Learn (%s): %s + %s -> %s", num_ops, source_fname, target_fname, codes_fname)
            third_party.call_learn_bpe(workspace_dir=args.workspace,
                                       source_fname=source_fname,
                                       target_fname=target_fname,
                                       model_fname=codes_fname,
                                       num_ops=num_ops)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe_model)

    # (7) Byte-pair encode data

    logging.info("=== Byte-pair encode train/dev/test data ===")
    step_dir_bpe = os.path.join(dir_task, DIR_DATA, DIR_BPE)
    complete_fname = os.path.join(step_dir_bpe, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_bpe)
    else:
        renew_step_dir(step_dir_bpe)
        # Encode each data file
        for fname in os.listdir(step_dir_tok):
            if fname.startswith("."):
                continue
            input_fname = os.path.join(step_dir_tok, fname)
            output_fname = os.path.join(step_dir_bpe, fname)
            if skip_bpe:
                logging.info("Link pre-encoded: %s -> %s", input_fname, output_fname)
                os.symlink(os.path.join("..", DIR_TOK, fname), output_fname)
            else:
                codes_fname = os.path.join(step_dir_bpe_model, FILE_BPE_CODES)
                logging.info("BPE: %s -> %s", input_fname, output_fname)
                third_party.call_apply_bpe(workspace_dir=args.workspace,
                                           input_fname=input_fname,
                                           output_fname=output_fname,
                                           model_fname=codes_fname)
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_bpe)

    # Done if only running data preparation steps
    if args.model == MODEL_NONE:
        return

    # (8) Run Sockeye training

    logging.info("=== Train translation model ===")
    logging.info("Model: %s", args.model)
    if args.model == MODEL_GNMT:
        logging.info("NOTE: This is an 8 layer LSTM model similar (but not exactly identical) to the 'GNMT' architecture.")
    step_dir_model = os.path.join(dir_task, DIR_PREFIX_MODEL + args.model)
    complete_fname = os.path.join(step_dir_model, FILE_COMPLETE)
    if os.path.exists(complete_fname):
        logging.info("Re-use completed step: %s", step_dir_model)
    else:
        log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.{{}}.{}.{}.{}.log".format(task_name, args.model, os.getpid()))
        call_sockeye_train(args.model,
                           step_dir_bpe,
                           step_dir_model,
                           log_fname.format("train"),
                           args.gpus,
                           test_mode=args.test)
        call_sockeye_average(step_dir_model, log_fname.format("average"))
        # Record success
        touch_file(complete_fname)
        logging.info("Step complete: %s", step_dir_model)

    # (9) Decode test sets

    logging.info("=== Decode test sets ===")
    logging.info("Settings: %s", args.decode_settings)
    step_dir_results = os.path.join(dir_task, DIR_RESULTS)
    if not os.path.exists(step_dir_results):
        logging.info("Create: %s", step_dir_results)
        os.makedirs(step_dir_results)
    # To collect BPE output names
    output_fnames_bpe = []
    # For each test file
    for fname in os.listdir(step_dir_bpe):
        if fname.startswith(PREFIX_TEST) and fname.endswith(SUFFIX_SRC_GZ):
            input_fname = os.path.join(step_dir_bpe, fname)
            # /path/to/results/test[.N].<model>.<settings>
            output_fname = os.path.join(step_dir_results, "{}.{}.{}.{}".format(args.model,
                                                                               args.decode_settings,
                                                                               fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                               SUFFIX_BPE))
            output_fnames_bpe.append(output_fname)
            # For the shared results directory, a command file indicates that
            # the step has completed successfully.
            command_fname = output_fname + "." + SUFFIX_COMMAND
            if os.path.exists(command_fname):
                logging.info("Re-use output: %s", output_fname)
            else:
                log_fname = os.path.join(args.workspace,
                                 DIR_LOGS,
                                 "sockeye.translate.{}.{}.{}.{}.log".format(task_name,
                                                                            args.model,
                                                                            fname[:-len(SUFFIX_SRC_GZ) - 1],
                                                                            os.getpid()))
                call_sockeye_translate(args=DECODE_ARGS[args.decode_settings],
                                       input_fname=input_fname,
                                       output_fname=output_fname,
                                       model_dir=step_dir_model,
                                       log_fname=log_fname,
                                       use_cpu=(args.gpus == 0))

    # (10) Evaluate test sets (bpe/tok/detok)

    lang_code = lang_codes[1] if lang_codes else None
    logging.info("=== Score outputs ===")
    # For each output file
    for fname_bpe in output_fnames_bpe:
        # Score byte-pair encoded
        fname_base = os.path.basename(fname_bpe)[:-len(SUFFIX_BPE)].split(".", 2)[2]
        fname_ref_bpe = os.path.join(step_dir_bpe, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_bpe = fname_bpe + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_bpe):
            logging.info("Re-use output: %s", fname_bleu_bpe)
        else:
            fname_log = os.path.join(args.workspace,
                         DIR_LOGS,
                         "sockeye_contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                              args.model,
                                                                              fname_base + SUFFIX_BPE,
                                                                              os.getpid()))
            call_sacrebleu(input_fname=fname_bpe,
                           ref_fname=fname_ref_bpe,
                           output_fname=fname_bleu_bpe,
                           log_fname=fname_log,
                           tokenized=True)
        # Score tokenized
        fname_tok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_TOK
        fname_ref_tok = os.path.join(step_dir_tok, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_tok = fname_tok + "." + SUFFIX_BLEU
        if os.path.exists(fname_bleu_tok):
            logging.info("Re-use output: %s", fname_bleu_tok)
        else:
            # Merge BPE
            logging.info("Merge BPE: %s -> %s", fname_bpe, fname_tok)
            third_party.merge_bpe(input_fname=fname_bpe, output_fname=fname_tok)
            fname_log = os.path.join(args.workspace,
                         DIR_LOGS,
                         "sockeye_contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                              args.model,
                                                                              fname_base + SUFFIX_TOK,
                                                                              os.getpid()))
            call_sacrebleu(input_fname=fname_tok,
                           ref_fname=fname_ref_tok,
                           output_fname=fname_bleu_tok,
                           log_fname=fname_log,
                           tokenized=True)
        # Score detokenized (WMT-compatible BLEU)
        fname_detok = fname_bpe[:-len(SUFFIX_BPE)] + SUFFIX_DETOK
        fname_ref_raw = os.path.join(step_dir_raw, fname_base + SUFFIX_TRG_GZ)
        fname_bleu_detok = fname_detok + "." + SUFFIX_SACREBLEU
        if os.path.exists(fname_bleu_detok):
            logging.info("Re-use output: %s", fname_bleu_detok)
        else:
            if not requires_tokenization:
                logging.info("WARNING: Task uses pre-tokenized data, cannot reliably detokenize to compute WMT-compatible scores")
                continue
            # Detokenize
            logging.info("Detokenize (%s): %s -> %s", lang_code, fname_tok, fname_detok)
            third_party.call_moses_detokenizer(workspace_dir=args.workspace,
                                               input_fname=fname_tok,
                                               output_fname=fname_detok,
                                               lang_code=lang_code)
            fname_log = os.path.join(args.workspace,
                         DIR_LOGS,
                         "sockeye_contrib.sacrebleu.sacrebleu.{}.{}.{}.{}.log".format(task_name,
                                                                              args.model,
                                                                              fname_base + SUFFIX_DETOK,
                                                                              os.getpid()))
            call_sacrebleu(input_fname=fname_detok,
                           ref_fname=fname_ref_raw,
                           output_fname=fname_bleu_detok,
                           log_fname=fname_log,
                           tokenized=False)

def main():
    default_workspace = os.path.join(os.path.expanduser("~"), DIR_SOCKEYE_AUTOPILOT)

    arg_parser = argparse.ArgumentParser(description="Sockeye Autopilot: end-to-end model training and evaluation.")
    arg_parser.add_argument("--workspace", type=str, metavar="DIR", default=default_workspace,
                            help="Base directory to use for building systems (download files, train models, etc.). Default: %(default)s.")
    arg_parser.add_argument("--task", type=str, choices=sorted(TASKS.keys()),
                            help="Pre-defined data set for model training.")
    arg_parser.add_argument("--model", type=str, choices=sorted(MODELS.keys()),
                            help="Type of translation model to train.")
    arg_parser.add_argument("--decode-settings", type=str, choices=sorted(DECODE_ARGS.keys()), default=DECODE_STANDARD,
                            help="Decoding settings. Default: %(default)s.")
    arg_parser.add_argument("--custom-task", type=str, metavar="NAME",
                            help="Name of custom task (used for directory naming).")
    arg_parser.add_argument("--custom-train", type=str, nargs=2, metavar=("SRC", "TRG"),
                            help="Custom training data (source and target).")
    arg_parser.add_argument("--custom-dev", type=str, nargs=2, metavar=("SRC", "TRG"),
                            help="Custom development data (source and target).")
    arg_parser.add_argument("--custom-test", type=str, nargs="+", metavar="SRC TRG",
                            help="Custom test data (pairs of source and target).")
    arg_parser.add_argument("--custom-text-type", type=str, choices=CUSTOM_TEXT_TYPES, default=CUSTOM_UTF8_RAW,
                            help="Level of pre-processing already applied to data for custom task: none (raw), tokenization, or byte-pair encoding. Default: %(default)s.")
    arg_parser.add_argument("--custom-lang", type=str, nargs=2, metavar=("SRC", "TRG"),
                            help="Source and target language codes for custom task (en, fr, de, etc.).")
    arg_parser.add_argument("--custom-bpe-op", type=int, default=32000,
                            help="Number of byte-pair encoding operations for custom task. Default: %(default)s.")
    arg_parser.add_argument("--gpus", type=int, metavar="N", default=1,
                            help="Number of GPUs to use. 0 for CPU only. Default: %(default)s.")
    arg_parser.add_argument("--test", action="store_true", default=False,
                            help="Run in test mode (much abbreviated system build).")

    args = arg_parser.parse_args()

    # Listed task or fully specified custom task
    utils.check_condition(args.task or all((args.custom_train, args.custom_dev, args.custom_test)),
            "Please specify --task or all of: --custom-task --custom-train --custom-dev --custom-test")

    # Required args for different custom tasks
    if not args.task:
        if args.custom_text_type == CUSTOM_UTF8_RAW:
            utils.check_condition(args.custom_lang, "Please specify --custom-lang for source and target tokenization")

    # Require explicit request to not train model
    if not args.model:
        raise RuntimeError("Please specify --model.  Use --model %s to run data preparation steps only" % MODEL_NONE)

    run_steps(args)

if __name__ == "__main__":
    main()


text:
[![ci](https://github.com/jacobsa/fuse/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/jacobsa/fuse/actions/workflows/ci.yml)
[![GoDoc](https://godoc.org/github.com/jacobsa/ogletest?status.svg)](https://godoc.org/github.com/jacobsa/fuse)

This package allows for writing and mounting user-space file systems from Go.
Install it as follows:

    go get -u github.com/jacobsa/fuse

Afterward, see the documentation for the following three packages:

 *  Package [fuse][] provides support for mounting a new file system and
    reading requests from the kernel.

 *  Package [fuseops][] enumerates the supported requests from the kernel, and
    provides documentation on their semantics.

 *  Package [fuseutil][], in particular the `FileSystem` interface, provides a
    convenient way to create a file system type and export it to the kernel via
    `fuse.Mount`.

Make sure to also see the sub-packages of the [samples][] package for examples
and tests.

This package owes its inspiration and most of its kernel-related code to
[bazil.org/fuse][bazil].

[fuse]: http://godoc.org/github.com/jacobsa/fuse
[fuseops]: http://godoc.org/github.com/jacobsa/fuse/fuseops
[fuseutil]: http://godoc.org/github.com/jacobsa/fuse/fuseutil
[samples]: http://godoc.org/github.com/jacobsa/fuse/samples
[bazil]: http://godoc.org/bazil.org/fuse


text:
<?php use_helper('I18N') ?>
<?php include_partial('a/simpleEditWithVariants', array('pageid' => $page->id, 'name' => $name, 'permid' => $permid, 'slot' => $slot, 'page' => $page)) ?>

<?php if (!strlen($value)): ?>
  <?php if ($editable): ?>
    <?php echo __('Click edit to add text.', null, 'apostrophe') ?>
  <?php endif ?>
<?php else: ?>
<?php echo $value ?>
<?php endif ?>


text:
#include <iostream>

#include "app.hpp"

int main(int argc, char *argv[]) {
	auto application = open_tcg::App::create();

	return application->run(argc, argv);
}


text:
#ifndef _PLURAL_EXP_H
#define _PLURAL_EXP_H

#ifndef internal_function
# define internal_function
#endif

#ifndef attribute_hidden
# define attribute_hidden
#endif

#ifdef __cplusplus
extern "C" {
#endif

enum expression_operator
{
  /* Without arguments:  */
  var,				/* The variable "n".  */
  num,				/* Decimal number.  */
  /* Unary operators:  */
  lnot,				/* Logical NOT.  */
  /* Binary operators:  */
  mult,				/* Multiplication.  */
  divide,			/* Division.  */
  module,			/* Modulo operation.  */
  plus,				/* Addition.  */
  minus,			/* Subtraction.  */
  less_than,			/* Comparison.  */
  greater_than,			/* Comparison.  */
  less_or_equal,		/* Comparison.  */
  greater_or_equal,		/* Comparison.  */
  equal,			/* Comparison for equality.  */
  not_equal,			/* Comparison for inequality.  */
  land,				/* Logical AND.  */
  lor,				/* Logical OR.  */
  /* Ternary operators:  */
  qmop				/* Question mark operator.  */
};

/* This is the representation of the expressions to determine the
   plural form.  */
struct expression
{
  int nargs;			/* Number of arguments.  */
  enum expression_operator operation;
  union
  {
    unsigned long int num;	/* Number value for `num'.  */
    struct expression *args[3];	/* Up to three arguments.  */
  } val;
};

/* This is the data structure to pass information to the parser and get
   the result in a thread-safe way.  */
struct parse_args
{
  const char *cp;
  struct expression *res;
};

/* Names for the libintl functions are a problem.  This source code is used
   1. in the GNU C Library library,
   2. in the GNU libintl library,
   3. in the GNU gettext tools.
   The function names in each situation must be different, to allow for
   binary incompatible changes in 'struct expression'.  Furthermore,
   1. in the GNU C Library library, the names have a __ prefix,
   2.+3. in the GNU libintl library and in the GNU gettext tools, the names
         must follow ANSI C and not start with __.
   So we have to distinguish the three cases.  */
#ifdef _LIBC
# define FREE_EXPRESSION __gettext_free_exp
# define PLURAL_PARSE __gettextparse
# define GERMANIC_PLURAL __gettext_germanic_plural
# define EXTRACT_PLURAL_EXPRESSION __gettext_extract_plural
#elif defined (IN_LIBINTL)
# define FREE_EXPRESSION libintl_gettext_free_exp
# define PLURAL_PARSE libintl_gettextparse
# define GERMANIC_PLURAL libintl_gettext_germanic_plural
# define EXTRACT_PLURAL_EXPRESSION libintl_gettext_extract_plural
#else
# define FREE_EXPRESSION free_plural_expression
# define PLURAL_PARSE parse_plural_expression
# define GERMANIC_PLURAL germanic_plural
# define EXTRACT_PLURAL_EXPRESSION extract_plural_expression
#endif

extern void FREE_EXPRESSION (struct expression *exp)
     internal_function;
extern int PLURAL_PARSE (void *arg);
extern struct expression GERMANIC_PLURAL attribute_hidden;
extern void EXTRACT_PLURAL_EXPRESSION (const char *nullentry,
				       const struct expression **pluralp,
				       unsigned long int *npluralsp)
     internal_function;

#if !defined (_LIBC) && !defined (IN_LIBINTL) && !defined (IN_LIBGLOCALE)
extern unsigned long int plural_eval (const struct expression *pexp,
				      unsigned long int n);
#endif

#ifdef __cplusplus
}
#endif

#endif /* _PLURAL_EXP_H */


text:
/*
Link: https://dba.stackexchange.com/a/36578/107045
Use coreinfo (utility by sysinternals) as this will give you
a. Logical to Physical Processor Map
b. Logical Processor to Socket Map
c. Logical Processor to NUMA Node Map as below :
Logical to Physical Processor Map:
**----------------------  Physical Processor 0 (Hyperthreaded)
--**--------------------  Physical Processor 1 (Hyperthreaded)
----**------------------  Physical Processor 2 (Hyperthreaded)
------**----------------  Physical Processor 3 (Hyperthreaded)
--------**--------------  Physical Processor 4 (Hyperthreaded)
----------**------------  Physical Processor 5 (Hyperthreaded)
------------**----------  Physical Processor 6 (Hyperthreaded)
--------------**--------  Physical Processor 7 (Hyperthreaded)
----------------**------  Physical Processor 8 (Hyperthreaded)
------------------**----  Physical Processor 9 (Hyperthreaded)
--------------------**--  Physical Processor 10 (Hyperthreaded)
----------------------**  Physical Processor 11 (Hyperthreaded)
Logical Processor to Socket Map:
************------------  Socket 0
------------************  Socket 1
Logical Processor to NUMA Node Map:
************------------  NUMA Node 0
------------************  NUMA Node 1
Now, based on the above info, the Ideal MaxDop setting should be calculated as
a.  It has 12 CPU's which are hyper threaded giving us 24 CPUs.
b.  It has 2 NUMA node [Node 0 and 1] each having 12 CPU's with Hyperthreading ON.
c.  Number of sockets are 2 [socket 0 and 1] which are housing 12 CPU's each.
Considering all above factors, the max degree of Parallelism should be set to 6 which is ideal value for server with above configuration.
So the answer is -- "it depends" on your processor footprint and the NUMA configuration and below table will summarize what I explained above:
8 or less processors    ===> 0 to N (where N= no. of processors)
More than 8 processors  ===> 8
NUMA configured         ===> MAXDOP should not exceed no of CPU's assigned to each
                                 NUMA node with max value capped to 8
Hyper threading Enabled ===> Should not exceed the number of physical processors.
Below is a quick and dirty TSQL script to generate Recommendation for MAXDOP setting

Author          :   Kin Shah
Purpose         :   Recommend MaxDop settings for the server instance
Tested RDBMS    :   SQL Server 2008R2, 2014, 2016, 2017 */

DECLARE @hyperthreadingRatio BIT;
DECLARE @logicalCPUs INT;
DECLARE @HTEnabled INT;
DECLARE @physicalCPU INT;
DECLARE @SOCKET INT;
DECLARE @logicalCPUPerNuma INT;
DECLARE @NoOfNUMA INT;

SELECT @logicalCPUs = cpu_count -- [Logical CPU Count]
     , @hyperthreadingRatio = hyperthread_ratio --  [Hyperthread Ratio]
     , @physicalCPU = cpu_count / hyperthread_ratio -- [Physical CPU Count]
     , @HTEnabled = CASE
                      WHEN cpu_count > hyperthread_ratio THEN 1
                      ELSE 0
                    END -- HTEnabled
FROM   sys.dm_os_sys_info
OPTION (recompile);

SELECT @logicalCPUPerNuma = Count(parent_node_id)
-- [NumberOfLogicalProcessorsPerNuma]
FROM   sys.dm_os_schedulers
WHERE  [status] = 'VISIBLE ONLINE'
       AND parent_node_id < 64
GROUP  BY parent_node_id
OPTION (recompile);

SELECT @NoOfNUMA = Count(DISTINCT parent_node_id)
FROM   sys.dm_os_schedulers -- find NO OF NUMA Nodes
WHERE  [status] = 'VISIBLE ONLINE'
       AND parent_node_id < 64

-- Report the recommendations ....
SELECT
--- 8 or less processors and NO HT enabled
CASE
  WHEN @logicalCPUs < 8
       AND @HTEnabled = 0 THEN 'MAXDOP setting should be : '
                               + Cast(@logicalCPUs AS VARCHAR(3))
  --- 8 or more processors and NO HT enabled
  WHEN @logicalCPUs >= 8
       AND @HTEnabled = 0 THEN 'MAXDOP setting should be : 8'
  --- 8 or more processors and HT enabled and NO NUMA
  WHEN @logicalCPUs >= 8
       AND @HTEnabled = 1
       AND @NoOfNUMA = 1 THEN 'MaxDop setting should be : '
                              + Cast(@logicalCPUPerNuma / @physicalCPU AS
                              VARCHAR(3))
  --- 8 or more processors and HT enabled and NUMA
  WHEN @logicalCPUs >= 8
       AND @HTEnabled = 1
       AND @NoOfNUMA > 1 THEN 'MaxDop setting should be : '
                              + Cast(@logicalCPUPerNuma / @physicalCPU AS
                              VARCHAR(3))
  ELSE ''
END AS Recommendations;


text:
import json, requests, datetime, sys, util, os

BOT_FILE = "bots.json"

def bot_exists(user_id, bot_info):
    return user_id in bot_info.keys()

def create_bot(group_id, user_id, name, avatar_url, bot_info):
    if user_id in bot_info.keys():
        print("Bot already registered for user " + str(user_id))
        return
    data = {
        "bot": {
            "name": str(name),
            "group_id": str(group_id),
            "avatar_url": avatar_url,
            "callback_url": "http://104.236.152.196:6000/" + user_id
        }
    }
    headers = { "Content-Type": "application/json" }
    res = util.post_rest('bots', data=data, headers=headers)
    if res is None:
        print("Bot creating POST request failed")
        return
    bot_info[user_id] = res['bot']
    print("Bot creating successful, bot ID: " + str(res['bot']['bot_id']))

def post_bot(user_id, message, bot_info):
    if not user_id in bot_info.keys():
        print("Bot does not exist for this user")
        return
    bot_id = bot_info[user_id]['bot_id']
    data = {
        'bot_id': bot_id,
        'text': message
    }
    headers = { "Content-Type": "application/json" }
    res = util.post_rest('bots/post', data=data, headers=headers)
    if res is None:
        print("Bot posting GET request failed")

def load_bot_info(group_id):
    path = "groups/" + group_id + "/" + BOT_FILE
    if not os.path.exists(path):
        return {}
    bot_file = open(path, 'r')
    bot_info = json.loads(bot_file.read())
    bot_file.close()
    return bot_info

def save_bot_info(group_id, bot_info):
    bot_file = open("groups/" + group_id + "/" + BOT_FILE, 'w')
    bot_file.write(json.dumps(bot_info))
    bot_file.close()


text:
import 'babel-polyfill';
import {
  getSettings,
  setSimQuality,
  setSimSpeed
} from '../profileService.js';

beforeEach(() => {
  global.localStorage.getItem.mockReset();
  global.localStorage.setItem.mockReset();
  global.localStorage.clear.mockReset();
  global.localStorage.removeItem.mockReset();
});

test('get settings default', async () => {
  const result = await getSettings();
  expect(result).toHaveProperty('simSpeed', 1);
  expect(result).toHaveProperty('qualitySettings', 0.5);
});

test('get stored settings', async () => {
  global.localStorage.getItem.mockImplementation((key) => {
    switch (key) {
      case 'settings.simSpeed': return 2.34
      case 'settings.quality':  return 'auto'
      default:                  return undefined;
    }
  })
  const result = await getSettings();
  expect(result).toHaveProperty('simSpeed', 2.34);
  expect(result).toHaveProperty('qualitySettings', 'auto');
});

test('set simulation speed', async () => {
  await setSimSpeed(4.391);
  expect(global.localStorage.setItem.mock.calls).toHaveLength(1);
  expect(global.localStorage.setItem.mock.calls[0][0]).toBe("settings.simSpeed");
  expect(global.localStorage.setItem.mock.calls[0][1]).toBe(4.391);
});

test('set quality speed', async () => {
  await setSimQuality(0.07);
  expect(global.localStorage.setItem.mock.calls).toHaveLength(1);
  expect(global.localStorage.setItem.mock.calls[0][0]).toBe("settings.quality");
  expect(global.localStorage.setItem.mock.calls[0][1]).toBe(0.07);

  global.localStorage.setItem.mockReset();
  await setSimQuality('auto');
  expect(global.localStorage.setItem.mock.calls).toHaveLength(1);
  expect(global.localStorage.setItem.mock.calls[0][0]).toBe("settings.quality");
  expect(global.localStorage.setItem.mock.calls[0][1]).toBe('auto');
});


text:
package com.jukusoft.libgdx.rpg.engine.story;

import com.jukusoft.libgdx.rpg.engine.time.GameTime;
import com.jukusoft.libgdx.rpg.engine.utils.ArrayUtils;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by Justin on 07.02.2017.
 */
public class StoryPart {

    protected List<String> lines = new ArrayList<>();
    protected String[] lineArray  = new String[] {};

    protected long startTime = 0l;
    protected long timeToRead = 20000l;

    public StoryPart () {
        //
    }

    public void addLine (String line) {
        this.lines.add(line);
    }

    public List<String> listLines () {
        return this.lines;
    }

    public int countLines () {
        return lines.size();
    }

    public String[] getLineArray () {
        return this.lineArray;
    }

    public float getPartProgress (long now) {
        long timeLeft = now - startTime;

        if (timeLeft > timeToRead) {
            timeLeft = timeToRead;
        }

        float delta = timeToRead - timeLeft;
        delta = timeToRead - delta;

        return delta / timeToRead;
    }

    public void start () {
        this.startTime = GameTime.getInstance().getTime();

        this.lineArray = ArrayUtils.convertStringListToArray(this.lines);
    }

    public boolean hasFinished (long now) {
        return (this.startTime + this.timeToRead) < now;
    }

}


text:
$(function() {
  // Convert the text area into a CodeMirror area in readOnly mode.
  var cm = CodeMirror.fromTextArea(document.querySelector("textarea"), {
    readOnly: true,
    lineNumbers: true,
    mode: "text/typescript",
    scrollbarStyle: "simple",
    gutters: [
      "Flow-lineuncovered",
      "CodeMirror-linenumbers"
    ]
  });

  // TODO: handle corrupted coverage data?
  var coverageData = JSON.parse(document.querySelector("#file-coverage-data").textContent);
  var uncoveredLocs = coverageData.expressions.uncovered_locs;

  // Create uncovered range markers.
  for (var loc of uncoveredLocs) {
    cm.markText(
      {line: loc.start.line - 1, ch: loc.start.column - 1},
      {line: loc.end.line - 1, ch: loc.end.column},
      {
        className: "cm-flow-uncovered",
      }
    );
  }

  // Create uncovered ranges scrollbar annotations.
  var scrollbarAnnotations = cm.annotateScrollbar({
    className: "cm-flow-uncovered",
    listenForUpdates: false,
  });

  scrollbarAnnotations.update(uncoveredLocs.map(function (loc) {
    return {
      from: {line: loc.start.line - 1, ch: loc.start.column -1},
      to: {line: loc.end.line - 1, ch: loc.start.column},
    };
  }));

  // Create the line gutters.
  for (var i = 0; i < cm.lineCount(); i++) {
    var count = uncoveredLocs.reduce(function (acc, loc) {
      if (i >= loc.start.line - 1 && i <= loc.end.line - 1) {
        acc += 1;
      }

      return acc;
    }, 0);

    if (count > 0) {
      var el = document.createElement("div");
      el.textContent =  count + "x";
      el.style.textAlign = "center";
      el.style.fontSize = "0.6em";
      var color = "rgba(255,0,0," + count * 0.2 + ")";
      el.style.background = color;
      cm.setGutterMarker(i, "Flow-lineuncovered", el);
    }
  }

  // Refresh the uncovered range markers background alpha
  // (number of time the marker span appears in an uncovered range * alpha step).
  CodeMirror.on(cm, "viewportChange", function(cm, marker){
    for (var el of document.querySelectorAll(".cm-flow-uncovered")) {
      var count = Array.from(el.classList).filter(function (cls) {
        if (cls !== "cm-flow-uncovered") {
          return false;
        }
        return true;
      }).length;
      var color = "rgba(255,0,0," + count * 0.2 + ")";
      el.style.background = color;
    }
  });

  // Configure the dropdowns

  $('.ui.dropdown.uncovered-locations')
    .dropdown({
      action: 'hide',
      onChange: function (value) {
        if (value == "") {
          return;
        }
        $('.ui.dropdown.uncovered-locations').dropdown('set value', "");
        cm.scrollIntoView({line: parseInt(value)}, 150);
      }
    });

  $('.ui.dropdown.syntax-highlighting')
    .dropdown({
      onChange: function (value) {
        var mode = "typescript";

        switch (value) {
        case "es":
          mode = "text/typescript";
          break;
        case "js":
          mode = "javascript";
          break;
        default:
          mode = "";
        }

        console.log("SET MODE", mode);
        cm.setOption("mode", mode);
      }
    });
});


text:
import { Injectable } from '@angular/core'
const Sqlite = require("nativescript-sqlite");

@Injectable()
export class PersonDao {
  db;
  conter;
  constructor() {
    new Sqlite("TestTable.db").then(db => {
      this.db = db;
      this.conter = Math.random();
    })
  }
}


text:
title: "Maanantain tunti"
tags: ena4 etusivu
---

Aikaisemmin ilmoitetusta poiketen maanantaina on tunti. Eli tulehan luokkaan 7 klo 9.45 :)


text:
from morelia.visitors import VisitorObserver

class Breadcrumbs(VisitorObserver):
    def __init__(self):
        self.__feature = None
        self.__scenario = None
        self.step = None

    def feature_started(self, node):
        self.feature = node

    def scenario_started(self, node):
        self.scenario = node

    def step_started(self, node):
        self.step = node

    @property
    def feature(self):
        return self.__feature

    @feature.setter
    def feature(self, value):
        self.__feature = value
        self.scenario = None

    @property
    def scenario(self):
        return self.__scenario

    @scenario.setter
    def scenario(self, value):
        self.__scenario = value
        self.step = None

    def __str__(self):
        feature = self.feature.interpolated_source() if self.feature else ""
        scenario = self.scenario.interpolated_source() if self.scenario else ""
        step = self.step.interpolated_source() if self.step else ""
        return "\n{}{}{}".format(feature, scenario, step)


text:
using System;
using MatterHackers.Agg;
using MatterHackers.Agg.UI;

namespace MatterHackers.MatterControl.CustomWidgets
{
	public abstract class TemperatureStatusWidget : FlowLayoutWidget
	{
		protected TextWidget actualTemp;
		protected ProgressBar progressBar;
		protected TextWidget targetTemp;
		protected EventHandler unregisterEvents;
		private int fontSize = 14;
		protected PrinterConfig printer;

		public TemperatureStatusWidget(PrinterConfig printer, string dispalyName, ThemeConfig theme)
		{
			this.printer = printer;
			var extruderName = new TextWidget(dispalyName, pointSize: fontSize, textColor: theme.TextColor)
			{
				AutoExpandBoundsToText = true,
				VAnchor = VAnchor.Center,
				Margin = new BorderDouble(right: 8)
			};

			this.AddChild(extruderName);

			progressBar = new ProgressBar(200, 6)
			{
				FillColor = theme.PrimaryAccentColor,
				Margin = new BorderDouble(right: 10),
				BorderColor = Color.Transparent,
				BackgroundColor = new Color(theme.TextColor, 50),
				VAnchor = VAnchor.Center,
			};
			this.AddChild(progressBar);

			actualTemp = new TextWidget("", pointSize: fontSize, textColor: theme.TextColor)
			{
				AutoExpandBoundsToText = true,
				VAnchor = VAnchor.Center,
				Margin = new BorderDouble(right: 0),
				Width = 60
			};
			this.AddChild(actualTemp);

			this.AddChild(new VerticalLine()
			{
				BackgroundColor = theme.TextColor,
				Margin = new BorderDouble(8, 0)
			});

			targetTemp = new TextWidget("", pointSize: fontSize, textColor: theme.TextColor)
			{
				AutoExpandBoundsToText = true,
				VAnchor = VAnchor.Center,
				Margin = new BorderDouble(right: 8),
				Width = 60
			};
			this.AddChild(targetTemp);

			UiThread.RunOnIdle(UpdateTemperatures);
		}

		public override void OnClosed(EventArgs e)
		{
			unregisterEvents?.Invoke(this, null);
		}

		public abstract void UpdateTemperatures();
	}
}


text:
from slimit import minify

from .base import Mangler

class JsMangler(Mangler):
    '''
    Mixin post-processor for minifying CSS files
    '''
    def __init__(self, target, mangle=True, mangle_toplevel=False):
        super().__init__(target)
        self.mangle = mangle
        self.mangle_toplevel = mangle_toplevel

    def can_process(self, file_obj):
        exts = file_obj.current_name.suffixes
        if exts[-1] != '.js':
            return False
        if len(exts) > 1 and exts[-2] == '.min':
            return False
        return True

    def process_file(self, file_obj):
        content = minify(file_obj.str,
                         mangle=self.mangle,
                         mangle_toplevel=self.mangle_toplevel)

        self.target.delete(file_obj.current_name)

        file_obj.str = content
        file_obj.add_suffix('.min', -2)
        yield file_obj