text:
"""Swarming bot main process.

This is the program that communicates with the Swarming server, ensures the code
is always up to date and executes a child process to run tasks and upload
results back.

It manages self-update and rebooting the host in case of problems.

Sections are:
  - Globals
  - Monitoring
  - bot_config handler
  - Public functions used by __main__.py
  - Sub process management
  - Bot lifetime management
"""

from __future__ import print_function

import argparse
import collections
import contextlib
import fnmatch
import json
import logging
import os
import platform
import shutil
import sys
import tempfile
import threading
import time
import traceback
import types
import zipfile

# Import _strptime before threaded code. datetime.datetime.strptime is
# threadsafe except for the initial import of the _strptime module.
# See https://bugs.python.org/issue7980.
import _strptime  # pylint: disable=unused-import

from api import bot
from api import os_utilities
from api import platforms
from bot_code import bot_auth
from bot_code import common
from bot_code import file_refresher
from bot_code import remote_client
from bot_code import remote_client_errors
from bot_code import singleton
from infra_libs import ts_mon
from utils import file_path
from utils import fs
from utils import net
from utils import on_error
from utils import subprocess42
from utils import tools
from utils import zip_package

### Globals

# Used to opportunistically set the error handler to notify the server when the
# process exits due to an exception.
_ERROR_HANDLER_WAS_REGISTERED = False

# If False (happens in tests), avoid catching broad Exception in the bot loop.
# Otherwise failed test assertions are also getting trapped, which makes tests
# hard to debug.
_TRAP_ALL_EXCEPTIONS = True

# Set to the zip's name containing this file. This is set to the absolute path
# to swarming_bot.zip when run as part of swarming_bot.zip. This value is
# overriden in unit tests.
#
# Note: this more or less requires the bot to be in a path without non-ASCII
# characters.
THIS_FILE = os.path.abspath(zip_package.get_main_script_path())
THIS_DIR = os.path.dirname(THIS_FILE)

# The singleton, initially unset.
SINGLETON = singleton.Singleton(THIS_DIR)

# Dirname of kvs cache used to store small files from cas.
_CAS_KVS_CACHE_DB = 'cas_kvs_cache_db'

# Allowlist files that can be present in the bot's directory. Anything else
# will be forcibly deleted on startup! Note that 'w' (work) is not in this list,
# as we want it to be deleted on startup.
# See
# https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/Bot.md
# for more details.
PASSLIST = (
    '*-cacert.pem',
    '.vpython3',
    'README',
    'README.md',
    'c',
    'cas_cache',
    'cipd_cache',
    'logs',
    'swarming.lck',
    'swarming_bot.1.zip',
    'swarming_bot.2.zip',
    'swarming_bot.zip',
    'tmp',
    _CAS_KVS_CACHE_DB,
)

# These settings are documented in ../config/bot_config.py.
# Keep in sync with ../config/bot_config.py. This is enforced by a unit test.
DEFAULT_SETTINGS = {
    'free_partition': {
        'root': {
            'size': 1 * 1024 * 1024 * 1024,
            'max_percent': 10.,
            'min_percent': 6.,
        },
        'bot': {
            'size': 4 * 1024 * 1024 * 1024,
            'max_percent': 15.,
            'min_percent': 7.,
            'wiggle': 250 * 1024 * 1024,
        },
    },
    'caches': {
        'isolated': {
            'size': 50 * 1024 * 1024 * 1024,
            'items': 50 * 1024,
        },
    },
}

# Keep in sync with ../../ts_mon_metrics.py
_IGNORED_DIMENSIONS = (
    'android_devices', 'caches', 'id', 'server_version', 'temp_band')

# Flag to decide if bot is running in test mode. This is mostly used by smoke
# and integration tests.
# TODO(1099655): Remove once we have fully enabled CIPD in both prod and tests.
_IN_TEST_MODE = False

# TODO(vadimsh): This is temporary to detect bots that dynamically change ID.
_ORIGINAL_BOT_ID = None

### Monitoring

_bucketer = ts_mon.GeometricBucketer(growth_factor=10**0.07,
                                     num_finite_buckets=100)

_hooks_durations = ts_mon.CumulativeDistributionMetric(
    'swarming/bots/hooks/durations',
    'Duration of bot hook calls in ms', [
        ts_mon.StringField('hookname'),
        ts_mon.StringField('pool'),
    ],
    bucketer=_bucketer,
    units=ts_mon.MetricsDataUnits.MILLISECONDS)

def _pool_from_dimensions(dimensions):
  """Return a canonical string of flattened dimensions."""
  # Keep in sync with ../../ts_mon_metrics.py
  pairs = []
  for key, values in dimensions.items():
    if key in _IGNORED_DIMENSIONS:
      continue
    # Strip all the prefixes of other values. values is already sorted.
    for i, value in enumerate(values):
      if not any(v.startswith(value) for v in values[i+1:]):
        pairs.append('%s:%s' % (key, value))
  return '|'.join(sorted(pairs))

def _monitor_call(func):
  """Decorates a functions and reports the runtime to ts_mon."""
  def hook(chained, botobj, name, *args, **kwargs):
    start = time.time()
    try:
      return func(chained, botobj, name, *args, **kwargs)
    finally:
      duration = max(0, (time.time() - start) * 1000)
      if botobj and botobj.dimensions:
        flat_dims = _pool_from_dimensions(botobj.dimensions)
        if flat_dims:
          logging.info('ts_mon hook_name=%r pool=%r', name, flat_dims)
          _hooks_durations.add(
              duration, fields={'hookname': name, 'pool': flat_dims})
      logging.info('%s(): %gs', name, round(duration/1000., 3))
  return hook

def _init_ts_mon():
  """Initializes ts_mon."""
  parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  ts_mon.add_argparse_options(parser)
  parser.set_defaults(
      ts_mon_target_type='task',
      ts_mon_task_service_name='swarming-bot',
      ts_mon_task_job_name='default',
      ts_mon_flush='auto',
      ts_mon_ca_certs=tools.get_cacerts_bundle(),
  )
  args = parser.parse_args([])
  ts_mon.process_argparse_options(args)

### bot_config handler

# Reference to the config/bot_config.py module inside the swarming_bot.zip file.
# This variable is initialized inside _get_bot_config().
_BOT_CONFIG = None
# Reference to the second bot_config.py module injected by the server. This
# variable is initialized inside _do_handshake().
_EXTRA_BOT_CONFIG = None
# Super Sticky quarantine string. This variable is initialized inside
# _set_quarantined() and be set at various places when a hook throws an
# exception. Restarting the bot will clear the quarantine, which includes
# updated the bot due to new bot_config or new bot code.
_QUARANTINED = None

def _set_quarantined(reason):
  """Sets the Super Sticky Quarantine string."""
  logging.error('_set_quarantined(%s)', reason)
  global _QUARANTINED
  _QUARANTINED = _QUARANTINED or reason

def _get_bot_config():
  """Returns the bot_config.py module. Imports it only once.

  This file is called implicitly by _call_hook() and _call_hook_safe().
  """
  global _BOT_CONFIG
  if not _BOT_CONFIG:
    from config import bot_config as _BOT_CONFIG
  return _BOT_CONFIG

def _register_extra_bot_config(content, rev, script):
  """Registers the server injected extra injected.py bot_config.

  This file is called implicitly by _call_hook() and _call_hook_safe().
  """
  global _EXTRA_BOT_CONFIG
  if isinstance(content, str):
    # compile will throw if there's a '# coding: utf-8' line and the string is
    # in unicode. <3 python.
    content = content.encode('utf-8')
  try:
    compiled = compile(content, 'injected.py', 'exec')
    _EXTRA_BOT_CONFIG = types.ModuleType('injected')
    exec(compiled, _EXTRA_BOT_CONFIG.__dict__)
    logging.debug('extra bot_config %s at rev %s was injected.', script, rev)
  except (SyntaxError, TypeError) as e:
    _set_quarantined(
        'handshake returned invalid injected bot_config.py: %s' % e)

@_monitor_call
def _call_hook(chained, botobj, name, *args, **kwargs):
  """Calls a hook function named `name` in bot_config.py.

  If `chained` is True, calls the general bot_config.py then the injected
  version.

  If `chained` is False, the injected bot_config version is called first, and
  only if not present the general bot_config version is called.
  """
  try:
    if not chained:
      # Injected version has higher priority.
      hook = getattr(_EXTRA_BOT_CONFIG, name, None)
      if hook:
        try:
          return hook(botobj, *args, **kwargs)

        except OSError:
          _log_process_info()

      hook = getattr(_get_bot_config(), name, None)
      if hook:
        return hook(botobj, *args, **kwargs)
      # The hook is not defined.
      return None

    # In the case of chained=True, call both hooks. Call the generic one first,
    # then the specialized.
    ret = None
    hook = getattr(_get_bot_config(), name, None)
    if hook:
      ret = hook(botobj, *args, **kwargs)
    hook = getattr(_EXTRA_BOT_CONFIG, name, None)
    if hook:
      # Ignores the previous return value.
      ret = hook(botobj, *args, **kwargs)
    return ret
  finally:
    # TODO(maruel): Handle host_reboot() request the same way.
    if botobj:
      msg = botobj.bot_restart_msg()
      if msg:
        # The hook requested a bot restart. Do it right after the hook call.
        _bot_restart(botobj, msg)

def _log_process_info():
  try:
    import psutil
    process_count = collections.Counter(
      proc.name() for proc in psutil.process_iter()
    )
    logging.info('Processes running: %s', process_count)

  except ImportError:
    logging.info('Fail to get process info. Missing psutil')

def _call_hook_safe(chained, botobj, name, *args):
  """Calls a hook function in bot_config.py.

  Like _call_hook() but traps most exceptions.
  """
  try:
    return _call_hook(chained, botobj, name, *args)
  except Exception as e:
    traceback.print_exc()
    logging.exception('%s() threw', name)
    msg = '%s\n%s' % (e, traceback.format_exc()[-2048:])
    if botobj:
      botobj.post_error('Failed to call hook %s(): %s' % (name, msg))
    return None

def _get_dimensions(botobj):
  """Returns bot_config.py's get_dimensions() dict.

  Traps exceptions, quarantining the bot if they happen.
  """
  out = _call_hook_safe(False, botobj, 'get_dimensions')
  if isinstance(out, dict):
    out = out.copy()
    out['server_version'] = [_get_server_version_safe()]
    return out
  try:
    _set_quarantined('get_dimensions(): expected a dict, got %r' % out)
    out = os_utilities.get_dimensions()
    out['quarantined'] = ['1']
    out['server_version'] = [_get_server_version_safe()]
    return out
  except Exception:
    logging.exception('os.utilities.get_dimensions() failed')
    return {
        'bot_error': ['bot_main:_get_dimensions'],
        'id': [_get_botid_safe()],
        'quarantined': ['1'],
        'server_version': [_get_server_version_safe()],
    }

def _get_server_version_safe():
  return get_config().get('server_version', 'N/A')

@tools.cached
def _get_botid_safe():
  """Paranoid version of get_hostname_short()."""
  try:
    return os_utilities.get_hostname_short()
  except Exception as e:
    logging.exception('os.utilities.get_hostname_short() failed')
    return 'error_%s' % str(e)

def _get_settings(botobj):
  """Returns settings for this bot.

  The way used to make it work safely is to take the default settings, then
  merge the custom settings. This way, a user can only specify a subset of the
  desired settings.

  The function won't alert on unknown settings. This is so bot_config.py can be
  updated in advance before pushing new bot_main.py. The main drawback is that
  it will make typos silently fail. CHECK FOR TYPOS in get_settings() in your
  bot_config.py.
  """
  settings = _call_hook_safe(False, botobj, 'get_settings')
  try:
    if isinstance(settings, dict):
      return _dict_deep_merge(DEFAULT_SETTINGS, settings)
  except (KeyError, TypeError, ValueError):
    logging.exception('get_settings() failed')
  return DEFAULT_SETTINGS

def _get_state(botobj, sleep_streak):
  """Returns dict with a state of the bot reported to the server with each poll.

  Traps exceptions, quarantining the bot if they happen.
  """
  state = _call_hook_safe(False, botobj, 'get_state')
  if not isinstance(state, dict):
    _set_quarantined('get_state(): expected a dict, got %r' % state)
    state = {'broken': state}

  # TODO(vadimsh): This is temporary to detect bots that dynamically change ID.
  # This global var is set before the poll loop and never changes after that
  # (unlike botobj.id which can potentially change). The server compares this
  # to the bot ID and logs a warning if they are different.
  if _ORIGINAL_BOT_ID:
    state['original_bot_id'] = _ORIGINAL_BOT_ID

  if not state.get('quarantined'):
    if not _is_base_dir_ok(botobj):
      # Use super hammer in case of dangerous environment.
      _set_quarantined('Can\'t run from the base directory')
    if _QUARANTINED:
      state['quarantined'] = _QUARANTINED

  if not state.get('quarantined'):
    try:
      # Reuse the data from 'state/disks'.
      disks = state.get('disks', {})
      err = _get_disks_quarantine(botobj, disks)
      if err:
        state['quarantined'] = err
        _cleanup_purgeable_space(botobj)
    except Exception as e:
      logging.exception('checking free or purgeable space failed')
      state['quarantined'] = '%s\n%s' % (e, traceback.format_exc()[-2048:])

  state['sleep_streak'] = sleep_streak
  return state

def _get_disks_quarantine(botobj, disks):
  """Returns a quarantine error message when there's not enough free space.

  It looks at both root partition and the current partition the bot is running
  in.
  """
  settings = _get_settings(botobj)['free_partition']
  # On Windows, drive letters are always lower case.
  root = 'c:\\' if sys.platform == 'win32' else '/'

  errors = []

  def _check_for_quarantine(r, i, key):
    min_free = _min_free_disk(i, settings[key])
    if int(i['free_mb']*1024*1024) < min_free:
      errors.append(
          'Not enough free disk space on %s. %.1fmib < %.1fmib' %
          (r, i['free_mb'], round(min_free / 1024. / 1024., 1)))

  # root may be missing in the case of netbooted devices.
  if root in disks:
    _check_for_quarantine(root, disks[root], 'root')

  # Try again with the bot's base directory. It is frequent to run the bot
  # from a secondary partition, to reduce the risk of OS failure due to full
  # root partition.
  # This code is similar to os_utilities.get_disk_size().
  path = botobj.base_dir
  case_insensitive = sys.platform in ('darwin', 'win32')
  if case_insensitive:
    path = path.lower()
  for mount, infos in sorted(disks.items(), key=lambda x: -len(x[0])):
    if path.startswith(mount.lower() if case_insensitive else mount):
      # Apply 'bot' check if bot is on its own partition, or it's on
      # root partition and there are no errors reported yet.
      if mount != root or not errors:
        _check_for_quarantine(mount, infos, 'bot')
      break
  if errors:
    return '\n'.join(errors)

def _update_bot_attributes(botobj, sleep_streak):
  """Queries environment and hooks for dimensions and state and updates Bot.

  This is generally pretty slow.
  """
  dims = _get_dimensions(botobj)
  state = _get_state(botobj, sleep_streak)
  logging.debug('Dimensions %s', dims)
  logging.debug('State %s', state)
  with botobj.mutate_internals() as mut:
    mut.update_dimensions(dims)
    mut.update_state(state)

def _cleanup_purgeable_space(botobj):
  """Frees up purgeable space by creating large files and removing them.

  It runs only on ARM Macmini.

  TODO(crbug.com/1142848): remove this workaround after bot code is fixed to
  recognize purgeable space.
  """
  # Skip if it's not ARM MMac.
  mac_model = botobj.dimensions.get('mac_model')
  if not mac_model or mac_model[0] != 'Macmini9,1':
    return

  start = time.time()
  logging.info('_cleanup_purgeable_space: creating large files.')

  tempdir = os.path.join(botobj.base_dir, 'largefiles')
  fs.mkdir(tempdir)

  for cnt in range(3):
    # create a largefile until it stops with 'No space left on device'.
    fname = os.path.join(tempdir, 'largefile%d' % cnt)
    cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=32m']
    output, _ = _Popen(botobj, cmd).communicate(None)
    logging.info(
        '_cleanup_purgeable_space: created a large file. cnt=%d. output=%s',
        cnt, output)

  # remove created all files and the temp directory.
  file_path.rmtree(tempdir)

  logging.info(
      '_cleanup_purgeable_space: removed the created files. %s sec elapsed.',
      time.time() - start)

def _get_authentication_headers(botobj):
  """Calls bot_config.get_authentication_headers() if it is defined.

  See remote_client.RemoteClient doc for the expected format of the return
  value.

  Doesn't catch exceptions. RemoteClient knows how to deal with them.
  """
  return _call_hook(False, botobj, 'get_authentication_headers') or (None, None)

def _on_shutdown_hook(b):
  """Called when the bot is restarting."""
  _call_hook_safe(True, b, 'on_bot_shutdown')
  # Aggressively set itself up so we ensure the auto-reboot configuration is
  # fine before restarting the host. This is important as some tasks delete the
  # autorestart script (!)
  setup_bot(True)

def _min_free_disk(infos, settings):
  """Returns the calculated minimum free disk space for this partition.

  See _get_settings() in ../config/bot_config.py for an explanation.
  """
  size = int(infos['size_mb']*1024*1024)
  x1 = settings['size'] or 0
  x2 = int(round(size * float(settings['max_percent'] or 0) * 0.01))
  # Select the lowest non-zero value.
  x = min(x1, x2) if (x1 and x2) else (x1 or x2)
  # Select the maximum value.
  return max(x, int(round(size * float(settings['min_percent'] or 0) * 0.01)))

def _dict_deep_merge(x, y):
  """Returns the union of x and y.

  y takes predescence.
  """
  if x is None:
    return y
  if y is None:
    return x
  if isinstance(x, dict):
    if isinstance(y, dict):
      return {k: _dict_deep_merge(x.get(k), y.get(k)) for k in set(x).union(y)}
    assert y is None, repr(y)
    return x
  if isinstance(y, dict):
    assert x is None, repr(x)
    return y
  # y is overriding x.
  return y

def _is_base_dir_ok(botobj):
  """Returns False if the bot must be quarantined at all cost."""
  if not botobj:
    # This can happen very early in the process lifetime.
    return THIS_DIR != os.path.expanduser('~')
  return botobj.base_dir != os.path.expanduser('~')

### Public functions used by __main__.py

def setup_bot(skip_reboot):
  """Calls bot_config.setup_bot() to have the bot self-configure itself.

  Reboots the host if bot_config.setup_bot() returns False, unless skip_reboot
  is also true.

  Does nothing if SWARMING_EXTERNAL_BOT_SETUP env var is set to 1. It is set in
  case bot's autostart configuration is managed elsewhere, and we don't want
  the bot itself to interfere.
  """
  if os.environ.get('SWARMING_EXTERNAL_BOT_SETUP') == '1':
    logging.info('Skipping setup_bot, SWARMING_EXTERNAL_BOT_SETUP is set')
    return

  botobj = get_bot(get_config())
  try:
    from config import bot_config
  except Exception as e:
    msg = '%s\n%s' % (e, traceback.format_exc()[-2048:])
    botobj.post_error('bot_config.py is bad: %s' % msg)
    return

  # TODO(maruel): Convert the should_continue return value to the hook calling
  # botobj.host_reboot() by itself.
  try:
    should_continue = bot_config.setup_bot(botobj)
  except Exception as e:
    msg = '%s\n%s' % (e, traceback.format_exc()[-2048:])
    botobj.post_error('bot_config.setup_bot() threw: %s' % msg)
    return

  if not should_continue and not skip_reboot:
    botobj.host_reboot('Starting new swarming bot: %s' % THIS_FILE)

@tools.cached
def generate_version():
  """Returns the bot's code version."""
  try:
    return zip_package.generate_version()
  except Exception as e:
    return 'Error: %s' % e

def get_attributes(botobj):
  """Returns the attributes sent to the server in /handshake.

  Each called function catches all exceptions so the bot doesn't die on startup,
  which is annoying to recover. In that case, we set a special property to catch
  these and help the admin fix the swarming_bot code more quickly.

  Arguments:
  - botobj: bot.Bot instance.
  """
  return {
    'dimensions': _get_dimensions(botobj),
    'state': _get_state(botobj, 0),
    'version': generate_version(),
  }

def get_bot(config):
  """Returns a valid Bot instance.

  Should only be called once in the process lifetime.

  It can be called by ../__main__.py, something to keep in mind.
  """
  # This variable is used to bootstrap the initial bot.Bot object, which then is
  # used to get the dimensions and state.
  attributes = {
    'dimensions': {'id': ['none']},
    'state': {},
    'version': generate_version(),
  }
  hostname = _get_botid_safe()
  base_dir = THIS_DIR
  # Use temporary Bot object to call get_attributes. Attributes are needed to
  # construct the "real" bot.Bot.
  attributes = get_attributes(
      bot.Bot(
          remote_client.createRemoteClient(config['server'], None, hostname,
                                           base_dir), attributes,
          config['server'], config['server_version'], base_dir,
          _on_shutdown_hook))

  # Make remote client callback use the returned bot object. We assume here
  # RemoteClient doesn't call its callback in the constructor (since 'botobj' is
  # undefined during the construction).
  botobj = bot.Bot(
      remote_client.createRemoteClient(
          config['server'], lambda: _get_authentication_headers(botobj),
          hostname, base_dir), attributes, config['server'],
      config['server_version'], base_dir, _on_shutdown_hook)
  return botobj

@tools.cached
def get_config():
  """Returns the data from config.json."""
  global _ERROR_HANDLER_WAS_REGISTERED
  try:
    with contextlib.closing(zipfile.ZipFile(THIS_FILE, 'r')) as f:
      config = json.load(f.open('config/config.json', 'r'))
    if config['server'].endswith('/'):
      raise ValueError('Invalid server entry %r' % config['server'])
  except (zipfile.BadZipfile, IOError, OSError, TypeError, ValueError):
    logging.exception('Invalid config.json!')
    config = {
      'server': '',
      'server_version': 'N/A',
    }
  if not _ERROR_HANDLER_WAS_REGISTERED and config['server']:
    on_error.report_on_exception_exit(config['server'])
    _ERROR_HANDLER_WAS_REGISTERED = True
  return config

### Sub process management

def _cleanup_bot_directory(botobj):
  """Delete anything not expected in the swarming bot directory.

  This helps with stale work directory or any unexpected junk that could cause
  this bot to self-quarantine. Do only this when running from the zip.
  """
  if not _is_base_dir_ok(botobj):
    # That's an important one-off check as cleaning the $HOME directory has
    # really bad effects on normal host.
    logging.error('Not cleaning root directory because of bad base directory')
    return
  for i in fs.listdir(botobj.base_dir):
    if any(fnmatch.fnmatch(i, w) for w in PASSLIST):
      continue
    try:
      p = os.path.join(botobj.base_dir, i)
      if fs.isdir(p):
        file_path.rmtree(p)
      else:
        file_path.remove(p)
    except (IOError, OSError) as e:
      botobj.post_error(
          'Failed to remove %s from bot\'s directory: %s' % (i, e))

def _run_isolated_flags(botobj):
  """Returns flags to pass to run_isolated.

  These are not meant to be processed by task_runner.py.
  """
  settings = _get_settings(botobj)
  partition = settings['free_partition']['bot']
  size = os_utilities.get_disk_size(THIS_FILE)
  min_free = (
      _min_free_disk({'size_mb': size}, partition) +
      partition['wiggle'])
  logging.info('size %d, partition %s, min_free %s', size, partition, min_free)
  args = [
      # Shared option.
      '--min-free-space',
      str(min_free),
      '--max-cache-size',
      str(settings['caches']['isolated']['size']),
      # CAS cache option.
      '--cas-cache',
      os.path.join(botobj.base_dir, 'cas_cache'),
      # Named cache option.
      '--named-cache-root',
      os.path.join(botobj.base_dir, 'c'),
  ]

  use_kvs = True
  # bot with small memory or inside docker causes out of memory.
  if os_utilities.get_physical_ram() < 2048:
    use_kvs = False

  if sys.platform == 'linux' and platforms.linux.get_inside_docker():
    use_kvs = False

  if sys.platform == 'win32' and platform.architecture()[0] == '32bit':
    # 32 bit windows may not work with kvs.
    # https://crbug.com/1207762
    use_kvs = False

  if use_kvs:
    args += [
        '--kvs-dir',
        os.path.join(botobj.base_dir, _CAS_KVS_CACHE_DB),
    ]

  if _IN_TEST_MODE:
    args += ['--cipd-enabled', 'false']

  return args

def _Popen(botobj, cmd, **kwargs):
  """Wraps subprocess42.Popen.

  Creates a 'detached' process as per subprocess42 description.

  On Windows, also create a separate console.
  """
  kwargs.setdefault('stdout', subprocess42.PIPE)
  if sys.platform == 'win32':
    prev = kwargs.get('creationflags', 0)
    kwargs['creationflags'] = prev | subprocess42.CREATE_NEW_CONSOLE
  else:
    kwargs['close_fds'] = True
  return subprocess42.Popen(
      cmd,
      stdin=subprocess42.PIPE,
      stderr=subprocess42.STDOUT,
      cwd=botobj.base_dir,
      detached=True,
      **kwargs)

def _clean_cache(botobj):
  """Asks run_isolated to clean its cache.

  This may take a while but it ensures that in the case of a run_isolated run
  failed and it temporarily used more space than _min_free_disk, it can cleans
  up the mess properly.

  It will remove unexpected files, remove corrupted files, trim the cache size
  based on the policies and update state.json.
  """
  cmd = [
    sys.executable, THIS_FILE, 'run_isolated',
    '--clean',
    '--log-file', os.path.join(botobj.base_dir, 'logs', 'run_isolated.log'),
  ]
  cmd.extend(_run_isolated_flags(botobj))
  logging.info('Running: %s', cmd)
  try:
    # Intentionally do not use a timeout, it can take a while to hash 50gb but
    # better be safe than sorry.
    proc = _Popen(botobj, cmd)
    output, _ = proc.communicate(None)
    logging.info('Result:\n%s', output)
    if proc.returncode:
      botobj.post_error(
          'swarming_bot.zip failure during run_isolated --clean:\n%s' % output)
  except OSError:
    botobj.post_error(
        'swarming_bot.zip internal failure during run_isolated --clean')

def _post_error_task(botobj, error, task_id):
  """Posts given error as failure cause for the task.

  This is used in case of internal code error, and this causes the task to
  become BOT_DIED.

  Arguments:
    botobj: A bot.Bot instance.
    error: String representing the problem.
    task_id: Task that had an internal error. When the Swarming server sends
        commands to a bot, even though they could be completely wrong, the
        server assumes the job as running. Thus this function acts as the
        exception handler for incoming commands from the Swarming server. If for
        any reason the local test runner script can not be run successfully,
        this function is invoked.
  """
  logging.error('Error: %s', error)
  return botobj.remote.post_task_error(task_id, error)

def _run_manifest(botobj, manifest):
  """Defers to task_runner.py.

  Return True if the task succeeded.
  """
  # Ensure the manifest is valid. This can throw a json decoding error. Also
  # raise if it is empty.
  if not manifest:
    raise ValueError('Empty manifest')

  # Necessary to signal an internal_failure. This occurs when task_runner fails
  # to execute the command. It is important to note that this data is extracted
  # before any I/O is done, like writting the manifest to disk.
  task_id = manifest['task_id']
  last_ditch_timeout = manifest['hard_timeout'] or None
  # The grace period is the time between SIGTERM and SIGKILL.
  grace_period = max(manifest['grace_period'] or 0, 30)
  if last_ditch_timeout:
    # One for the child process, one for run_isolated, one for task_runner.
    last_ditch_timeout += 3 * grace_period
    # CIPD, isolated download time, plus named cache cleanup is not counted for
    # hard timeout so add more time; hard_timeout is handled by run_isolated.
    last_ditch_timeout += max(manifest['io_timeout'] or 0, 1200)

  # Get the server info to pass to the task runner so it can provide updates.
  url = botobj.remote.server
  if 'host' in manifest:
    # The URL in the manifest includes the version - eg not https://chromium-
    # swarm-dev.appspot.com, but https://<some-version>-dot-chromiium-swarm-
    # dev.appspot.com. That way, if a new server version becomes the default,
    # old bots will continue to work with a server version that can manipulate
    # the old data (the new server will only ever have to read it, which is
    # much simpler) while new bots won't accidentally contact an old server
    # which the GAE engine hasn't gotten around to updating yet.
    url = manifest['host']

  task_dimensions = manifest['dimensions']
  task_result = {}

  failure = False
  internal_failure = False
  msg = None
  auth_params_dumper = None
  must_reboot_reason = None
  # Use 'w' instead of 'work' because path length is precious on Windows.
  work_dir = os.path.join(botobj.base_dir, 'w')
  try:
    try:
      if fs.isdir(work_dir):
        file_path.rmtree(work_dir)
    except OSError:
      # If a previous task created an undeleteable file/directory inside 'w',
      # make sure that following tasks are not affected. This is done by working
      # around the undeleteable directory by creating a temporary directory
      # instead. This is not normal behavior. The bot will report a failure on
      # start.
      work_dir = tempfile.mkdtemp(dir=botobj.base_dir, prefix='w')
    else:
      try:
        fs.makedirs(work_dir)
      except OSError:
        # Sometimes it's a race condition, so do a last ditch attempt.
        work_dir = tempfile.mkdtemp(dir=botobj.base_dir, prefix='w')

    env = os.environ.copy()
    env['SWARMING_TASK_ID'] = task_id
    env['SWARMING_SERVER'] = botobj.server

    task_in_file = os.path.join(work_dir, 'task_runner_in.json')
    with fs.open(task_in_file, 'w') as f:
      f.write(json.dumps(manifest))
    handle, bot_file = tempfile.mkstemp(
        prefix='bot_file', suffix='.json', dir=work_dir)
    os.close(handle)
    task_result_file = os.path.join(work_dir, 'task_runner_out.json')
    if fs.exists(task_result_file):
      fs.remove(task_result_file)

    # Start a thread that periodically puts authentication headers and other
    # authentication related information to a file on disk. task_runner reads it
    # from there before making authenticated HTTP calls.
    #
    # TODO(vadimsh): Switch to pipes or local sockets if the latency tokens
    # propagation here becomes an issue.
    auth_params_file = os.path.join(work_dir, 'bot_auth_params.json')
    auth_params_dumper = file_refresher.FileRefresherThread(
        auth_params_file,
        lambda: bot_auth.prepare_auth_params_json(botobj, manifest))
    auth_params_dumper.start()

    command = [
        sys.executable,
        THIS_FILE,
        'task_runner',
        '--swarming-server',
        url,
        '--default-swarming-server',
        botobj.server,
        '--in-file',
        task_in_file,
        '--out-file',
        task_result_file,
        '--cost-usd-hour',
        str(botobj.state.get('cost_usd_hour') or 0.),
        # Include the time taken to poll the task in the cost.
        # TODO(vadimsh): Remove this, it doesn't add much value, just
        # complicates code.
        '--start',
        str(time.time()),
        '--bot-file',
        bot_file,
        '--auth-params-file',
        auth_params_file,
    ]
    # Flags for run_isolated.py are passed through by task_runner.py as-is
    # without interpretation.
    command.append('--')
    command.extend(_run_isolated_flags(botobj))
    _call_hook_safe(True, botobj, 'on_before_task', bot_file, command, env)
    logging.debug('Running command: %s', command)

    base_log = os.path.join(botobj.base_dir, 'logs')
    if not fs.isdir(base_log):
      # It was observed that this directory may be unexpectedly deleted.
      # Recreate as needed, otherwise it may throw at the open() call below.
      fs.mkdir(base_log)
    log_path = os.path.join(base_log, 'task_runner_stdout.log')
    os_utilities.roll_log(log_path)
    os_utilities.trim_rolled_log(log_path)
    with fs.open(log_path, 'a+b') as f:
      proc = _Popen(botobj, command, stdout=f, env=env)
      logging.info('Subprocess for task_runner started.')
      try:
        proc.wait(last_ditch_timeout)
      except subprocess42.TimeoutExpired:
        # That's the last ditch effort; as task_runner should have completed a
        # while ago and had enforced the io_timeout or run_isolated for
        # hard_timeout.
        logging.error('Sending SIGTERM to task_runner')
        proc.terminate()
        internal_failure = True
        msg = 'task_runner hung'
        try:
          proc.wait(2*grace_period)
        except subprocess42.TimeoutExpired:
          logging.error('Sending SIGKILL to task_runner')
          proc.kill()
        proc.wait()
        return False

    logging.info('task_runner exit: %d', proc.returncode)
    if fs.exists(task_result_file):
      with fs.open(task_result_file, 'rb') as fd:
        task_result = json.load(fd)

    if proc.returncode:
      # STATUS_DLL_INIT_FAILED generally means that something bad happened, and
      # a reboot magically clears things out. :(
      if sys.platform == 'win32' and proc.returncode == -1073741502:
        must_reboot_reason = ('Working around STATUS_DLL_INIT_FAILED by '
                              'task_runner')
      msg = 'Execution failed: internal error (%d).' % proc.returncode
      internal_failure = True
    elif not task_result:
      logging.warning('task_runner failed to write metadata')
      msg = 'Execution failed: internal error (no metadata).'
      internal_failure = True
    elif task_result['must_signal_internal_failure']:
      msg = (
        'Execution failed: %s' % task_result['must_signal_internal_failure'])
      internal_failure = True

    failure = bool(task_result.get('exit_code')) if task_result else False
    return not internal_failure and not failure
  except Exception as e:
    # Failures include IOError when writing if the disk is full, OSError if
    # swarming_bot.zip doesn't exist anymore, etc.
    logging.exception('_run_manifest failed')
    msg = 'Internal exception occurred: %s\n%s' % (
        e, traceback.format_exc()[-2048:])
    internal_failure = True
  finally:
    if auth_params_dumper:
      auth_params_dumper.stop()
    if internal_failure:
      _post_error_task(botobj, msg, task_id)
    logging.info('calling on_after_task: failure=%s, internal_failure=%s, '
                 'task_dimensions=%s, task_result=%s',
                 failure, internal_failure, task_dimensions, task_result)
    _call_hook_safe(
        True, botobj, 'on_after_task', failure, internal_failure,
        task_dimensions, task_result)
    if fs.isdir(work_dir):
      try:
        file_path.rmtree(work_dir)
      except Exception:
        botobj.post_error('Failed to delete work directory %s: %s' %
                          (work_dir, traceback.format_exc()[-2048:]))
        # Failure to delete could be due to a proc with open file handles. Just
        # reboot the machine in that case.
        must_reboot_reason = 'Failure to remove %s' % work_dir
    if must_reboot_reason:
      botobj.host_reboot(must_reboot_reason)

### Bot lifetime management

def _run_bot(arg_error):
  """Runs _run_bot_inner() with a signal handler."""
  # The quit_bit is to signal that the bot process must shutdown. It is
  # different from a request to restart the bot process or reboot the host.
  quit_bit = threading.Event()
  def handler(sig, _):
    # A signal terminates the bot process, it doesn't cause it to restart.
    logging.info('Got signal %s', sig)
    quit_bit.set()

  # TODO(maruel): Set quit_bit when stdin is closed on Windows.

  with subprocess42.set_signal_handler(subprocess42.STOP_SIGNALS, handler):
    return _run_bot_inner(arg_error, quit_bit)

def _run_bot_inner(arg_error, quit_bit):
  """Runs the bot until an event occurs.

  One of the three following even can occur:
  - host reboots
  - bot process restarts (this includes self-update)
  - bot process shuts down (this includes a signal is received)
  """
  # If Swarming bot inherited SWARMING_BOT_ID env var, it must eventually use
  # it for all communications with the server. We capture it very early because
  # hooks may potentially modify it, we want to detect this.
  global _ORIGINAL_BOT_ID
  _ORIGINAL_BOT_ID = os.environ.get('SWARMING_BOT_ID')

  config = get_config()
  if config.get('enable_ts_monitoring'):
    _init_ts_mon()
  try:
    # First thing is to get an arbitrary url. This also ensures the network is
    # up and running, which is necessary before trying to get the FQDN below.
    # There's no need to do error handling here - the "ping" is just to "wake
    # up" the network; if there's something seriously wrong, the handshake will
    # fail and we'll handle it there.
    hostname = _get_botid_safe()
    base_dir = os.path.dirname(THIS_FILE)
    remote = remote_client.createRemoteClient(config['server'], None, hostname,
                                              base_dir)
    remote.ping()
  except Exception:
    # url_read() already traps pretty much every exceptions. This except
    # clause is kept there "just in case".
    logging.exception('server_ping threw')

  # If we are on GCE, we want to make sure GCE metadata server responds, since
  # we use the metadata to derive bot ID, dimensions and state.
  if platforms.is_gce():
    logging.info('Running on GCE, waiting for the metadata server')
    platforms.gce.wait_for_metadata(quit_bit)
    if quit_bit.is_set():
      logging.info('Early quit 1')
      return 0

  # Next we make sure the bot can make authenticated calls by grabbing the auth
  # headers, retrying on errors a bunch of times. We don't give up if it fails
  # though (maybe the bot will "fix itself" later).
  botobj = get_bot(config)
  try:
    botobj.remote.initialize(quit_bit)
  except remote_client.InitializationError as exc:
    botobj.post_error('failed to grab auth headers: %s' % exc.last_error)
    logging.error('Can\'t grab auth headers, continuing anyway...')

  if arg_error:
    botobj.post_error('Bootstrapping error: %s' % arg_error)

  if quit_bit.is_set():
    logging.info('Early quit 2')
    return 0

  _call_hook_safe(True, botobj, 'on_bot_startup')

  # Initial attributes passed to bot.Bot in get_bot above were constructed for
  # 'fake' bot ID ('none'). Refresh them to match the real bot ID, now that we
  # have fully initialize bot.Bot object. Note that 'get_dimensions' and
  # 'get_state' may depend on actions done by 'on_bot_startup' hook, that's why
  # we do it here and not in 'get_bot'.
  _update_bot_attributes(botobj, 0)
  if quit_bit.is_set():
    logging.info('Early quit 3')
    return 0

  _do_handshake(botobj, quit_bit)

  if quit_bit.is_set():
    logging.info('Early quit 4')
    return 0

  # Let the bot to finish the initialization, now that it knows its server
  # defined dimensions.
  _call_hook_safe(True, botobj, 'on_handshake')

  _cleanup_bot_directory(botobj)
  _clean_cache(botobj)

  if quit_bit.is_set():
    logging.info('Early quit 5')
    return 0

  # This environment variable is accessible to the tasks executed by this bot.
  os.environ['SWARMING_BOT_ID'] = botobj.id
  if not _ORIGINAL_BOT_ID:
    _ORIGINAL_BOT_ID = botobj.id

  # Spin until getting a termination signal.
  _BotLoopState(botobj, quit_bit).run()

  # Tell the server we are going away.
  botobj.post_event('bot_shutdown', 'Signal was received')
  return 0

def _sleep_until_signaled(quit_bit, timeout):
  """Sleeps until the threading.Event is signaled or the duration expires.

  Returns True if quit_bit is signaled upon return.
  """
  # TODO(vadimsh): On Python 3.4 and older wait(...) may raise an exception if
  # it is interrupted by a signal. When this happens we should resume sleeping,
  # but this requires a monotonic clock to be precise. Monotonic clocks also
  # appeared on all platforms only in Python 3.5. Instead use time.time() and
  # just make sure we never sleep longer than the originally requested
  # `timeout`. That way the function will at least terminate fast enough, even
  # if the clock jumps. It may be grossly inaccurate, but we don't need accuracy
  # and clock shifts should be rare anyway.
  while timeout > 0:
    start = time.time()
    try:
      quit_bit.wait(timeout)
      break
    except IOError:
      logging.exception('IOError while waiting in threading.Event')
      timeout -= max(0, time.time() - start)
  return quit_bit.is_set()

class _BotLoopState:
  """The state of the main bot poll loop."""

  def __init__(self, botobj, quit_bit):
    # Instance of Bot.
    self._bot = botobj
    # threading.Event signaled when the bot should gracefully exit.
    self._quit_bit = quit_bit
    # Number of consecutive Swarming poll failures (either remote or local).
    self._consecutive_errors = 0
    # Number of consecutive times the server asked the bot to sleep.
    self._consecutive_sleeps = 0
    # Sleep duration requested by the Swarming server the last time.
    self._requested_sleep_duration = 0.0
    # When the last task finished running (successfully or not).
    self._last_task_time = time.time()
    # Number of times the bot failed to exit when asked. Never resets.
    self._bad_bot_state_errors = 0

  def run(self):
    """Spins executing Swarming commands until getting a termination signal."""
    while not _sleep_until_signaled(self._quit_bit, self.sleep_duration):
      # Note: on_before_poll should always be called before get_state and
      # get_dimensions (i.e. before _update_bot_attributes). It's part of the
      # bot hooks API contract.
      _call_hook_safe(False, self._bot, 'on_before_poll')
      _update_bot_attributes(self._bot, self._consecutive_sleeps)
      cmd, param, error = None, None, None
      try:
        cmd, param = self._bot.remote.poll(self._bot.attributes)
        logging.debug('Swarming server response:\n%s: %s', cmd, param)
      except remote_client_errors.PollError as e:
        error = 'poll RPC failed: %s' % e
      _call_hook_safe(False, self._bot, 'on_after_poll', cmd)

      try:
        if cmd == 'sleep':
          self.cmd_sleep(param['duration'], param['rbe'])
        elif cmd == 'terminate':
          self.cmd_terminate(param)
        elif cmd == 'run':
          self.cmd_run(param)
        elif cmd == 'update':
          self.cmd_update(param)
        elif cmd in ('host_reboot', 'restart'):
          self.cmd_host_reboot(param)
        elif cmd == 'bot_restart':
          self.cmd_bot_restart(param)
        elif cmd:
          error = 'unrecognized Swarming command %s' % cmd
      except Exception as e:
        if not _TRAP_ALL_EXCEPTIONS:
          raise
        logging.exception('Unexpected error executing Swarming command %s', cmd)
        msg = '%s\n%s' % (e, traceback.format_exc()[-2048:])
        self._bot.post_error(msg)
        error = 'unexpected error executing Swarming command %s' % cmd

      # This manages the state of the error retry loop.
      self.handler_done(error)

      # Call `on_bot_idle` hook only if the bot didn't execute any task this
      # cycle. This is part of `on_bot_idle` hook API contract: if the bot
      # executes tasks back to back with no idle time in-between, the hook
      # should **not** be called.
      if self.idle:
        _call_hook_safe(True, self._bot, 'on_bot_idle', self.idle_duration)

  @property
  def idle(self):
    """True if the bot didn't execute any tasks the last poll cycle."""
    return (self._consecutive_sleeps or self._consecutive_errors
            or self._bad_bot_state_errors)

  @property
  def idle_duration(self):
    """Duration since the bot finished executing the last task."""
    return max(0, time.time() - self._last_task_time)

  @property
  def sleep_duration(self):
    """Calculates how long to wait between Swarming polls."""
    if self._bad_bot_state_errors:
      # Something super sad happened. The bot is quarantined already, but if it
      # doesn't help, reduce the poll frequency to avoid busy-looping and DDoS
      # of the server. This state is reset only upon bot process restart.
      return 2**min(self._bad_bot_state_errors, 8)
    if self._consecutive_errors:
      # There are transient errors polling the server. Slow down. This is reset
      # on a successful poll.
      return 1.4**min(self._consecutive_errors, 16)
    # If there were no errors, just sleep as long as the server asked us.
    return self._requested_sleep_duration

  def handler_done(self, error):
    """Called whenever a command is finished processing."""
    if error:
      logging.error('Poll loop error: %s', error)
      self._consecutive_errors += 1
    else:
      self._consecutive_errors = 0

  def cmd_sleep(self, duration, rbe):
    """Called when Swarming asks the bot to sleep."""
    self._consecutive_sleeps += 1
    self._requested_sleep_duration = duration
    _maybe_update_lkgbc(self._bot)
    # TODO(vadimsh): Move RBE state management elsewhere.
    with self._bot.mutate_internals() as mut:
      mut.update_rbe_state(rbe)

  def cmd_terminate(self, task_id):
    """Called when Swarming asks the bot to gracefully terminate."""
    self._quit_bit.set()
    try:
      # Duration must be set or server IEs. For that matter, we've never cared
      # if there's an error here before, so let's preserve that behavior
      # (though anything that's not a remote_client.InternalError will make
      # it through, again preserving prior behavior).
      self._bot.remote.post_task_update(task_id, {'duration': 0}, None, 0)
    except remote_client_errors.InternalError:
      pass

  def cmd_run(self, manifest):
    """Called when Swarming asks the bot to execute a task."""
    # Actually execute the task. This can block for many hours.
    success = _run_manifest(self._bot, manifest)
    # Unconditionally clean up cache after each task. This is done *after* the
    # task is terminated, so that:
    # - there's no task overhead
    # - if there's an exception while cleaning, it's not logged in the task
    _clean_cache(self._bot)
    if success:
      # Completed a task successfully so update swarming_bot.zip if necessary.
      _update_lkgbc(self._bot)
    # This is used to track the bot idleness.
    self._consecutive_sleeps = 0
    self._requested_sleep_duration = 0.0
    self._last_task_time = time.time()

  def cmd_update(self, version):
    """Called when Swarming asks the bot to self-update."""
    _update_bot(self._bot, version)
    self._should_have_exited_but_didnt('Failed to self-update the bot')

  def cmd_host_reboot(self, message):
    """Called when Swarming asks the bot to reboot the host machine."""
    self._bot.host_reboot(message)
    self._should_have_exited_but_didnt('Failed to reboot the host')

  def cmd_bot_restart(self, message):
    """Called when Swarming asks the bot to restart its own process."""
    _bot_restart(self._bot, message)
    self._should_have_exited_but_didnt('Failed to restart the bot process')

  def _should_have_exited_but_didnt(self, message):
    _set_quarantined(message)
    self._bad_bot_state_errors += 1

def _do_handshake(botobj, quit_bit):
  """Connects to /handshake and reads the bot_config if specified."""
  # This is the first authenticated request to the server. If the bot is
  # misconfigured, the request may fail with HTTP 401 or HTTP 403. Instead of
  # dying right away, spin in a loop, hoping the bot will "fix itself"
  # eventually. Authentication errors in /handshake are logged on the server and
  # generate error reports, so bots stuck in this state are discoverable.
  sleep_time = 0
  while not _sleep_until_signaled(quit_bit, sleep_time):
    resp = botobj.remote.do_handshake(botobj.attributes)
    if resp:
      logging.info('Connected to %s', resp.get('server_version'))
      if resp.get('bot_version') != botobj.bot_version:
        logging.warning(
            'Found out we\'ll need to update: server said %s; we\'re %s',
            resp.get('bot_version'), botobj.bot_version)
      # Register the Python module with bot hooks.
      content = resp.get('bot_config')
      rev = resp.get('bot_config_rev')
      script = resp.get('bot_config_name')
      if content:
        _register_extra_bot_config(content, rev, script)
      # Remember the server-provided per-bot configuration. '/handshake' is
      # the only place where the server returns it. The bot will be sending
      # the 'bot_group_cfg_version' back in each /poll (as part of 'state'),
      # so that the server can instruct the bot to restart itself when
      # config changes.
      with botobj.mutate_internals() as mut:
        mut.update_bot_config(script, rev)
        cfg_version = resp.get('bot_group_cfg_version')
        if cfg_version:
          mut.update_bot_group_cfg(cfg_version, resp.get('bot_group_cfg'))
      break
    sleep_time = min(300, sleep_time * 2) if sleep_time else 5
    logging.error(
        'Failed to contact for handshake, retrying in %d sec...', sleep_time)

def _update_bot(botobj, version):
  """Downloads the new version of the bot code and then runs it.

  Use alternating files; first load swarming_bot.1.zip, then swarming_bot.2.zip,
  never touching swarming_bot.zip which was the originally bootstrapped file.

  LKGBC is handled by _update_lkgbc() and _maybe_update_lkgbc().

  Returns only in case of failure to get the new bot code.
  """
  # Alternate between .1.zip and .2.zip.
  new_zip = 'swarming_bot.1.zip'
  if os.path.basename(THIS_FILE) == new_zip:
    new_zip = 'swarming_bot.2.zip'
  new_zip = os.path.join(botobj.base_dir, new_zip)

  # Download as a new file.
  try:
    botobj.remote.get_bot_code(new_zip, version)
  except remote_client.BotCodeError as e:
    botobj.post_error(str(e))
  else:
    _bot_restart(botobj, 'Updating to %s' % version, filepath=new_zip)

def _bot_restart(botobj, message, filepath=None):
  """Restarts the bot process, optionally in a new file.

  The function will return if the new bot code is not valid.
  """
  filepath = filepath or THIS_FILE
  s = fs.stat(filepath)
  logging.info('Restarting to %s; %d bytes.', filepath, s.st_size)
  sys.stdout.flush()
  sys.stderr.flush()

  proc = _Popen(botobj, [sys.executable, filepath, 'is_fine'])
  output, _ = proc.communicate()
  if proc.returncode:
    botobj.post_error(
        'New bot code is bad: proc exit = %s. stdout:\n%s' %
        (proc.returncode, output))
    if sys.platform == 'win32' and proc.returncode == -1073741502:
      # STATUS_DLL_INIT_FAILED generally means that something bad happened, and
      # a reboot magically clears things out. :(
      botobj.host_reboot(
          'Working around STATUS_DLL_INIT_FAILED when restarting the bot')
    return

  botobj.post_event('bot_shutdown', 'About to restart: %s' % message)

  # Sleep a bit to make sure new bot process connects to a GAE instance with
  # the fresh bot group config cache (it gets refreshed each second). This makes
  # sure the bot doesn't accidentally pick up the old config after restarting
  # and connecting to an instance with a stale cache.
  time.sleep(2)

  # Don't forget to release the singleton before restarting itself.
  SINGLETON.release()

  # Do not call on_bot_shutdown.
  # On OSX, launchd will be unhappy if we quit so the old code bot process has
  # to outlive the new code child process. Launchd really wants the main process
  # to survive, and it'll restart it if it disappears. os.exec*() replaces the
  # process so this is fine.
  cmd = [filepath, 'start_slave', '--survive']
  if _IN_TEST_MODE:
    cmd.append('--test-mode')
  logging.debug('Restarting bot, cmd: %s', cmd)
  ret = common.exec_python(cmd)
  if ret in (1073807364, -1073741510):
    # 1073807364 is returned when the process is killed due to shutdown. No need
    # to alert anyone in that case.
    # -1073741510 is returned when rebooting too. This can happen when the
    # parent code was running the old version and gets confused and decided to
    # poll again.
    # In any case, zap out the error code.
    ret = 0
  elif ret:
    botobj.post_error('Bot failed to respawn after update: %s' % ret)
  sys.exit(ret)

def _update_lkgbc(botobj):
  """Updates the Last Known Good Bot Code if necessary.

  Returns True if LKGBC was updated.
  """
  try:
    if not fs.isfile(THIS_FILE):
      # TODO(maruel): Try to download the code again from the server.
      botobj.post_error('Missing file %s for LKGBC' % THIS_FILE)
      return False

    golden = os.path.join(botobj.base_dir, 'swarming_bot.zip')
    if fs.isfile(golden):
      org = fs.stat(golden)
      cur = fs.stat(THIS_FILE)
      if org.st_size == org.st_size and org.st_mtime >= cur.st_mtime:
        return False

    # Copy the current file back to LKGBC.
    shutil.copy(THIS_FILE, golden)
    return True
  except Exception as e:
    botobj.post_error('Failed to update LKGBC: %s' % e)
    return False

def _maybe_update_lkgbc(botobj):
  """Updates the Last Known Good Bot Code (LKGBC) when it is older than 1 one
  week.

  This either means:
  - The bot code is particularly hosed, for an extended period of time.
  - All tasks are failing, which may legitimitely happen sometimes when the devs
    don't care about a particular configuration.
  - The bot is completely idle, so it never gets the chance to update LKGBC.

  We decide that the first situation is rare enough that it's preferable to
  explcitly handle the later two. We've seen bots being idle for extended
  periods of time (well over a year), which introduces all sorts of
  'interesting' problems.

  Returns True if LKGBC was updated.
  """
  try:
    if not fs.isfile(THIS_FILE):
      # TODO(maruel): Try to download the code again from the server.
      return False
    golden = os.path.join(botobj.base_dir, 'swarming_bot.zip')
    if fs.isfile(golden):
      org = fs.stat(golden)
      cur = fs.stat(THIS_FILE)
      if org.st_size == org.st_size and org.st_mtime >= cur.st_mtime:
        return False
      if org.st_mtime >= time.time() - 7*24*60*60:
        return False

    # Copy the current file back to LKGBC.
    shutil.copy(THIS_FILE, golden)
    return True
  except Exception as e:
    botobj.post_error('Failed to update LKGBC while idle: %s' % e)
    return False

def main(argv):
  subprocess42.inhibit_os_error_reporting()

  # Disable magical auto-detection of OAuth config. bot_main.py prepares auth
  # headers on its own explicitly (via get_authentication_headers hook) when
  # using 'net' library through RemoteClientNative class and auto-configured
  # auth in net.py may interfere with this. We also disable auto-detection in
  # task_runner.py (since it also uses special mechanism for getting auth
  # headers from bot_main.py). We do _not_ disable auto-detection in
  # run_isolated.py, since at this layer we have an auth context (setup by
  # task_runner.py) and it is correctly getting recognized by the auto-detection
  # in net.py.
  net.disable_oauth_config()

  # Add SWARMING_HEADLESS into environ so subcommands know that they are running
  # in a headless (non-interactive) mode.
  os.environ['SWARMING_HEADLESS'] = '1'

  # Kick out any lingering env vars that will be set later.
  os.environ.pop('SWARMING_SERVER', None)
  os.environ.pop('SWARMING_TASK_ID', None)

  # The only reason this is kept is to enable the unit test to use --help to
  # quit the process.
  parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  parser.add_argument('unsupported', nargs='*', help=argparse.SUPPRESS)
  parser.add_argument('--test-mode', action='store_true')
  args = parser.parse_args(argv)

  global _IN_TEST_MODE
  _IN_TEST_MODE = args.test_mode
  if _IN_TEST_MODE:
    logging.debug('bot_main running in TEST mode')

  if sys.platform == 'win32':
    if not file_path.enable_privilege('SeShutdownPrivilege'):
      logging.error('Failed to enable SeShutdownPrivilege')

  # Enforces that only one process with a bot in this directory can be run on
  # this host at once.
  if not SINGLETON.acquire():
    if sys.platform == 'darwin':
      msg = (
          'Found a previous bot, %d rebooting as a workaround for '
          'https://crbug.com/569610.') % os.getpid()
    else:
      msg = ('Found a previous bot, %d rebooting as a workaround for '
             'https://crbug.com/1061531' % os.getpid())
    print(msg, file=sys.stderr)
    _PRE_REBOOT_SLEEP_SECS = 5 * 60
    print('Sleeping for %s secs' % _PRE_REBOOT_SLEEP_SECS, file=sys.stderr)
    time.sleep(_PRE_REBOOT_SLEEP_SECS)
    os_utilities.host_reboot(msg)
    return 1

  base_dir = os.path.dirname(THIS_FILE)
  for t in ('out', 'err'):
    log_path = os.path.join(base_dir, 'logs', 'bot_std%s.log' % t)
    os_utilities.roll_log(log_path)
    os_utilities.trim_rolled_log(log_path)

  error = None
  if len(args.unsupported) != 0:
    error = 'Unexpected arguments: %s' % args
  try:
    return _run_bot(error)
  finally:
    _call_hook_safe(
        True, bot.Bot(None, None, None, None, base_dir, None),
        'on_bot_shutdown')
    logging.info('main() returning')


text:
namespace Microsoft.Azure.ApplicationInsights.Query.Models
{
    using Newtonsoft.Json;
    using System.Linq;

    /// <summary>
    /// The request info
    /// </summary>
    public partial class EventsRequestInfo
    {
        /// <summary>
        /// Initializes a new instance of the EventsRequestInfo class.
        /// </summary>
        public EventsRequestInfo()
        {
            CustomInit();
        }

        /// <summary>
        /// Initializes a new instance of the EventsRequestInfo class.
        /// </summary>
        /// <param name="name">The name of the request</param>
        /// <param name="url">The URL of the request</param>
        /// <param name="success">Indicates if the request was
        /// successful</param>
        /// <param name="duration">The duration of the request</param>
        /// <param name="performanceBucket">The performance bucket of the
        /// request</param>
        /// <param name="resultCode">The result code of the request</param>
        /// <param name="source">The source of the request</param>
        /// <param name="id">The ID of the request</param>
        public EventsRequestInfo(string name = default(string), string url = default(string), string success = default(string), double? duration = default(double?), string performanceBucket = default(string), string resultCode = default(string), string source = default(string), string id = default(string))
        {
            Name = name;
            Url = url;
            Success = success;
            Duration = duration;
            PerformanceBucket = performanceBucket;
            ResultCode = resultCode;
            Source = source;
            Id = id;
            CustomInit();
        }

        /// <summary>
        /// An initialization method that performs custom operations like setting defaults
        /// </summary>
        partial void CustomInit();

        /// <summary>
        /// Gets or sets the name of the request
        /// </summary>
        [JsonProperty(PropertyName = "name")]
        public string Name { get; set; }

        /// <summary>
        /// Gets or sets the URL of the request
        /// </summary>
        [JsonProperty(PropertyName = "url")]
        public string Url { get; set; }

        /// <summary>
        /// Gets or sets indicates if the request was successful
        /// </summary>
        [JsonProperty(PropertyName = "success")]
        public string Success { get; set; }

        /// <summary>
        /// Gets or sets the duration of the request
        /// </summary>
        [JsonProperty(PropertyName = "duration")]
        public double? Duration { get; set; }

        /// <summary>
        /// Gets or sets the performance bucket of the request
        /// </summary>
        [JsonProperty(PropertyName = "performanceBucket")]
        public string PerformanceBucket { get; set; }

        /// <summary>
        /// Gets or sets the result code of the request
        /// </summary>
        [JsonProperty(PropertyName = "resultCode")]
        public string ResultCode { get; set; }

        /// <summary>
        /// Gets or sets the source of the request
        /// </summary>
        [JsonProperty(PropertyName = "source")]
        public string Source { get; set; }

        /// <summary>
        /// Gets or sets the ID of the request
        /// </summary>
        [JsonProperty(PropertyName = "id")]
        public string Id { get; set; }

    }
}

redpajama_set_name: RedPajamaGithub


text:
layout: article
title: "Naturally lit with sunlight"
excerpt: "PaperFaces portrait of @rossbelmont drawn with Paper by 53 on an iPad."
image:
  feature: paperfaces-rossbelmont-twitter-lg.jpg
  thumb: paperfaces-rossbelmont-twitter-150.jpg
category: paperfaces
tags: [portrait, illustration, paper by 53, beard]
---

PaperFaces portrait of [@rossbelmont](http://twitter.com/rossbelmont).

{% include paperfaces-boilerplate-2.html %}

Work in progress screenshots (Paper by 53).


text:
        <form class="form-horizontal" method="post" action="<?=base_url('usuario/update')?>">
          <input name="idusuario" type="hidden" class="idusuario">

            <button type="button" class="close" data-dismiss="modal" aria-label="Close">×</button>

 Editar Dados Usuário

              Nome:

                <input name="nome" type="text" placeholder="Nome" class="form-control input-md nome" required minlength="5" maxlength="40">

              Email:

                  <input name="email" type="email" placeholder="email@site.com"  class="form-control input-md email" required >

              Ativo

                  <input type="radio" class="ativo-sim" name="ativo" value="sim">Sim

                  <input type="radio" class="ativo-nao" name="ativo" value="não">Não

              Perfil:

                <select name="idperfil" class="form-control input-md perfil" required>
                  <option value="">Selecione uma opção</option>
                  <?php foreach($perfis as $perfil):?>
                    <option value="<?=$perfil['idperfil']?>"><?=strtoupper($perfil['nome_perfil'])?></option>
                  <?php endforeach; ?>
                </select>

              Ver Relatórios

                  <input type="radio" class="ver_relatorios-sim" name="ver_relatorios" value="sim">Sim

                  <input type="radio" class="ver_relatorios-nao" name="ver_relatorios" value="não">Não

            <?php if($this->session->userdata('idperfil')==1):?>
              <button type="button" class="btn btn-default pull-left" id="btn_del">Excluir</button>
              <button type="button" class="btn btn-default pull-left" id="btn_reset">Resetar Senha</button>
            <?php endif;?>
            <button type="button" class="btn btn-default" data-dismiss="modal">Cancelar</button>
            <button type="submit" class="btn btn-primary">Salvar</button>

    </form>


text:
<?php

namespace Phalcon\Annotations\Adapter {

    /**
     * Phalcon\Annotations\Adapter\Xcache
     *
     * Stores the parsed annotations to XCache. This adapter is suitable for production
     *
     *
     * $annotations = new \Phalcon\Annotations\Adapter\Xcache();
     *
     */
    class Xcache extends \Phalcon\Annotations\Adapter implements \Phalcon\Annotations\AdapterInterface
    {

        /**
         * Reads parsed annotations from XCache
         *
         * @param string $key
         * @return \Phalcon\Annotations\Reflection
         */
        public function read($key)
        {
        }

        /**
         * Writes parsed annotations to XCache
         *
         * @param string $key
         * @param \Phalcon\Annotations\Reflection $data
         */
        public function write($key, $data)
        {
        }

    }
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.AspNetCore.Cors.Infrastructure;

namespace AutoRenter.Api
{
    public class CorsProvider
    {
        private readonly bool isDevelopment;
        private readonly string[] origins;

        private readonly string[] methods = new List<string>()
        {
            { "GET" },
            { "POST" },
            { "PUT" },
            { "DELETE" },
            { "OPTIONS" }
        }.ToArray();

        private readonly string[] exposedHeaders = new List<string>()
        {
            { "x-total-count" },
        }.ToArray();

        public CorsProvider(bool isDevelopment, string[] origins)
        {
            this.isDevelopment = isDevelopment;
            this.origins = origins;
        }

        public CorsPolicy GetCorsPolicy()
        {
            return isDevelopment ? DevelopmentPolicy() : ProductionPolicy();
        }

        private CorsPolicy DevelopmentPolicy()
        {
            var targetOrigins = origins.Any()
                ? origins
                : new List<string>() { "http://localhost:8080", "http://127.0.0.1:8080" }.ToArray();

            return BuildPolicy(targetOrigins, TimeSpan.FromSeconds(5));
        }

        private CorsPolicy ProductionPolicy()
        {
            var targetOrigins = origins.Any()
                ? origins
                : new string[0];

            return BuildPolicy(targetOrigins, TimeSpan.FromMinutes(10));
        }

        private CorsPolicy BuildPolicy(string[] targetOrigins, TimeSpan preflightMaxAge)
        {
            var corsBuilder = new CorsPolicyBuilder();
            corsBuilder.WithOrigins(targetOrigins)
                       .WithMethods(methods)
                       .AllowAnyHeader()
                       .WithExposedHeaders(exposedHeaders)
                       .SetPreflightMaxAge(preflightMaxAge);

            return corsBuilder.Build();
        }
    }
}


text:
package com.osmet.tcs.config;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import org.springframework.oxm.jaxb.Jaxb2Marshaller;
import org.springframework.ws.config.annotation.EnableWs;
import org.springframework.ws.config.annotation.WsConfigurerAdapter;
import org.springframework.ws.server.endpoint.adapter.method.MethodArgumentResolver;
import org.springframework.ws.soap.server.endpoint.SoapFaultAnnotationExceptionResolver;
import org.springframework.ws.soap.server.endpoint.SoapFaultDefinition;
import org.springframework.ws.soap.server.endpoint.SoapFaultMappingExceptionResolver;
import org.springframework.ws.wsdl.wsdl11.DefaultWsdl11Definition;
import org.springframework.xml.xsd.SimpleXsdSchema;
import org.springframework.xml.xsd.XsdSchema;
import org.springframework.xml.xsd.commons.CommonsXsdSchemaCollection;

import java.util.List;
import java.util.Properties;

/**
 * Created by osmet on 21.06.16.
 */
@Configuration
@EnableWs
@ComponentScan(basePackages = {"com.osmet.tcs.ws"})
public class MyEndpointConfig extends WsConfigurerAdapter {

    public static final String targetNamespace = "http://osmet.com/transport-company";

    @Bean(name = "TransportCompany") //wsdl name in http://localhost:8080/tcs/TransportCompany.wsdl
    public DefaultWsdl11Definition transportCompanyWsdl11Definition(XsdSchema transportCompanySchema) {
        DefaultWsdl11Definition wsdl11Definition = new DefaultWsdl11Definition();
        wsdl11Definition.setPortTypeName("TransportCompanyPortType");
        wsdl11Definition.setTargetNamespace("http://osmet.com/transport-company");
        wsdl11Definition.setServiceName("TransportCompany");
        wsdl11Definition.setSchema(transportCompanySchema);
        return wsdl11Definition;
    }

    @Bean
    public XsdSchema transportCompanySchema() {
        return new SimpleXsdSchema(new ClassPathResource("/transport-company.xsd"));
    }

    @Bean
    public Jaxb2Marshaller marshaller() {
        Jaxb2Marshaller marshaller = new Jaxb2Marshaller();
        marshaller.setPackagesToScan("com.osmet.tcs.model", "com.osmet.tcs.schema");
        return marshaller;
    }

}


text:
namespace PaintsNow {
	class GLSLShaderGenerator : public IReflect {
	public:
		GLSLShaderGenerator(IRender::Resource::ShaderDescription::Stage s, uint32_t& pinputIndex, uint32_t& poutputIndex, uint32_t& pbindingIndex);

		static const String& GetFrameCode();
		static String FormatCode(const String& input);
		void Complete();
		void Property(IReflectObject& s, Unique typeID, Unique refTypeID, const char* name, void* base, void* ptr, const MetaChainBase* meta) override;
		void Method(const char* name, const TProxy<>* p, const Param& retValue, const std::vector<Param>& params, const MetaChainBase* meta) override;

		IRender::Resource::ShaderDescription::Stage stage;
		bool forceLayout; // SPIRV requires force layout
		bool vulkanNDC; // Vulkan requires reverse-Y
		bool reserved[2];

		String declaration;
		String initialization;
		String finalization;

		uint32_t debugVertexBufferIndex;
		uint32_t propertyLevel;
		uint32_t& inputIndex;
		uint32_t& outputIndex;
		uint32_t& bindingIndex;

		template <class T>
		struct Binding {
			Binding() {}
			Binding(const T* p, uint32_t b, const String& n) : pointer(p), binding(b), name(n) {}

			const T* pointer;
			uint32_t binding;
			String name;
		};

		std::vector<Binding<IShader::BindBuffer> > bufferBindings;
		std::vector<Binding<IShader::BindTexture> > textureBindings;
		std::map<String, String> mapStructureDefinition;
		std::vector<String> structures;
		std::vector<IAsset::Material::Variable> constants;

	protected:
		std::map<const IShader::BindBuffer*, std::pair<String, String> > mapBufferDeclaration;
		std::map<const IShader::BindBuffer*, bool> mapBufferEnabled;
	};
}


text:
// About: Examples
// http://pesblog.github.com/zepto-touch-click

;(function ( $, window, undefined ) {

    var pluginName = 'touchClick';
    var defaults = {
    	className: 'active',
        callback: function(){
            console.log('touchClick');
        }
    };

    var isTouch = ('ontouchend' in window);
    var touchstartEvent = isTouch ? 'touchstart.'+pluginName : 'mousedown.'+pluginName;
    var touchmoveEvent = isTouch ? 'touchmove.'+pluginName : 'mousemove.'+pluginName;
    var touchendEvent = isTouch ? 'touchend.'+pluginName : 'mouseup.'+pluginName;

    function Plugin( element, className, callback ) {
	    this.element = element;
        if (typeof className === 'function'){
            this.options = {};
            this.options.className = defaults.className;
            this.options.callback = className;
        } else {
            this.options = {
                className: className || defaults.className,
                callback: callback || defaults.callback
            };
        }
	    this._defaults = defaults;
	    this._name = pluginName;
	    this.init();
    }

    Plugin.prototype.init = function () {
        var self = this;
        var $element = $(self.element);
        var className = self.options.className;
        var callback = self.options.callback;
        $element
            .bind(touchstartEvent, function(e) {
                this.touchClickStart = true;
                $element.addClass( className );
            })
            .bind(touchmoveEvent, function(e) {
                if ( this.touchClickStart ) {
                    this.touchClickStart = undefined;
                    $element.removeClass( className );
                }
            })
            .bind(touchendEvent, function(e) {
                var dom = this;
                if ( this.touchClickStart ) {
                    this.touchClickStart = undefined;
                    $element.removeClass( className );
                    setTimeout(function(){
                        $.proxy(callback, dom)(e)
                    }, 0);
                }
            });
    };

    $.fn[pluginName] = function ( className, callback ) {
	    return this.each(function () {
		    new Plugin( this, className, callback );
	    });
    };

})( Zepto, window, undefined );


text:
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Fitbit.Api.Portable;
using Fitbit.Models;
using FluentAssertions;
using NUnit.Framework;

namespace Fitbit.Portable.Tests
{
    [TestFixture]
    public class WaterTests
    {
        [Test] [Category("Portable")]
        public async void GetWaterAsync_Success()
        {
            string content = SampleDataHelper.GetContent("GetWater-WaterData.json");

            var responseMessage = new Func<HttpResponseMessage>(() =>
            {
                return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(content) };
            });

            var verification = new Action<HttpRequestMessage, CancellationToken>((message, token) =>
            {
                Assert.AreEqual(HttpMethod.Get, message.Method);
                Assert.AreEqual("https://api.fitbit.com/1/user/-/foods/log/water/date/2015-01-12.json", message.RequestUri.AbsoluteUri);
            });

            var fitbitClient = Helper.CreateFitbitClient(responseMessage, verification);

            var response = await fitbitClient.GetWaterAsync(new DateTime(2015, 1, 12));

            ValidateWaterData(response);
        }

        [Test] [Category("Portable")]
        public void GetWaterAsync_Errors()
        {
            var responseMessage = Helper.CreateErrorResponse();
            var verification = new Action<HttpRequestMessage, CancellationToken>((message, token) =>
            {
                Assert.AreEqual(HttpMethod.Get, message.Method);
            });

            var fitbitClient = Helper.CreateFitbitClient(responseMessage, verification);

            Func<Task<Food>> result = () => fitbitClient.GetFoodAsync(new DateTime(2015, 1, 12));

            result.ShouldThrow<FitbitException>();
        }

        [Test] [Category("Portable")]
        public async void PostWaterLogAsync_Success()
        {
            string content = SampleDataHelper.GetContent("LogWater-WaterLog.json");

            var responseMessage = new Func<HttpResponseMessage>(() =>
            {
                return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(content) };
            });

            var verification = new Action<HttpRequestMessage, CancellationToken>((message, token) =>
            {
                Assert.AreEqual(HttpMethod.Post, message.Method);
                Assert.AreEqual("https://api.fitbit.com/1/user/-/foods/log/water.json?amount=300&date=2015-01-12", message.RequestUri.AbsoluteUri);
            });

            var fitbitClient = Helper.CreateFitbitClient(responseMessage, verification);

            var response = await fitbitClient.LogWaterAsync(new DateTime(2015, 1, 12), new WaterLog { Amount = 300 });

            Assert.AreEqual(300, response.Amount);
        }

        [Test] [Category("Portable")]
        public async void DeleteWaterLogAsync_Success()
        {
            var responseMessage = new Func<HttpResponseMessage>(() =>
            {
                return new HttpResponseMessage(HttpStatusCode.NoContent);
            });

            var verification = new Action<HttpRequestMessage, CancellationToken>((message, token) =>
            {
                Assert.AreEqual(HttpMethod.Delete, message.Method);
                Assert.AreEqual("https://api.fitbit.com/1/user/-/foods/log/water/1234.json", message.RequestUri.AbsoluteUri);
            });

            var fitbitClient = Helper.CreateFitbitClient(responseMessage, verification);

            await fitbitClient.DeleteWaterLogAsync(1234);

            //Assert.IsTrue(response.Success);
        }

        [Test] [Category("Portable")]
        public void Can_Deserialize_Water_Data_Json()
        {
            string content = SampleDataHelper.GetContent("GetWater-WaterData.json");

            var deserializer = new JsonDotNetSerializer();

            WaterData result = deserializer.Deserialize<WaterData>(content);

            ValidateWaterData(result);
        }

        [Test] [Category("Portable")]
        public void Can_Deserialize_Water_Log_Json()
        {
            string content = SampleDataHelper.GetContent("LogWater-WaterLog.json");

            var deserializer = new JsonDotNetSerializer { RootProperty = "waterLog"};

            WaterLog result = deserializer.Deserialize<WaterLog>(content);

            Assert.IsNotNull(result);
            Assert.AreEqual(508728882, result.LogId);
            Assert.AreEqual(300, result.Amount);
        }

        private void ValidateWaterData(WaterData result)
        {
            var firstWaterLog = result.Water.FirstOrDefault();

            Assert.IsNotNull(result);
            Assert.AreEqual(1300, result.Summary.Water);
            Assert.AreEqual(5, result.Water.Count);

            Assert.IsNotNull(firstWaterLog);
            Assert.AreEqual(200, firstWaterLog.Amount);
            Assert.AreEqual(508693835, firstWaterLog.LogId);
        }
    }
}


text:
/* See items.c */
uint64_t get_cas_id(void);

/*@null@*/
item *do_item_alloc(char *key, const size_t nkey, const int flags, const rel_time_t exptime, const int nbytes, const uint32_t cur_hv);
void item_free(item *it);
bool item_size_ok(const size_t nkey, const int flags, const int nbytes);

int  do_item_link(item *it, const uint32_t hv);     /** may fail if transgresses limits */
void do_item_unlink(item *it, const uint32_t hv);
void do_item_unlink_nolock(item *it, const uint32_t hv);
void do_item_remove(item *it);
void do_item_update(item *it);   /** update LRU time to current and reposition */
void do_item_update_nolock(item *it);
int  do_item_replace(item *it, item *new_it, const uint32_t hv);

/*@null@*/
char *do_item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes);
void do_item_stats(ADD_STAT add_stats, void *c);
void do_item_stats_totals(ADD_STAT add_stats, void *c);
/*@null@*/
void do_item_stats_sizes(ADD_STAT add_stats, void *c);
void do_item_flush_expired(void);

item *do_item_get(const char *key, const size_t nkey, const uint32_t hv);
item *do_item_touch(const char *key, const size_t nkey, uint32_t exptime, const uint32_t hv);
void item_stats_reset(void);
extern pthread_mutex_t cache_lock;
void item_stats_evictions(uint64_t *evicted);

enum crawler_result_type {
    CRAWLER_OK=0, CRAWLER_RUNNING, CRAWLER_BADCLASS
};

int start_item_crawler_thread(void);
int stop_item_crawler_thread(void);
int init_lru_crawler(void);
enum crawler_result_type lru_crawler_crawl(char *slabs);


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <title>Retrofit</title>
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <meta name="description" content="A type-safe HTTP client for Android and Java">
    <link href="static/bootstrap-combined.min.css" rel="stylesheet">
    <link href="static/app.css" rel="stylesheet">
    <link href="static/app-theme.css" rel="stylesheet">
    <link href="https://fonts.googleapis.com/css?family=Roboto:400,300italic,100,100italic,300" rel="stylesheet" type="text/css">

Retrofit

            <menu>

Download Latest

            </menu>

A type-safe HTTP client for Android and Java

Introduction

Retrofit turns your HTTP API into a Java interface.

public interface GitHubService {
  @GET("/users/{user}/repos")
  Call<List<Repo>> listRepos(@Path("user") String user);
}

The Retrofit class generates an implementation of the GitHubService interface.

Retrofit retrofit = new Retrofit.Builder()
    .baseUrl("https://api.github.com")
    .build();

GitHubService service = retrofit.create(GitHubService.class);

Each Call from the created GitHubService can make a synchronous or asynchronous HTTP request to the remote webserver.

Call<List<Repo>> repos = service.listRepos("octocat");

Use annotations to describe the HTTP request:

URL parameter replacement and query parameter support

Object conversion to request body (e.g., JSON, protocol buffers)

Multipart request body and file upload

Note: This site is still in the process of being expanded for the new 2.0 APIs.

API Declaration

Annotations on the interface methods and its parameters indicate how a request will be handled.

Request Method

Every method must have an HTTP annotation that provides the request method and relative URL. There are five built-in annotations: GET, POST, PUT, DELETE, and HEAD. The relative URL of the resource is specified in the annotation.

@GET("/users/list")

You can also specify query parameters in the URL.

@GET("/users/list?sort=desc")

URL Manipulation

A request URL can be updated dynamically using replacement blocks and parameters on the method. A replacement block is an alphanumeric string surrounded by { and }. A corresponding parameter must be annotated with @Path using the same string.

@GET("/group/{id}/users")
List<User> groupList(@Path("id") int groupId);

Query parameters can also be added.

@GET("/group/{id}/users")
List<User> groupList(@Path("id") int groupId, @Query("sort") String sort);

For complex query parameter combinations a Map can be used.

@GET("/group/{id}/users")
List<User> groupList(@Path("id") int groupId, @QueryMap Map<String, String> options);

Request Body

An object can be specified for use as an HTTP request body with the @Body annotation.

@POST("/users/new")
Call<User> createUser(@Body User user);

The object will also be converted using a converter specified on the Retrofit instance. If no converter is added, only RequestBody can be used.

Form Encoded and Multipart

Methods can also be declared to send form-encoded and multipart data.

Form-encoded data is sent when @FormUrlEncoded is present on the method. Each key-value pair is annotated with @Field containing the name and the object providing the value.

@FormUrlEncoded
@POST("/user/edit")
Call<User> updateUser(@Field("first_name") String first, @Field("last_name") String last);

Multipart requests are used when @Multipart is present on the method. Parts are declared using the @Part annotation.

@Multipart
@PUT("/user/photo")
Call<User> updateUser(@Part("photo") RequestBody photo, @Part("description") RequestBody description);

Multipart parts use one of Retrofit's converters or they can implement RequestBody to handle their own serialization.

Header Manipulation

You can set static headers for a method using the @Headers annotation.

@Headers("Cache-Control: max-age=640000")
@GET("/widget/list")
Call<List<Widget>> widgetList();

@Headers({
    "Accept: application/vnd.github.v3.full+json",
    "User-Agent: Retrofit-Sample-App"
})
@GET("/users/{username}")
Call<User> getUser(@Path("username") String username);

Note that headers do not overwrite each other. All headers with the same name will be included in the request.

A request Header can be updated dynamically using the @Header annotation. A corresponding parameter must be provided to the @Header. If the value is null, the header will be omitted. Otherwise, toString will be called on the value, and the result used.

@GET("/user")
Call<User> getUser(@Header("Authorization") String authorization)

Headers that need to be added to every request can be specified using an OkHttp interceptor.

Synchronous vs. Asynchronous

Call instances can be executed either synchronously or asynchronously. Each instance can only be used once, but calling clone() will create a new instance that can be used.

On Android, callbacks will be executed on the main thread. On the JVM, callbacks will happen on the same thread that executed the HTTP request.

Retrofit Configuration

Retrofit is the class through which your API interfaces are turned into callable objects. By default, Retrofit will give you sane defaults for your platform but it allows for customization.

Converters

By default, Retrofit can only deserialize HTTP bodies into OkHttp's ResponseBody type and it can only accept its RequestBody type for @Body.

Converters can be added to support other types. Six sibling modules adapt popular serialization libraries for your convenience.

Gson: com.squareup.retrofit:converter-gson

Jackson: com.squareup.retrofit:converter-jackson

Moshi: com.squareup.retrofit:converter-moshi

Protobuf: com.squareup.retrofit:converter-protobuf

Wire: com.squareup.retrofit:converter-wire

Simple XML: com.squareup.retrofit:converter-simplexml

Custom Converters

If you need to communicate with an API that uses a content-format that Retrofit does not support out of the box (e.g. YAML, txt, custom format) or you wish to use a different library to implement an existing format, you can easily create your own converter. Create a class that implements the Converter.Factory interface and pass in an instance when building your adapter.

Download

↓ Latest JAR

The source code to the Retrofit, its samples, and this website is available on GitHub.

Maven

<dependency>
  <groupId>com.squareup.retrofit</groupId>
  <artifactId>retrofit</artifactId>
  <version>(insert latest version)</version>
</dependency>

Gradle

compile 'com.squareup.retrofit:retrofit:(insert latest version)'

Retrofit requires at minimum Java 7 or Android 2.3.

ProGuard

If you are using Proguard in your project add the following lines to your configuration:

-dontwarn retrofit.**
-keep class retrofit.** { *; }
-keepattributes Signature
-keepattributes Exceptions

Contributing

If you would like to contribute code you can do so through GitHub by forking the repository and sending a pull request.

When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible. Please also make sure your code compiles by running mvn clean verify.

Before your code can be accepted into the project you must also sign the Individual Contributor License Agreement (CLA).

License

Copyright 2013 Square, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Introduction

API Declaration

Retrofit Configuration

Download

Contributing

License

Javadoc

StackOverflow


text:
(function () {
    'use strict';
}());
/*
 * @author WMXPY
 * @contect wm@wmpcxpy.com
 */
let control = {
    trans: false
}
jQuery(document).ready(function () {
    $.backstretch("../../imgs/indexbg.jpg");
    if (getURLVar('language') != null) main.languages = getURLVar('language');
    main.getexamlist();
    main.updatelanguage();
    document.getElementById('re-wave').addEventListener('click', () => {
        displaytips(rewave, main.languages);
    });
});
let rewave = [{
    icon: 'fa-language',
    id: 'languagec',
    tips: {
        en: '切换语言',
        cn: 'Switch Language'
    },
    fun: () => {
        if (main.languages == 'chinese') {
            main.languages = 'english';
        } else {
            main.languages = 'chinese'
        }
        main.updatelanguage();
    }
}];
var main = new Vue({
    el: '#main',
    data: {
        filelist: [],
        languages: 'chinese',
        display: {},
        test: 'test'
    },
    methods: {
        updatelanguage: function () {
            // console.log(languages);
            if (this.languages == 'chinese') {
                this.display = languages.ob.cn;
            } else {
                this.display = languages.ob.en;
            }
        },
        tesst: function () {
            console.log('test');
        },
        getexamlist: () => {
            featurefunctions.getlist(0, data => {
                data = JSON.parse(data);
                main.filelist = data;
            });
        },
        startquiz: target => {
            console.log(target);
        }
    }
})


text:
EnJin.EntityWorld = function(cellSize) {
  if(cellSize) {
    EnJin.typeCheck(cellSize, 'cellSize', 'Number');
  }
  this._entities = [];
  this._dynamicEntities = [];
  this._entitiesOnScreen = [];

  this._gravity = {
    x: 0,
    y: 10
  };

  this._cellSize = Math.abs(cellSize) || 64;

  this._spriteDrawCall = undefined;
  this._spriteGrid = new EnJin.SpriteGrid(this);

  this._collisionWorld = new Axis2D.World(this._cellSize);
};

EnJin.EntityWorld.prototype = {
  update: function() {
    this._collisionWorld.update();

    this._dynamicEntities.forEach(function(entity){
      var updateLoop = entity.updateLoop,
          animations = entity.animations,
          collider = entity.collider,
          physics = entity.physics,
          sprite = entity.sprite,
          cPos;

      if(sprite) {
        this._spriteGrid._placeInGrid(sprite);
      }

      if(collider) {
        cPos = collider.getPosition();
        entity._transform.setPosition(cPos.x, cPos.y);
      }

      // update animations before updateLoop so updateLoop script can overwrite
      // animations if needed
      for(var key in animations) {
        animations[key].update();
      }

      if(updateLoop) {
        updateLoop.script.call(updateLoop.vars, entity, entity.userData);
      }

      // update physics after updateLoop for getting accurate info on collider
      // in the updateLoop next round after physics were added (touches.. etc)
      if(physics) {
        physics.update();
      }
    }, this);
  },
  draw: function(interpolation, x, y, width, height) {
    var onScreenColliders = this._collisionWorld.queryRect(x, y, width, height),
        onScreenSprites = this._spriteGrid.queryRect(x, y, width, height),
        entitiesToDraw = [];

    // clear last on screen entities
    this._entitiesOnScreen.forEach(function(entity){
      entity._onScreen = false;
    }, this);
    this._entitiesOnScreen = [];

    // dynamic entities always have updateLoop and/or drawLoops
    // so static entities don't need to know if they are on screen or not
    this._dynamicEntities.forEach(function(entity){
      var drawLoop = entity.drawLoop,
          ePos = entity._transform.getPosition();

      if(ePos.x >= x && ePos.y >= y && ePos.x < x+width && ePos.y < y+height) {
        entity._onScreen = true;
        this._entitiesOnScreen.push(entity);
      }

      // drawLoop functions should decide if to draw or not by using isOnScreen
      // so we add all entities with drawLoops..
      if(drawLoop) {
        entitiesToDraw.push(entity);
      }
    }, this);

    onScreenColliders.forEach(function(collider){
      this._addToEntitiesOnScreen(collider.userData);
    }, this);

    onScreenSprites.forEach(function(sprite){
      var ent = sprite.getEntity();
      this._addToEntitiesOnScreen(ent);
      entitiesToDraw.push(ent);
    }, this);

    entitiesToDraw.sort(function(a, b){
      return a.zIndex - b.zIndex;
    });

    entitiesToDraw.forEach(function(entity) {
      var sprite = entity.sprite,
          drawLoop = entity.drawLoop;

      if(drawLoop && drawLoop.drawBeforeSprite) {
        drawLoop.script.call(drawLoop.vars, entity, entity.userData);
      }

      if(sprite && this._spriteDrawCall && entity.isOnScreen()) {
        this._spriteDrawCall(interpolation, sprite);
      }

      if(drawLoop && !drawLoop.drawBeforeSprite) {
        drawLoop.script.call(drawLoop.vars, entity, entity.userData);
      }
    }, this);
  },
  createEntity: function(name, x, y, z) {
    return new EnJin.Entity(this, name, x, y, z);
  },
  setCollisionCallback: function(callback) {
    this._collisionWorld.setCollisionCallback(function(colliders) {
      var entities = colliders.map(function(collider){
        return collider.userData;
      });
      callback(entities);
    });
  },
  setSpriteDrawCall: function(callback) {
    EnJin.typeCheck(callback, 'callback', 'Function');
    this._spriteDrawCall = callback;
  },
  setGravity: function(x, y) {
    EnJin.typeCheck(x, 'x', 'Number');
    EnJin.typeCheck(y, 'y', 'Number');
    this._gravity.x = x;
    this._gravity.y = y;
  },
  getOnScreenEntities: function() {
    return this._entitiesOnScreen;
  },
  setCellSize: function(cellSize) {
    EnJin.typeCheck(cellSize, 'cellSize', 'Number');
    this._cellSize = Math.abs(cellSize);
    this._collisionWorld.setCellSize(this._cellSize);
    this._entities.forEach(function(entity){
      if(entity.sprite) {
        this._spriteGrid._placeInGrid(entity.sprite);
      }
    }, this);
  },
  getCellSize: function() {
    return this._cellSize;
  },
  getCollisionWorld: function() {
    return this._collisionWorld;
  },
  countEntities: function() {
    return this._entities.length;
  },
  _addToEntitiesOnScreen: function(entity) {
    entity._onScreen = true;
    if(this._entitiesOnScreen.indexOf(entity) === -1) {
      this._entitiesOnScreen.push(entity);
    }
  }
};


text:
<?xml version="1.0" encoding="UTF-8"?>
<document id="DDI-DrugBank.d130">
    <sentence id="DDI-DrugBank.d130.s0" text="Lincomycin has been shown to have neuromuscular blocking properties that may enhance the action of other neuromuscular blocking agents.">
        <entity id="DDI-DrugBank.d130.s0.e0" charOffset="0-9"
            type="drug" text="Lincomycin"/>
        <entity id="DDI-DrugBank.d130.s0.e1" charOffset="105-133"
            type="group" text="neuromuscular blocking agents"/>
        <pair id="DDI-DrugBank.d130.s0.p0" e1="DDI-DrugBank.d130.s0.e0"
            e2="DDI-DrugBank.d130.s0.e1" ddi="true" type="effect"/>
    </sentence>
    <sentence id="DDI-DrugBank.d130.s1" text="Therefore, it should be used in caution in patients receiving such agents."/>
    <sentence id="DDI-DrugBank.d130.s2" text="Antagonism between lincomycin and erythromycin in vitro has been demonstrated.">
        <entity id="DDI-DrugBank.d130.s2.e0" charOffset="19-28"
            type="drug" text="lincomycin"/>
        <entity id="DDI-DrugBank.d130.s2.e1" charOffset="34-45"
            type="drug" text="erythromycin"/>
        <pair id="DDI-DrugBank.d130.s2.p0" e1="DDI-DrugBank.d130.s2.e0"
            e2="DDI-DrugBank.d130.s2.e1" ddi="true" type="effect"/>
    </sentence>
    <sentence id="DDI-DrugBank.d130.s3" text="Because of possible clinical significance, the two drugs should not be administered concurrently."/>
</document>


text:
use std::env;
use std::ffi::OsStr;
use std::fs::{self, File, OpenOptions};
use std::io;
cfg_if::cfg_if! {
    if #[cfg(not(target_os = "wasi"))] {
        use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
    } else {
        #[cfg(feature = "nightly")]
        use std::os::wasi::fs::MetadataExt;
    }
}
use crate::util;
use std::path::Path;

#[cfg(not(target_os = "redox"))]
use rustix::fs::{cwd, linkat, renameat, unlinkat, AtFlags};

pub fn create_named(path: &Path, open_options: &mut OpenOptions) -> io::Result<File> {
    open_options.read(true).write(true).create_new(true);

    #[cfg(not(target_os = "wasi"))]
    {
        open_options.mode(0o600);
    }

    open_options.open(path)
}

fn create_unlinked(path: &Path) -> io::Result<File> {
    let tmp;
    // shadow this to decrease the lifetime. It can't live longer than `tmp`.
    let mut path = path;
    if !path.is_absolute() {
        let cur_dir = env::current_dir()?;
        tmp = cur_dir.join(path);
        path = &tmp;
    }

    let f = create_named(path, &mut OpenOptions::new())?;
    // don't care whether the path has already been unlinked,
    // but perhaps there are some IO error conditions we should send up?
    let _ = fs::remove_file(path);
    Ok(f)
}

#[cfg(target_os = "linux")]
pub fn create(dir: &Path) -> io::Result<File> {
    use rustix::{fs::OFlags, io::Errno};
    OpenOptions::new()
        .read(true)
        .write(true)
        .custom_flags(OFlags::TMPFILE.bits() as i32) // do not mix with `create_new(true)`
        .open(dir)
        .or_else(|e| {
            match Errno::from_io_error(&e) {
                // These are the three "not supported" error codes for O_TMPFILE.
                Some(Errno::OPNOTSUPP) | Some(Errno::ISDIR) | Some(Errno::NOENT) => {
                    create_unix(dir)
                }
                _ => Err(e),
            }
        })
}

#[cfg(not(target_os = "linux"))]
pub fn create(dir: &Path) -> io::Result<File> {
    create_unix(dir)
}

fn create_unix(dir: &Path) -> io::Result<File> {
    util::create_helper(
        dir,
        OsStr::new(".tmp"),
        OsStr::new(""),
        crate::NUM_RAND_CHARS,
        |path| create_unlinked(&path),
    )
}

#[cfg(any(not(target_os = "wasi"), feature = "nightly"))]
pub fn reopen(file: &File, path: &Path) -> io::Result<File> {
    let new_file = OpenOptions::new().read(true).write(true).open(path)?;
    let old_meta = file.metadata()?;
    let new_meta = new_file.metadata()?;
    if old_meta.dev() != new_meta.dev() || old_meta.ino() != new_meta.ino() {
        return Err(io::Error::new(
            io::ErrorKind::NotFound,
            "original tempfile has been replaced",
        ));
    }
    Ok(new_file)
}

#[cfg(all(target_os = "wasi", not(feature = "nightly")))]
pub fn reopen(_file: &File, _path: &Path) -> io::Result<File> {
    return Err(io::Error::new(
        io::ErrorKind::Other,
        "this operation is supported on WASI only on nightly Rust (with `nightly` feature enabled)",
    ));
}

#[cfg(not(target_os = "redox"))]
pub fn persist(old_path: &Path, new_path: &Path, overwrite: bool) -> io::Result<()> {
    if overwrite {
        renameat(cwd(), old_path, cwd(), new_path)?;
    } else {
        // On Linux, use `renameat_with` to avoid overwriting an existing name,
        // if the kernel and the filesystem support it.
        #[cfg(any(target_os = "android", target_os = "linux"))]
        {
            use rustix::fs::{renameat_with, RenameFlags};
            use rustix::io::Errno;
            use std::sync::atomic::{AtomicBool, Ordering::Relaxed};

            static NOSYS: AtomicBool = AtomicBool::new(false);
            if !NOSYS.load(Relaxed) {
                match renameat_with(cwd(), old_path, cwd(), new_path, RenameFlags::NOREPLACE) {
                    Ok(()) => return Ok(()),
                    Err(Errno::NOSYS) => NOSYS.store(true, Relaxed),
                    Err(Errno::INVAL) => {}
                    Err(e) => return Err(e.into()),
                }
            }
        }

        // Otherwise use `linkat` to create the new filesystem name, which
        // will fail if the name already exists, and then `unlinkat` to remove
        // the old name.
        linkat(cwd(), old_path, cwd(), new_path, AtFlags::empty())?;
        // Ignore unlink errors. Can we do better?
        let _ = unlinkat(cwd(), old_path, AtFlags::empty());
    }
    Ok(())
}

#[cfg(target_os = "redox")]
pub fn persist(old_path: &Path, new_path: &Path, overwrite: bool) -> io::Result<()> {
    // XXX implement when possible
    Err(io::Error::from_raw_os_error(syscall::ENOSYS))
}

pub fn keep(_: &Path) -> io::Result<()> {
    Ok(())
}