text:
"""Module containing stages that generate and/or archive artifacts."""

from __future__ import print_function

import glob
import itertools
import json
import logging
import multiprocessing
import os
import shutil

from chromite.cbuildbot import commands
from chromite.cbuildbot import failures_lib
from chromite.cbuildbot import cbuildbot_config
from chromite.cbuildbot import constants
from chromite.cbuildbot import prebuilts
from chromite.cbuildbot.stages import generic_stages
from chromite.lib import cros_build_lib
from chromite.lib import git
from chromite.lib import osutils
from chromite.lib import parallel
from chromite.lib import portage_util
from chromite.scripts import cros_generate_breakpad_symbols

_FULL_BINHOST = 'FULL_BINHOST'
_PORTAGE_BINHOST = 'PORTAGE_BINHOST'

class NothingToArchiveException(Exception):
  """Thrown if ArchiveStage found nothing to archive."""
  def __init__(self, message='No images found to archive.'):
    super(NothingToArchiveException, self).__init__(message)

class ArchiveStage(generic_stages.BoardSpecificBuilderStage,
                   generic_stages.ArchivingStageMixin):
  """Archives build and test artifacts for developer consumption.

  Attributes:
    release_tag: The release tag. E.g. 2981.0.0
    version: The full version string, including the milestone.
        E.g. R26-2981.0.0-b123
  """

  option_name = 'archive'
  config_name = 'archive'

  # This stage is intended to run in the background, in parallel with tests.
  def __init__(self, builder_run, board, chrome_version=None, **kwargs):
    super(ArchiveStage, self).__init__(builder_run, board, **kwargs)
    self.chrome_version = chrome_version

    # TODO(mtennant): Places that use this release_tag attribute should
    # move to use self._run.attrs.release_tag directly.
    self.release_tag = getattr(self._run.attrs, 'release_tag', None)

    self._recovery_image_status_queue = multiprocessing.Queue()
    self._release_upload_queue = multiprocessing.Queue()
    self._upload_queue = multiprocessing.Queue()
    self.artifacts = []

  def WaitForRecoveryImage(self):
    """Wait until artifacts needed by SignerTest stage are created.

    Returns:
      True if artifacts created successfully.
      False otherwise.
    """
    cros_build_lib.Info('Waiting for recovery image...')
    status = self._recovery_image_status_queue.get()
    # Put the status back so other SignerTestStage instances don't starve.
    self._recovery_image_status_queue.put(status)
    return status

  @staticmethod
  def SingleMatchGlob(path_pattern):
    """Returns the last match (after sort) if multiple found."""
    files = glob.glob(path_pattern)
    files.sort()
    if not files:
      raise NothingToArchiveException('No %s found!' % path_pattern)
    elif len(files) > 1:
      cros_build_lib.PrintBuildbotStepWarnings()
      cros_build_lib.Warning('Expecting one result for %s package, but '
                             'found multiple.', path_pattern)
    return files[-1]

  def ArchiveStrippedChrome(self):
    """Generate and upload stripped Chrome package."""

    # If chrome is not installed, skip archiving.
    chroot_path = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
    board_path = os.path.join(chroot_path, 'build', self._current_board)
    if not portage_util.IsPackageInstalled(constants.CHROME_CP,
                                           board_path):
      return

    cmd = ['strip_package', '--board', self._current_board,
           constants.CHROME_PN]
    cros_build_lib.RunCommand(cmd, cwd=self._build_root, enter_chroot=True)
    pkg_dir = os.path.join(
        self._build_root, constants.DEFAULT_CHROOT_DIR, 'build',
        self._current_board, 'stripped-packages')
    chrome_tarball = self.SingleMatchGlob(
        os.path.join(pkg_dir, constants.CHROME_CP) + '-*')
    filename = os.path.basename(chrome_tarball)
    os.link(chrome_tarball, os.path.join(self.archive_path, filename))
    self._upload_queue.put([filename])

  def BuildAndArchiveDeltaSysroot(self):
    """Generate and upload delta sysroot for initial build_packages."""
    extra_env = {}
    if self._run.config.useflags:
      extra_env['USE'] = ' '.join(self._run.config.useflags)
    in_chroot_path = git.ReinterpretPathForChroot(self.archive_path)
    cmd = ['generate_delta_sysroot', '--out-dir', in_chroot_path,
           '--board', self._current_board]
    # TODO(mtennant): Make this condition into one run param.
    if not self._run.config.build_tests or not self._run.options.tests:
      cmd.append('--skip-tests')
    cros_build_lib.RunCommand(cmd, cwd=self._build_root, enter_chroot=True,
                              extra_env=extra_env)
    self._upload_queue.put([constants.DELTA_SYSROOT_TAR])

  def LoadArtifactsList(self, board, image_dir):
    """Load the list of artifacts to upload for this board.

    It attempts to load a JSON file, scripts/artifacts.json, from the
    overlay directories for this board. This file specifies the artifacts
    to generate, if it can't be found, it will use a default set that
    uploads every .bin file as a .tar.xz file except for
    chromiumos_qemu_image.bin.

    See BuildStandaloneArchive in cbuildbot_commands.py for format docs.
    """
    custom_artifacts_file = portage_util.ReadOverlayFile(
        'scripts/artifacts.json', board=board)
    artifacts = None

    if custom_artifacts_file is not None:
      json_file = json.loads(custom_artifacts_file)
      artifacts = json_file.get('artifacts')

    if artifacts is None:
      artifacts = []
      for image_file in glob.glob(os.path.join(image_dir, '*.bin')):
        basename = os.path.basename(image_file)
        if basename != constants.VM_IMAGE_BIN:
          info = {'input': [basename], 'archive': 'tar', 'compress': 'xz'}
          artifacts.append(info)

    for artifact in artifacts:
      # Resolve the (possible) globs in the input list, and store
      # the actual set of files to use in 'paths'
      paths = []
      for s in artifact['input']:
        glob_paths = glob.glob(os.path.join(image_dir, s))
        if not glob_paths:
          logging.warning('No artifacts generated for input: %s', s)
        else:
          for path in glob_paths:
            paths.append(os.path.relpath(path, image_dir))
      artifact['paths'] = paths
    self.artifacts = artifacts

  def IsArchivedFile(self, filename):
    """Return True if filename is the name of a file being archived."""
    for artifact in self.artifacts:
      for path in itertools.chain(artifact['paths'], artifact['input']):
        if os.path.basename(path) == filename:
          return True
    return False

  def PerformStage(self):
    buildroot = self._build_root
    config = self._run.config
    board = self._current_board
    debug = self._run.debug
    upload_url = self.upload_url
    archive_path = self.archive_path
    image_dir = self.GetImageDirSymlink()

    extra_env = {}
    if config['useflags']:
      extra_env['USE'] = ' '.join(config['useflags'])

    if not archive_path:
      raise NothingToArchiveException()

    # The following functions are run in parallel (except where indicated
    # otherwise)
    # \- BuildAndArchiveArtifacts
    #    \- ArchiveReleaseArtifacts
    #       \- ArchiveFirmwareImages
    #       \- BuildAndArchiveAllImages
    #          (builds recovery image first, then launches functions below)
    #          \- BuildAndArchiveFactoryImages
    #          \- ArchiveStandaloneArtifacts
    #             \- ArchiveStandaloneArtifact
    #          \- ArchiveZipFiles
    #          \- ArchiveHWQual
    #       \- PushImage (blocks on BuildAndArchiveAllImages)
    #    \- ArchiveManifest
    #    \- ArchiveStrippedChrome
    #    \- ArchiveImageScripts

    def ArchiveManifest():
      """Create manifest.xml snapshot of the built code."""
      output_manifest = os.path.join(archive_path, 'manifest.xml')
      cmd = ['repo', 'manifest', '-r', '-o', output_manifest]
      cros_build_lib.RunCommand(cmd, cwd=buildroot, capture_output=True)
      self._upload_queue.put(['manifest.xml'])

    def BuildAndArchiveFactoryImages():
      """Build and archive the factory zip file.

      The factory zip file consists of the factory toolkit and the factory
      install image. Both are built here.
      """
      # Build factory install image and create a symlink to it.
      factory_install_symlink = None
      if 'factory_install' in config['images']:
        alias = commands.BuildFactoryInstallImage(buildroot, board, extra_env)
        factory_install_symlink = self.GetImageDirSymlink(alias)
        if config['factory_install_netboot']:
          commands.MakeNetboot(buildroot, board, factory_install_symlink)

      # Build the factory toolkit.
      chroot_dir = os.path.join(buildroot, constants.DEFAULT_CHROOT_DIR)
      chroot_tmp_dir = os.path.join(chroot_dir, 'tmp')
      with osutils.TempDir(base_dir=chroot_tmp_dir, sudo_rm=True) as tempdir:
        # Build the factory toolkit.
        if config['factory_toolkit']:
          toolkit_dir = os.path.join(tempdir, 'factory_toolkit')
          os.makedirs(toolkit_dir)
          commands.MakeFactoryToolkit(
              buildroot, board, toolkit_dir, self._run.attrs.release_tag)

        # Build and upload factory zip if needed.
        if factory_install_symlink or config['factory_toolkit']:
          filename = commands.BuildFactoryZip(
              buildroot, board, archive_path, factory_install_symlink,
              toolkit_dir, self._run.attrs.release_tag)
          self._release_upload_queue.put([filename])

    def ArchiveStandaloneArtifact(artifact_info):
      """Build and upload a single archive."""
      if artifact_info['paths']:
        for path in commands.BuildStandaloneArchive(archive_path, image_dir,
                                                    artifact_info):
          self._release_upload_queue.put([path])

    def ArchiveStandaloneArtifacts():
      """Build and upload standalone archives for each image."""
      if config['upload_standalone_images']:
        parallel.RunTasksInProcessPool(ArchiveStandaloneArtifact,
                                       [[x] for x in self.artifacts])

    def ArchiveZipFiles():
      """Build and archive zip files.

      This includes:
        - image.zip (all images in one big zip file)
        - the au-generator.zip used for update payload generation.
      """
      # Zip up everything in the image directory.
      image_zip = commands.BuildImageZip(archive_path, image_dir)
      self._release_upload_queue.put([image_zip])

      # Archive au-generator.zip.
      filename = 'au-generator.zip'
      shutil.copy(os.path.join(image_dir, filename), archive_path)
      self._release_upload_queue.put([filename])

    def ArchiveHWQual():
      """Build and archive the HWQual images."""
      # TODO(petermayo): This logic needs to be exported from the BuildTargets
      # stage rather than copied/re-evaluated here.
      # TODO(mtennant): Make this autotest_built concept into a run param.
      autotest_built = (config['build_tests'] and self._run.options.tests and
                        config['upload_hw_test_artifacts'])

      if config['hwqual'] and autotest_built:
        # Build the full autotest tarball for hwqual image. We don't upload it,
        # as it's fairly large and only needed by the hwqual tarball.
        cros_build_lib.Info('Archiving full autotest tarball locally ...')
        tarball = commands.BuildFullAutotestTarball(self._build_root,
                                                    self._current_board,
                                                    image_dir)
        commands.ArchiveFile(tarball, archive_path)

        # Build hwqual image and upload to Google Storage.
        hwqual_name = 'chromeos-hwqual-%s-%s' % (board, self.version)
        filename = commands.ArchiveHWQual(buildroot, hwqual_name, archive_path,
                                          image_dir)
        self._release_upload_queue.put([filename])

    def ArchiveFirmwareImages():
      """Archive firmware images built from source if available."""
      archive = commands.BuildFirmwareArchive(buildroot, board, archive_path)
      if archive:
        self._release_upload_queue.put([archive])

    def BuildAndArchiveAllImages():
      # Generate the recovery image. To conserve loop devices, we try to only
      # run one instance of build_image at a time. TODO(davidjames): Move the
      # image generation out of the archive stage.
      self.LoadArtifactsList(self._current_board, image_dir)

      # For recovery image to be generated correctly, BuildRecoveryImage must
      # run before BuildAndArchiveFactoryImages.
      if self.IsArchivedFile(constants.BASE_IMAGE_BIN):
        commands.BuildRecoveryImage(buildroot, board, image_dir, extra_env)
        self._recovery_image_status_queue.put(True)
        # Re-generate the artifacts list so we include the newly created
        # recovery image.
        self.LoadArtifactsList(self._current_board, image_dir)

      if config['images']:
        parallel.RunParallelSteps([BuildAndArchiveFactoryImages,
                                   ArchiveHWQual,
                                   ArchiveStandaloneArtifacts,
                                   ArchiveZipFiles])

    def ArchiveImageScripts():
      """Archive tarball of generated image manipulation scripts."""
      target = os.path.join(archive_path, constants.IMAGE_SCRIPTS_TAR)
      files = glob.glob(os.path.join(image_dir, '*.sh'))
      files = [os.path.basename(f) for f in files]
      cros_build_lib.CreateTarball(target, image_dir, inputs=files)
      self._upload_queue.put([constants.IMAGE_SCRIPTS_TAR])

    def PushImage():
      # This helper script is only available on internal manifests currently.
      if not config['internal']:
        return

      self.GetParallel('debug_tarball_generated', pretty_name='debug tarball')

      # Now that all data has been generated, we can upload the final result to
      # the image server.
      # TODO: When we support branches fully, the friendly name of the branch
      # needs to be used with PushImages
      sign_types = []
      if config['name'].endswith('-%s' % cbuildbot_config.CONFIG_TYPE_FIRMWARE):
        sign_types += ['firmware']
      urls = commands.PushImages(
          board=board,
          archive_url=upload_url,
          dryrun=debug or not config['push_image'],
          profile=self._run.options.profile or config['profile'],
          sign_types=sign_types)
      self.board_runattrs.SetParallel('instruction_urls_per_channel', urls)

    def ArchiveReleaseArtifacts():
      with self.ArtifactUploader(self._release_upload_queue, archive=False):
        steps = [BuildAndArchiveAllImages, ArchiveFirmwareImages]
        parallel.RunParallelSteps(steps)
      PushImage()

    def BuildAndArchiveArtifacts():
      # Run archiving steps in parallel.
      steps = [ArchiveReleaseArtifacts, ArchiveManifest]
      if config['images']:
        steps.extend([self.ArchiveStrippedChrome, ArchiveImageScripts])
      if config['create_delta_sysroot']:
        steps.append(self.BuildAndArchiveDeltaSysroot)

      with self.ArtifactUploader(self._upload_queue, archive=False):
        parallel.RunParallelSteps(steps)

    if not self._run.config.afdo_generate_min:
      BuildAndArchiveArtifacts()

  def _HandleStageException(self, exc_info):
    # Tell the HWTestStage not to wait for artifacts to be uploaded
    # in case ArchiveStage throws an exception.
    self._recovery_image_status_queue.put(False)
    self.board_runattrs.SetParallel('instruction_urls_per_channel', None)
    return super(ArchiveStage, self)._HandleStageException(exc_info)

class CPEExportStage(generic_stages.BoardSpecificBuilderStage,
                     generic_stages.ArchivingStageMixin):
  """Handles generation & upload of package CPE information."""

  config_name = 'cpe_export'

  @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  def PerformStage(self):
    """Generate debug symbols and upload debug.tgz."""
    buildroot = self._build_root
    board = self._current_board
    useflags = self._run.config.useflags

    logging.info('Generating CPE export.')
    result = commands.GenerateCPEExport(buildroot, board, useflags)

    logging.info('Writing CPE export to files for archive.')
    warnings_filename = os.path.join(self.archive_path,
                                     'cpe-warnings-chromeos-%s.txt' % board)
    results_filename = os.path.join(self.archive_path,
                                    'cpe-chromeos-%s.json' % board)

    osutils.WriteFile(warnings_filename, result.error)
    osutils.WriteFile(results_filename, result.output)

    logging.info('Uploading CPE files.')
    self.UploadArtifact(os.path.basename(warnings_filename), archive=False)
    self.UploadArtifact(os.path.basename(results_filename), archive=False)

class DebugSymbolsStage(generic_stages.BoardSpecificBuilderStage,
                        generic_stages.ArchivingStageMixin):
  """Handles generation & upload of debug symbols."""

  config_name = 'debug_symbols'

  @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  def PerformStage(self):
    """Generate debug symbols and upload debug.tgz."""
    buildroot = self._build_root
    board = self._current_board

    commands.GenerateBreakpadSymbols(buildroot, board, self._run.debug)
    self.board_runattrs.SetParallel('breakpad_symbols_generated', True)

    steps = [self.UploadDebugTarball]
    failed_list = os.path.join(self.archive_path, 'failed_upload_symbols.list')
    if self._run.config.upload_symbols:
      steps.append(lambda: self.UploadSymbols(buildroot, board, failed_list))

    parallel.RunParallelSteps(steps)

  def UploadDebugTarball(self):
    """Generate and upload the debug tarball."""
    try:
      filename = commands.GenerateDebugTarball(
          self._build_root, self._current_board, self.archive_path,
          self._run.config.archive_build_debug)
    except cros_build_lib.RunCommandError:
      # TODO(yjhong): Remove this after crbug.com/339934 is fixed.
      debug_root = cros_build_lib.FromChrootPath(
          cros_generate_breakpad_symbols.FindDebugDir(self._current_board))
      logging.warning('Failed to generate the debug tarball. Listing the '
                      'files in %s:', debug_root)
      for subpath in [os.path.join('bin'),
                      os.path.join('usr', 'bin'),
                      os.path.join('usr', 'sbin')]:
        logging.warning(osutils.StatFilesInDirectory(
            os.path.join(debug_root, subpath),
            recursive=True, to_string=True))
      raise

    self.UploadArtifact(filename, archive=False)
    cros_build_lib.Info('Announcing availability of debug tarball now.')
    self.board_runattrs.SetParallel('debug_tarball_generated', True)

  def UploadSymbols(self, buildroot, board, failed_list):
    """Upload generated debug symbols."""
    if self._run.options.remote_trybot or self._run.debug:
      # For debug builds, limit ourselves to just uploading 1 symbol.
      # This way trybots and such still exercise this code.
      cnt = 1
      official = False
    else:
      cnt = None
      official = self._run.config.chromeos_official

    commands.UploadSymbols(buildroot, board, official, cnt, failed_list)

    if os.path.exists(failed_list):
      self.UploadArtifact(os.path.basename(failed_list), archive=False)

  def _SymbolsNotGenerated(self):
    """Tell other stages that our symbols were not generated."""
    self.board_runattrs.SetParallelDefault('breakpad_symbols_generated', False)
    self.board_runattrs.SetParallelDefault('debug_tarball_generated', False)

  def HandleSkip(self):
    """Tell other stages to not wait on us if we are skipped."""
    self._SymbolsNotGenerated()
    return super(DebugSymbolsStage, self).HandleSkip()

  def _HandleStageException(self, exc_info):
    """Tell other stages to not wait on us if we die for some reason."""
    self._SymbolsNotGenerated()
    return super(DebugSymbolsStage, self)._HandleStageException(exc_info)

class MasterUploadPrebuiltsStage(generic_stages.BuilderStage):
  """Syncs prebuilt binhost files across slaves."""
  # TODO(mtennant): This class represents logic spun out from
  # UploadPrebuiltsStage that is specific to a master builder. This is
  # currently used by the Commit Queue and the Master PFQ builder, but
  # could be used by other master builders that upload prebuilts,
  # e.g., x86-alex-pre-flight-branch. When completed the
  # UploadPrebuiltsStage code can be thinned significantly.
  option_name = 'prebuilts'
  config_name = 'prebuilts'

  def _GenerateCommonArgs(self):
    """Generate common prebuilt arguments."""
    generated_args = []
    if self._run.options.debug:
      generated_args.extend(['--debug', '--dry-run'])

    profile = self._run.options.profile or self._run.config['profile']
    if profile:
      generated_args.extend(['--profile', profile])

    # Generate the version if we are a manifest_version build.
    if self._run.config.manifest_version:
      version = self._run.GetVersion()
      generated_args.extend(['--set-version', version])

    return generated_args

  @staticmethod
  def _AddOptionsForSlave(slave_config):
    """Private helper method to add upload_prebuilts args for a slave builder.

    Args:
      slave_config: The build config of a slave builder.

    Returns:
      An array of options to add to upload_prebuilts array that allow a master
      to submit prebuilt conf modifications on behalf of a slave.
    """
    args = []
    if slave_config['prebuilts']:
      for slave_board in slave_config['boards']:
        args.extend(['--slave-board', slave_board])
        slave_profile = slave_config['profile']
        if slave_profile:
          args.extend(['--slave-profile', slave_profile])

    return args

  @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  def PerformStage(self):
    """Syncs prebuilt binhosts for slave builders."""
    # Common args we generate for all types of builds.
    generated_args = self._GenerateCommonArgs()
    # Args we specifically add for public/private build types.
    public_args, private_args = [], []
    # Gather public/private (slave) builders.
    public_builders, private_builders = [], []

    # Distributed builders that use manifest-versions to sync with one another
    # share prebuilt logic by passing around versions.
    assert cbuildbot_config.IsPFQType(self._prebuilt_type)

    # Public pfqs should upload host preflight prebuilts.
    public_args.append('--sync-host')

    # Update all the binhost conf files.
    generated_args.append('--sync-binhost-conf')
    for slave_config in self._GetSlaveConfigs():
      if slave_config['prebuilts'] == constants.PUBLIC:
        public_builders.append(slave_config['name'])
        public_args.extend(self._AddOptionsForSlave(slave_config))
      elif slave_config['prebuilts'] == constants.PRIVATE:
        private_builders.append(slave_config['name'])
        private_args.extend(self._AddOptionsForSlave(slave_config))

    # Upload the public prebuilts, if any.
    if public_builders:
      prebuilts.UploadPrebuilts(
          category=self._prebuilt_type, chrome_rev=self._chrome_rev,
          private_bucket=False, buildroot=self._build_root, board=None,
          extra_args=generated_args + public_args)

    # Upload the private prebuilts, if any.
    if private_builders:
      prebuilts.UploadPrebuilts(
          category=self._prebuilt_type, chrome_rev=self._chrome_rev,
          private_bucket=True, buildroot=self._build_root, board=None,
          extra_args=generated_args + private_args)

class UploadPrebuiltsStage(generic_stages.BoardSpecificBuilderStage):
  """Uploads binaries generated by this build for developer use."""

  option_name = 'prebuilts'
  config_name = 'prebuilts'

  def __init__(self, builder_run, board, **kwargs):
    super(UploadPrebuiltsStage, self).__init__(builder_run, board, **kwargs)

  def GenerateCommonArgs(self):
    """Generate common prebuilt arguments."""
    generated_args = []
    if self._run.options.debug:
      generated_args.extend(['--debug', '--dry-run'])

    profile = self._run.options.profile or self._run.config.profile
    if profile:
      generated_args.extend(['--profile', profile])

    # Generate the version if we are a manifest_version build.
    if self._run.config.manifest_version:
      version = self._run.GetVersion()
      generated_args.extend(['--set-version', version])

    if self._run.config.git_sync:
      # Git sync should never be set for pfq type builds.
      assert not cbuildbot_config.IsPFQType(self._prebuilt_type)
      generated_args.extend(['--git-sync'])

    return generated_args

  @classmethod
  def _AddOptionsForSlave(cls, slave_config, board):
    """Private helper method to add upload_prebuilts args for a slave builder.

    Args:
      slave_config: The build config of a slave builder.
      board: The name of the "master" board on the master builder.

    Returns:
      An array of options to add to upload_prebuilts array that allow a master
      to submit prebuilt conf modifications on behalf of a slave.
    """
    args = []
    if slave_config['prebuilts']:
      for slave_board in slave_config['boards']:
        if slave_config['master'] and slave_board == board:
          # Ignore self.
          continue

        args.extend(['--slave-board', slave_board])
        slave_profile = slave_config['profile']
        if slave_profile:
          args.extend(['--slave-profile', slave_profile])

    return args

  @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  def PerformStage(self):
    """Uploads prebuilts for master and slave builders."""
    prebuilt_type = self._prebuilt_type
    board = self._current_board
    binhosts = []

    # Whether we publish public or private prebuilts.
    public = self._run.config.prebuilts == constants.PUBLIC
    # Common args we generate for all types of builds.
    generated_args = self.GenerateCommonArgs()
    # Args we specifically add for public/private build types.
    public_args, private_args = [], []
    # Public / private builders.
    public_builders, private_builders = [], []

    # Distributed builders that use manifest-versions to sync with one another
    # share prebuilt logic by passing around versions.
    if cbuildbot_config.IsPFQType(prebuilt_type):
      # Public pfqs should upload host preflight prebuilts.
      if prebuilt_type != constants.CHROME_PFQ_TYPE:
        public_args.append('--sync-host')

      # Deduplicate against previous binhosts.
      binhosts.extend(self._GetPortageEnvVar(_PORTAGE_BINHOST, board).split())
      binhosts.extend(self._GetPortageEnvVar(_PORTAGE_BINHOST, None).split())
      for binhost in filter(None, binhosts):
        generated_args.extend(['--previous-binhost-url', binhost])

      if self._run.config.master and board == self._boards[-1]:
        # The master builder updates all the binhost conf files, and needs to do
        # so only once so as to ensure it doesn't try to update the same file
        # more than once. As multiple boards can be built on the same builder,
        # we arbitrarily decided to update the binhost conf files when we run
        # upload_prebuilts for the last board. The other boards are treated as
        # slave boards.
        generated_args.append('--sync-binhost-conf')
        for c in self._GetSlaveConfigs():
          if c['prebuilts'] == constants.PUBLIC:
            public_builders.append(c['name'])
            public_args.extend(self._AddOptionsForSlave(c, board))
          elif c['prebuilts'] == constants.PRIVATE:
            private_builders.append(c['name'])
            private_args.extend(self._AddOptionsForSlave(c, board))

    # Upload the public prebuilts, if any.
    if public_builders or public:
      public_board = board if public else None
      prebuilts.UploadPrebuilts(
          category=prebuilt_type, chrome_rev=self._chrome_rev,
          private_bucket=False, buildroot=self._build_root,
          board=public_board, extra_args=generated_args + public_args)

    # Upload the private prebuilts, if any.
    if private_builders or not public:
      private_board = board if not public else None
      prebuilts.UploadPrebuilts(
          category=prebuilt_type, chrome_rev=self._chrome_rev,
          private_bucket=True, buildroot=self._build_root, board=private_board,
          extra_args=generated_args + private_args)

class DevInstallerPrebuiltsStage(UploadPrebuiltsStage):
  """Stage that uploads DevInstaller prebuilts."""

  config_name = 'dev_installer_prebuilts'

  @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  def PerformStage(self):
    generated_args = generated_args = self.GenerateCommonArgs()
    prebuilts.UploadDevInstallerPrebuilts(
        binhost_bucket=self._run.config.binhost_bucket,
        binhost_key=self._run.config.binhost_key,
        binhost_base_url=self._run.config.binhost_base_url,
        buildroot=self._build_root,
        board=self._current_board,
        extra_args=generated_args)

class UploadTestArtifactsStage(generic_stages.BoardSpecificBuilderStage,
                               generic_stages.ArchivingStageMixin):
  """Upload needed hardware test artifacts."""

  def BuildAutotestTarballs(self):
    """Build the autotest tarballs."""
    with osutils.TempDir(prefix='cbuildbot-autotest') as tempdir:
      with self.ArtifactUploader(strict=True) as queue:
        cwd = os.path.abspath(
            os.path.join(self._build_root, 'chroot', 'build',
                         self._current_board, constants.AUTOTEST_BUILD_PATH,
                         '..'))

        control_files_tarball = commands.BuildAutotestControlFilesTarball(
            self._build_root, cwd, tempdir)
        queue.put([control_files_tarball])

        packages_tarball = commands.BuildAutotestPackagesTarball(
            self._build_root, cwd, tempdir)
        queue.put([packages_tarball])

        # Tar up the test suites.
        test_suites_tarball = commands.BuildAutotestTestSuitesTarball(
            self._build_root, cwd, tempdir)
        queue.put([test_suites_tarball])

  def _GeneratePayloads(self, image_name, **kwargs):
    """Generate and upload payloads for |image_name|.

    Args:
      image_name: The image to use.
      **kwargs: Keyword arguments to pass to commands.GeneratePayloads.
    """
    with osutils.TempDir(prefix='cbuildbot-payloads') as tempdir:
      with self.ArtifactUploader() as queue:
        image_path = os.path.join(self.GetImageDirSymlink(), image_name)
        commands.GeneratePayloads(self._build_root, image_path, tempdir,
                                  **kwargs)
        for payload in os.listdir(tempdir):
          queue.put([os.path.join(tempdir, payload)])

  def BuildUpdatePayloads(self):
    """Archives update payloads when they are ready."""
    got_images = self.GetParallel('images_generated', pretty_name='images')
    if not got_images:
      return

    if 'test' in self._run.config.images:
      image_name = 'chromiumos_test_image.bin'
    elif 'dev' in self._run.config.images:
      image_name = 'chromiumos_image.bin'
    else:
      image_name = 'chromiumos_base_image.bin'
    logging.info('Generating payloads to upload for %s', image_name)
    self._GeneratePayloads(image_name, full=True, stateful=True)
    self.board_runattrs.SetParallel('payloads_generated', True)
    self._GeneratePayloads(image_name, delta=True)
    self.board_runattrs.SetParallel('delta_payloads_generated', True)

  @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  def PerformStage(self):
    """Upload any needed HWTest artifacts."""
    steps = []
    if (self._run.ShouldBuildAutotest() and
        self._run.config.upload_hw_test_artifacts):
      steps.append(self.BuildAutotestTarballs)

    if self._run.config.upload_hw_test_artifacts:
      steps.append(self.BuildUpdatePayloads)

    parallel.RunParallelSteps(steps)

  def _HandleStageException(self, exc_info):
    # Tell the HWTestStage not to wait for payloads to be uploaded
    # in case UploadHWTestArtifacts throws an exception.
    self.board_runattrs.SetParallelDefault('payloads_generated', False)
    self.board_runattrs.SetParallelDefault('delta_payloads_generated', False)
    return super(UploadTestArtifactsStage, self)._HandleStageException(exc_info)

# TODO(mtennant): This class continues to exist only for subclasses that still
# need self.archive_stage.  Hopefully, we can get rid of that need, eventually.
class ArchivingStage(generic_stages.BoardSpecificBuilderStage,
                     generic_stages.ArchivingStageMixin):
  """Helper for stages that archive files.

  See ArchivingStageMixin for functionality.

  Attributes:
    archive_stage: The ArchiveStage instance for this board.
  """

  def __init__(self, builder_run, board, archive_stage, **kwargs):
    super(ArchivingStage, self).__init__(builder_run, board, **kwargs)
    self.archive_stage = archive_stage


text:
import { vtkObject } from "../../../interfaces";
import { Size } from "../../../types";
import vtkViewStream from './ViewStream';

/**
 *
 */
export interface IImageStreamInitialValues {
	viewStreams?: any[],
	serverAnimationFPS?: number,
}

// Return type of wslink/src/WebsocketConnection, getSession() method.
type WebsocketSession = any;

export interface vtkImageStream extends vtkObject {

	/**
	 *
	 */
	connect(session: WebsocketSession): void;

	/**
	 *
	 * @param {String} [viewId] The ID of the view.
	 * @param {Size} [size] The size of the view.
	 */
	createViewStream(viewId?: string, size?: Size): vtkViewStream;

	/**
	 *
	 */
	delete(): void;

	/**
	 *
	 */
	disconnect(): void;

	/**
	 *
	 */
	getProtocol(): any;

	/**
	 *
	 */
	getServerAnimationFPS(): number;

	/**
	 *
	 */
	registerViewStream(): void;

	/**
	 *
	 * @param serverAnimationFPS
	 */
	setServerAnimationFPS(serverAnimationFPS: number): boolean;

	/**
	 *
	 */
	unregisterViewStream(): void;
}

/**
 * Method used to decorate a given object (publicAPI+model) with vtkImageStream characteristics.
 *
 * @param publicAPI object on which methods will be bounds (public)
 * @param model object on which data structure will be bounds (protected)
 * @param {IImageStreamInitialValues} [initialValues] (default: {})
 */
export function extend(publicAPI: object, model: object, initialValues?: IImageStreamInitialValues): void;

/**
 * Method used to create a new instance of vtkImageStream
 * @param {IImageStreamInitialValues} [initialValues] for pre-setting some of its content
 */
export function newInstance(initialValues?: IImageStreamInitialValues): vtkImageStream;

/**
 * vtkImageStream.
 */
export declare const vtkImageStream: {
	newInstance: typeof newInstance;
	extend: typeof extend;
}
export default vtkImageStream;

redpajama_set_name: RedPajamaGithub


text:
import React from "react"
import { withPrefix } from "gatsby-link"
import { defaultOptions } from "./internals"

exports.onRenderBody = ({ setHeadComponents }, pluginOptions) => {
  let { output, createLinkInHead } = { ...defaultOptions, ...pluginOptions }

  if (!createLinkInHead) {
    return
  }

  if (output.charAt(0) !== `/`) {
    output = `/` + output
  }

  setHeadComponents([
    <link
      key={`gatsby-plugin-sitemap`}
      rel="sitemap"
      type="application/xml"
      href={withPrefix(output)}
    />,
  ])
}


text:
title: Victoria Auckland
gender: female
graduated: 2010
submitted: false

---


text:
PyPika - Python Query Builder
=============================

.. _intro_start:

|BuildStatus|  |CoverageStatus|  |Codacy|  |Docs|  |PyPi|  |License|

Abstract
--------

What is |Brand|?

|Brand| is a Python API for building SQL queries. The motivation behind |Brand| is to provide a simple interface for
building SQL queries without limiting the flexibility of handwritten SQL. Designed with data analysis in mind, |Brand|
leverages the builder design pattern to construct queries to avoid messy string formatting and concatenation. It is also
easily extended to take full advantage of specific features of SQL database vendors.

What are the design goals for |Brand|?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

|Brand| is a fast, expressive and flexible way to replace handwritten SQL (or even ORM for the courageous souls amongst you).
Validation of SQL correctness is not an explicit goal of |Brand|. With such a large number of
SQL database vendors providing a robust validation of input data is difficult. Instead you are encouraged to check inputs you provide to |Brand| or appropriately handle errors raised from
your SQL database - just as you would have if you were writing SQL yourself.

.. _intro_end:

Read the docs: http://pypika.readthedocs.io/en/latest/

Installation
------------

.. _installation_start:

|Brand| supports python ``3.6+``.  It may also work on pypy, cython, and jython, but is not being tested for these versions.

To install |Brand| run the following command:

.. code-block:: bash

    pip install pypika

.. _installation_end:

Tutorial
--------

.. _tutorial_start:

The main classes in pypika are ``pypika.Query``, ``pypika.Table``, and ``pypika.Field``.

.. code-block:: python

    from pypika import Query, Table, Field

Selecting Data
^^^^^^^^^^^^^^

The entry point for building queries is ``pypika.Query``.  In order to select columns from a table, the table must
first be added to the query.  For simple queries with only one table, tables and columns can be references using
strings.  For more sophisticated queries a ``pypika.Table`` must be used.

.. code-block:: python

    q = Query.from_('customers').select('id', 'fname', 'lname', 'phone')

To convert the query into raw SQL, it can be cast to a string.

.. code-block:: python

    str(q)

Alternatively, you can use the `Query.get_sql()` function:

.. code-block:: python

    q.get_sql()

Tables, Columns, Schemas, and Databases
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

In simple queries like the above example, columns in the "from" table can be referenced by passing string names into
the ``select`` query builder function. In more complex examples, the ``pypika.Table`` class should be used. Columns can be
referenced as attributes on instances of ``pypika.Table``.

.. code-block:: python

    from pypika import Table, Query

    customers = Table('customers')
    q = Query.from_(customers).select(customers.id, customers.fname, customers.lname, customers.phone)

Both of the above examples result in the following SQL:

.. code-block:: sql

    SELECT id,fname,lname,phone FROM customers

An alias for the table can be given using the ``.as_`` function on ``pypika.Table``

.. code-block:: sql

    customers = Table('x_view_customers').as_('customers')
    q = Query.from_(customers).select(customers.id, customers.phone)

.. code-block:: sql

    SELECT id,phone FROM x_view_customers customers

A schema can also be specified. Tables can be referenced as attributes on the schema.

.. code-block:: sql

    from pypika import Table, Query, Schema

    views = Schema('views')
    q = Query.from_(views.customers).select(customers.id, customers.phone)

.. code-block:: sql

    SELECT id,phone FROM views.customers

Also references to databases can be used. Schemas can be referenced as attributes on the database.

.. code-block:: sql

    from pypika import Table, Query, Database

    my_db = Database('my_db')
    q = Query.from_(my_db.analytics.customers).select(customers.id, customers.phone)

.. code-block:: sql

    SELECT id,phone FROM my_db.analytics.customers

Results can be ordered by using the following syntax:

.. code-block:: python

    from pypika import Order
    Query.from_('customers').select('id', 'fname', 'lname', 'phone').orderby('id', order=Order.desc)

This results in the following SQL:

.. code-block:: sql

    SELECT "id","fname","lname","phone" FROM "customers" ORDER BY "id" DESC

Arithmetic
""""""""""

Arithmetic expressions can also be constructed using pypika.  Operators such as `+`, `-`, `*`, and `/` are implemented
by ``pypika.Field`` which can be used simply with a ``pypika.Table`` or directly.

.. code-block:: python

    from pypika import Field

    q = Query.from_('account').select(
        Field('revenue') - Field('cost')
    )

.. code-block:: sql

    SELECT revenue-cost FROM accounts

Using ``pypika.Table``

.. code-block:: python

    accounts = Table('accounts')
    q = Query.from_(accounts).select(
        accounts.revenue - accounts.cost
    )

.. code-block:: sql

    SELECT revenue-cost FROM accounts

An alias can also be used for fields and expressions.

.. code-block:: sql

    q = Query.from_(accounts).select(
        (accounts.revenue - accounts.cost).as_('profit')
    )

.. code-block:: sql

    SELECT revenue-cost profit FROM accounts

More arithmetic examples

.. code-block:: python

    table = Table('table')
    q = Query.from_(table).select(
        table.foo + table.bar,
        table.foo - table.bar,
        table.foo * table.bar,
        table.foo / table.bar,
        (table.foo+table.bar) / table.fiz,
    )

.. code-block:: sql

    SELECT foo+bar,foo-bar,foo*bar,foo/bar,(foo+bar)/fiz FROM table

Filtering
"""""""""

Queries can be filtered with ``pypika.Criterion`` by using equality or inequality operators

.. code-block:: python

    customers = Table('customers')
    q = Query.from_(customers).select(
        customers.id, customers.fname, customers.lname, customers.phone
    ).where(
        customers.lname == 'Mustermann'
    )

.. code-block:: sql

    SELECT id,fname,lname,phone FROM customers WHERE lname='Mustermann'

Query methods such as select, where, groupby, and orderby can be called multiple times.  Multiple calls to the where
method will add additional conditions as

.. code-block:: python

    customers = Table('customers')
    q = Query.from_(customers).select(
        customers.id, customers.fname, customers.lname, customers.phone
    ).where(
        customers.fname == 'Max'
    ).where(
        customers.lname == 'Mustermann'
    )

.. code-block:: sql

    SELECT id,fname,lname,phone FROM customers WHERE fname='Max' AND lname='Mustermann'

Filters such as IN and BETWEEN are also supported

.. code-block:: python

    customers = Table('customers')
    q = Query.from_(customers).select(
        customers.id,customers.fname
    ).where(
        customers.age[18:65] & customers.status.isin(['new', 'active'])
    )

.. code-block:: sql

    SELECT id,fname FROM customers WHERE age BETWEEN 18 AND 65 AND status IN ('new','active')

Filtering with complex criteria can be created using boolean symbols ``&``, ``|``, and ``^``.

AND

.. code-block:: python

    customers = Table('customers')
    q = Query.from_(customers).select(
        customers.id, customers.fname, customers.lname, customers.phone
    ).where(
        (customers.age >= 18) & (customers.lname == 'Mustermann')
    )

.. code-block:: sql

    SELECT id,fname,lname,phone FROM customers WHERE age>=18 AND lname='Mustermann'

OR

.. code-block:: python

    customers = Table('customers')
    q = Query.from_(customers).select(
        customers.id, customers.fname, customers.lname, customers.phone
    ).where(
        (customers.age >= 18) | (customers.lname == 'Mustermann')
    )

.. code-block:: sql

    SELECT id,fname,lname,phone FROM customers WHERE age>=18 OR lname='Mustermann'

XOR

.. code-block:: python

 customers = Table('customers')
 q = Query.from_(customers).select(
     customers.id, customers.fname, customers.lname, customers.phone
 ).where(
     (customers.age >= 18) ^ customers.is_registered
 )

.. code-block:: sql

    SELECT id,fname,lname,phone FROM customers WHERE age>=18 XOR is_registered

Convenience Methods
"""""""""""""""""""

In the `Criterion` class, there are the static methods `any` and `all` that allow building chains AND and OR expressions with a list of terms.

.. code-block:: python

    from pypika import Criterion

    customers = Table('customers')
    q = Query.from_(customers).select(
        customers.id,
        customers.fname
    ).where(
        Criterion.all([
            customers.is_registered,
            customers.age >= 18,
            customers.lname == "Jones",
        ])
    )

.. code-block:: sql

    SELECT id,fname FROM customers WHERE is_registered AND age>=18 AND lname = "Jones"

Grouping and Aggregating
""""""""""""""""""""""""

Grouping allows for aggregated results and works similar to ``SELECT`` clauses.

.. code-block:: python

    from pypika import functions as fn

    customers = Table('customers')
    q = Query \
        .from_(customers) \
        .where(customers.age >= 18) \
        .groupby(customers.id) \
        .select(customers.id, fn.Sum(customers.revenue))

.. code-block:: sql

    SELECT id,SUM("revenue") FROM "customers" WHERE "age">=18 GROUP BY "id"

After adding a ``GROUP BY`` clause to a query, the ``HAVING`` clause becomes available.  The method
``Query.having()`` takes a ``Criterion`` parameter similar to the method ``Query.where()``.

.. code-block:: python

    from pypika import functions as fn

    payments = Table('payments')
    q = Query \
        .from_(payments) \
        .where(payments.transacted[date(2015, 1, 1):date(2016, 1, 1)]) \
        .groupby(payments.customer_id) \
        .having(fn.Sum(payments.total) >= 1000) \
        .select(payments.customer_id, fn.Sum(payments.total))

.. code-block:: sql

    SELECT customer_id,SUM(total) FROM payments
    WHERE transacted BETWEEN '2015-01-01' AND '2016-01-01'
    GROUP BY customer_id HAVING SUM(total)>=1000

Joining Tables and Subqueries
"""""""""""""""""""""""""""""

Tables and subqueries can be joined to any query using the ``Query.join()`` method.  Joins can be performed with either
a ``USING`` or ``ON`` clauses.  The ``USING`` clause can be used when both tables/subqueries contain the same field and
the ``ON`` clause can be used with a criterion. To perform a join, ``...join()`` can be chained but then must be
followed immediately by ``...on(<criterion>)`` or ``...using(*field)``.

Join Types
~~~~~~~~~~

All join types are supported by |Brand|.

.. code-block:: python

    Query \
        .from_(base_table)
        ...
        .join(join_table, JoinType.left)
        ...

.. code-block:: python

    Query \
        .from_(base_table)
        ...
        .left_join(join_table) \
        .left_outer_join(join_table) \
        .right_join(join_table) \
        .right_outer_join(join_table) \
        .inner_join(join_table) \
        .outer_join(join_table) \
        .full_outer_join(join_table) \
        .cross_join(join_table) \
        .hash_join(join_table) \
        ...

See the list of join types here ``pypika.enums.JoinTypes``

Example of a join using `ON`
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. code-block:: python

    history, customers = Tables('history', 'customers')
    q = Query \
        .from_(history) \
        .join(customers) \
        .on(history.customer_id == customers.id) \
        .select(history.star) \
        .where(customers.id == 5)

.. code-block:: sql

    SELECT "history".* FROM "history" JOIN "customers" ON "history"."customer_id"="customers"."id" WHERE "customers"."id"=5

As a shortcut, the ``Query.join().on_field()`` function is provided for joining the (first) table in the ``FROM`` clause
with the joined table when the field name(s) are the same in both tables.

Example of a join using `ON`
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. code-block:: python

    history, customers = Tables('history', 'customers')
    q = Query \
        .from_(history) \
        .join(customers) \
        .on_field('customer_id', 'group') \
        .select(history.star) \
        .where(customers.group == 'A')

.. code-block:: sql

    SELECT "history".* FROM "history" JOIN "customers" ON "history"."customer_id"="customers"."customer_id" AND "history"."group"="customers"."group" WHERE "customers"."group"='A'

Example of a join using `USING`
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. code-block:: python

    history, customers = Tables('history', 'customers')
    q = Query \
        .from_(history) \
        .join(customers) \
        .using('customer_id') \
        .select(history.star) \
        .where(customers.id == 5)

.. code-block:: sql

    SELECT "history".* FROM "history" JOIN "customers" USING "customer_id" WHERE "customers"."id"=5

Example of a correlated subquery in the `SELECT`
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. code-block:: python

    history, customers = Tables('history', 'customers')
    last_purchase_at = Query.from_(history).select(
        history.purchase_at
    ).where(history.customer_id==customers.customer_id).orderby(
        history.purchase_at, order=Order.desc
    ).limit(1)
    q = Query.from_(customers).select(
        customers.id, last_purchase_at.as_('last_purchase_at')
    )

.. code-block:: sql

    SELECT
      "id",
      (SELECT "history"."purchase_at"
       FROM "history"
       WHERE "history"."customer_id" = "customers"."customer_id"
       ORDER BY "history"."purchase_at" DESC
       LIMIT 1) "last_purchase_at"
    FROM "customers"

Unions
""""""

Both ``UNION`` and ``UNION ALL`` are supported. ``UNION DISTINCT`` is synonomous with "UNION`` so |Brand| does not
provide a separate function for it.  Unions require that queries have the same number of ``SELECT`` clauses so
trying to cast a unioned query to string will throw a ``SetOperationException`` if the column sizes are mismatched.

To create a union query, use either the ``Query.union()`` method or `+` operator with two query instances. For a
union all, use ``Query.union_all()`` or the `*` operator.

.. code-block:: python

    provider_a, provider_b = Tables('provider_a', 'provider_b')
    q = Query.from_(provider_a).select(
        provider_a.created_time, provider_a.foo, provider_a.bar
    ) + Query.from_(provider_b).select(
        provider_b.created_time, provider_b.fiz, provider_b.buz
    )

.. code-block:: sql

    SELECT "created_time","foo","bar" FROM "provider_a" UNION SELECT "created_time","fiz","buz" FROM "provider_b"

Intersect
"""""""""

``INTERSECT`` is supported. Intersects require that queries have the same number of ``SELECT`` clauses so
trying to cast a intersected query to string will throw a ``SetOperationException`` if the column sizes are mismatched.

To create a intersect query, use the ``Query.intersect()`` method.

.. code-block:: python

    provider_a, provider_b = Tables('provider_a', 'provider_b')
    q = Query.from_(provider_a).select(
        provider_a.created_time, provider_a.foo, provider_a.bar
    )
    r = Query.from_(provider_b).select(
        provider_b.created_time, provider_b.fiz, provider_b.buz
    )
    intersected_query = q.intersect(r)

.. code-block:: sql

    SELECT "created_time","foo","bar" FROM "provider_a" INTERSECT SELECT "created_time","fiz","buz" FROM "provider_b"

Minus
"""""

``MINUS`` is supported. Minus require that queries have the same number of ``SELECT`` clauses so
trying to cast a minus query to string will throw a ``SetOperationException`` if the column sizes are mismatched.

To create a minus query, use either the ``Query.minus()`` method or `-` operator with two query instances.

.. code-block:: python

    provider_a, provider_b = Tables('provider_a', 'provider_b')
    q = Query.from_(provider_a).select(
        provider_a.created_time, provider_a.foo, provider_a.bar
    )
    r = Query.from_(provider_b).select(
        provider_b.created_time, provider_b.fiz, provider_b.buz
    )
    minus_query = q.minus(r)

    (or)

    minus_query = Query.from_(provider_a).select(
        provider_a.created_time, provider_a.foo, provider_a.bar
    ) - Query.from_(provider_b).select(
        provider_b.created_time, provider_b.fiz, provider_b.buz
    )

.. code-block:: sql

    SELECT "created_time","foo","bar" FROM "provider_a" MINUS SELECT "created_time","fiz","buz" FROM "provider_b"

EXCEPT
""""""

``EXCEPT`` is supported. Minus require that queries have the same number of ``SELECT`` clauses so
trying to cast a except query to string will throw a ``SetOperationException`` if the column sizes are mismatched.

To create a except query, use the ``Query.except_of()`` method.

.. code-block:: python

    provider_a, provider_b = Tables('provider_a', 'provider_b')
    q = Query.from_(provider_a).select(
        provider_a.created_time, provider_a.foo, provider_a.bar
    )
    r = Query.from_(provider_b).select(
        provider_b.created_time, provider_b.fiz, provider_b.buz
    )
    minus_query = q.except_of(r)

.. code-block:: sql

    SELECT "created_time","foo","bar" FROM "provider_a" EXCEPT SELECT "created_time","fiz","buz" FROM "provider_b"

Date, Time, and Intervals
"""""""""""""""""""""""""

Using ``pypika.Interval``, queries can be constructed with date arithmetic.  Any combination of intervals can be
used except for weeks and quarters, which must be used separately and will ignore any other values if selected.

.. code-block:: python

    from pypika import functions as fn

    fruits = Tables('fruits')
    q = Query.from_(fruits) \
        .select(fruits.id, fruits.name) \
        .where(fruits.harvest_date + Interval(months=1) < fn.Now())

.. code-block:: sql

    SELECT id,name FROM fruits WHERE harvest_date+INTERVAL 1 MONTH<NOW()

Tuples
""""""

Tuples are supported through the class ``pypika.Tuple`` but also through the native python tuple wherever possible.
Tuples can be used with ``pypika.Criterion`` in **WHERE** clauses for pairwise comparisons.

.. code-block:: python

    from pypika import Query, Tuple

    q = Query.from_(self.table_abc) \
        .select(self.table_abc.foo, self.table_abc.bar) \
        .where(Tuple(self.table_abc.foo, self.table_abc.bar) == Tuple(1, 2))

.. code-block:: sql

    SELECT "foo","bar" FROM "abc" WHERE ("foo","bar")=(1,2)

Using ``pypika.Tuple`` on both sides of the comparison is redundant and |Brand| supports native python tuples.

.. code-block:: python

    from pypika import Query, Tuple

    q = Query.from_(self.table_abc) \
        .select(self.table_abc.foo, self.table_abc.bar) \
        .where(Tuple(self.table_abc.foo, self.table_abc.bar) == (1, 2))

.. code-block:: sql

    SELECT "foo","bar" FROM "abc" WHERE ("foo","bar")=(1,2)

Tuples can be used in **IN** clauses.

.. code-block:: python

    Query.from_(self.table_abc) \
            .select(self.table_abc.foo, self.table_abc.bar) \
            .where(Tuple(self.table_abc.foo, self.table_abc.bar).isin([(1, 1), (2, 2), (3, 3)]))

.. code-block:: sql

    SELECT "foo","bar" FROM "abc" WHERE ("foo","bar") IN ((1,1),(2,2),(3,3))

Strings Functions
"""""""""""""""""

There are several string operations and function wrappers included in |Brand|.  Function wrappers can be found in the
``pypika.functions`` package.  In addition, `LIKE` and `REGEX` queries are supported as well.

.. code-block:: python

    from pypika import functions as fn

    customers = Tables('customers')
    q = Query.from_(customers).select(
        customers.id,
        customers.fname,
        customers.lname,
    ).where(
        customers.lname.like('Mc%')
    )

.. code-block:: sql

    SELECT id,fname,lname FROM customers WHERE lname LIKE 'Mc%'

.. code-block:: python

    from pypika import functions as fn

    customers = Tables('customers')
    q = Query.from_(customers).select(
        customers.id,
        customers.fname,
        customers.lname,
    ).where(
        customers.lname.regex(r'^[abc][a-zA-Z]+&')
    )

.. code-block:: sql

    SELECT id,fname,lname FROM customers WHERE lname REGEX '^[abc][a-zA-Z]+&';

.. code-block:: python

    from pypika import functions as fn

    customers = Tables('customers')
    q = Query.from_(customers).select(
        customers.id,
        fn.Concat(customers.fname, ' ', customers.lname).as_('full_name'),
    )

.. code-block:: sql

    SELECT id,CONCAT(fname, ' ', lname) full_name FROM customers

Custom Functions
""""""""""""""""

Custom Functions allows us to use any function on queries, as some functions are not covered by PyPika as default, we can appeal
to Custom functions.

.. code-block:: python

    from pypika import CustomFunction

    customers = Tables('customers')
    DateDiff = CustomFunction('DATE_DIFF', ['interval', 'start_date', 'end_date'])

    q = Query.from_(customers).select(
        customers.id,
        customers.fname,
        customers.lname,
        DateDiff('day', customers.created_date, customers.updated_date)
    )

.. code-block:: sql

    SELECT id,fname,lname,DATE_DIFF('day',created_date,updated_date) FROM customers

Case Statements
"""""""""""""""

Case statements allow fow a number of conditions to be checked sequentially and return a value for the first condition
met or otherwise a default value.  The Case object can be used to chain conditions together along with their output
using the ``when`` method and to set the default value using ``else_``.

.. code-block:: python

    from pypika import Case, functions as fn

    customers = Tables('customers')
    q = Query.from_(customers).select(
        customers.id,
        Case()
           .when(customers.fname == "Tom", "It was Tom")
           .when(customers.fname == "John", "It was John")
           .else_("It was someone else.").as_('who_was_it')
    )

.. code-block:: sql

    SELECT "id",CASE WHEN "fname"='Tom' THEN 'It was Tom' WHEN "fname"='John' THEN 'It was John' ELSE 'It was someone else.' END "who_was_it" FROM "customers"

With Clause
"""""""""""""""

With clause allows give a sub-query block a name, which can be referenced in several places within the main SQL query.
The SQL WITH clause is basically a drop-in replacement to the normal sub-query.

.. code-block:: python

    from pypika import Table, AliasedQuery, Query

    customers = Table('customers')

    sub_query = (Query
                .from_(customers)
                .select('*'))

    test_query = (Query
                .with_(sub_query, "an_alias")
                .from_(AliasedQuery("an_alias"))
                .select('*'))

You can use as much as `.with_()` as you want.

.. code-block:: sql

    WITH an_alias AS (SELECT * FROM "customers") SELECT * FROM an_alias

Inserting Data
^^^^^^^^^^^^^^

Data can be inserted into tables either by providing the values in the query or by selecting them through another query.

By default, data can be inserted by providing values for all columns in the order that they are defined in the table.

Insert with values
""""""""""""""""""

.. code-block:: python

    customers = Table('customers')

    q = Query.into(customers).insert(1, 'Jane', 'Doe', 'jane@example.com')

.. code-block:: sql

    INSERT INTO customers VALUES (1,'Jane','Doe','jane@example.com')

.. code-block:: python

    customers =  Table('customers')

    q = customers.insert(1, 'Jane', 'Doe', 'jane@example.com')

.. code-block:: sql

    INSERT INTO customers VALUES (1,'Jane','Doe','jane@example.com')

Multiple rows of data can be inserted either by chaining the ``insert`` function or passing multiple tuples as args.

.. code-block:: python

    customers = Table('customers')

    q = Query.into(customers).insert(1, 'Jane', 'Doe', 'jane@example.com').insert(2, 'John', 'Doe', 'john@example.com')

.. code-block:: python

    customers = Table('customers')

    q = Query.into(customers).insert((1, 'Jane', 'Doe', 'jane@example.com'),
                                     (2, 'John', 'Doe', 'john@example.com'))

Insert with constraint violation handling
"""""""""""""""""""""""""""""""""""""""""

MySQL
~~~~~

.. code-block:: python

    customers = Table('customers')

    q = MySQLQuery.into(customers) \
        .insert(1, 'Jane', 'Doe', 'jane@example.com') \
        .on_duplicate_key_ignore())

.. code-block:: sql

    INSERT INTO `customers` VALUES (1,'Jane','Doe','jane@example.com') ON DUPLICATE KEY IGNORE

.. code-block:: python

    customers = Table('customers')

    q = MySQLQuery.into(customers) \
        .insert(1, 'Jane', 'Doe', 'jane@example.com') \
        .on_duplicate_key_update(customers.email, Values(customers.email))

.. code-block:: sql

    INSERT INTO `customers` VALUES (1,'Jane','Doe','jane@example.com') ON DUPLICATE KEY UPDATE `email`=VALUES(`email`)

``.on_duplicate_key_update`` works similar to ``.set`` for updating rows, additionally it provides the ``Values``
wrapper to update to the value specified in the ``INSERT`` clause.

PostgreSQL
~~~~~~~~~~

.. code-block:: python

    customers = Table('customers')

    q = PostgreSQLQuery.into(customers) \
        .insert(1, 'Jane', 'Doe', 'jane@example.com') \
        .on_conflict(customers.email) \
        .do_nothing()

.. code-block:: sql

    INSERT INTO "customers" VALUES (1,'Jane','Doe','jane@example.com') ON CONFLICT ("email") DO NOTHING

.. code-block:: python

    customers = Table('customers')

    q = PostgreSQLQuery.into(customers) \
        .insert(1, 'Jane', 'Doe', 'jane@example.com') \
        .on_conflict(customers.email) \
        .do_update(customers.email, 'bob@example.com')

.. code-block:: sql

    INSERT INTO "customers" VALUES (1,'Jane','Doe','jane@example.com') ON CONFLICT ("email") DO UPDATE SET "email"='bob@example.com'

Insert from a SELECT Sub-query
""""""""""""""""""""""""""""""

.. code-block:: sql

    INSERT INTO "customers" VALUES (1,'Jane','Doe','jane@example.com'),(2,'John','Doe','john@example.com')

To specify the columns and the order, use the ``columns`` function.

.. code-block:: python

    customers = Table('customers')

    q = Query.into(customers).columns('id', 'fname', 'lname').insert(1, 'Jane', 'Doe')

.. code-block:: sql

    INSERT INTO customers (id,fname,lname) VALUES (1,'Jane','Doe','jane@example.com')

Inserting data with a query works the same as querying data with the additional call to the ``into`` method in the
builder chain.

.. code-block:: python

    customers, customers_backup = Tables('customers', 'customers_backup')

    q = Query.into(customers_backup).from_(customers).select('*')

.. code-block:: sql

    INSERT INTO customers_backup SELECT * FROM customers

.. code-block:: python

    customers, customers_backup = Tables('customers', 'customers_backup')

    q = Query.into(customers_backup).columns('id', 'fname', 'lname')
        .from_(customers).select(customers.id, customers.fname, customers.lname)

.. code-block:: sql

    INSERT INTO customers_backup SELECT "id", "fname", "lname" FROM customers

The syntax for joining tables is the same as when selecting data

.. code-block:: python

    customers, orders, orders_backup = Tables('customers', 'orders', 'orders_backup')

    q = Query.into(orders_backup).columns('id', 'address', 'customer_fname', 'customer_lname')
        .from_(customers)
        .join(orders).on(orders.customer_id == customers.id)
        .select(orders.id, customers.fname, customers.lname)

.. code-block:: sql

   INSERT INTO "orders_backup" ("id","address","customer_fname","customer_lname")
   SELECT "orders"."id","customers"."fname","customers"."lname" FROM "customers"
   JOIN "orders" ON "orders"."customer_id"="customers"."id"

Updating Data
^^^^^^^^^^^^^^
PyPika allows update queries to be constructed with or without where clauses.

.. code-block:: python

    customers = Table('customers')

    Query.update(customers).set(customers.last_login, '2017-01-01 10:00:00')

    Query.update(customers).set(customers.lname, 'smith').where(customers.id == 10)

.. code-block:: sql

    UPDATE "customers" SET "last_login"='2017-01-01 10:00:00'

    UPDATE "customers" SET "lname"='smith' WHERE "id"=10

The syntax for joining tables is the same as when selecting data

.. code-block:: python

    customers, profiles = Tables('customers', 'profiles')

    Query.update(customers)
         .join(profiles).on(profiles.customer_id == customers.id)
         .set(customers.lname, profiles.lname)

.. code-block:: sql

   UPDATE "customers"
   JOIN "profiles" ON "profiles"."customer_id"="customers"."id"
   SET "customers"."lname"="profiles"."lname"

Using ``pypika.Table`` alias to perform the update

.. code-block:: python

    customers = Table('customers')

    customers.update()
            .set(customers.lname, 'smith')
            .where(customers.id == 10)

.. code-block:: sql

    UPDATE "customers" SET "lname"='smith' WHERE "id"=10

Using ``limit`` for performing update

.. code-block:: python

    customers = Table('customers')

    customers.update()
            .set(customers.lname, 'smith')
            .limit(2)

.. code-block:: sql

    UPDATE "customers" SET "lname"='smith' LIMIT 2

Parametrized Queries
^^^^^^^^^^^^^^^^^^^^

PyPika allows you to use ``Parameter(str)`` term as a placeholder for parametrized queries.

.. code-block:: python

    customers = Table('customers')

    q = Query.into(customers).columns('id', 'fname', 'lname')
        .insert(Parameter(':1'), Parameter(':2'), Parameter(':3'))

.. code-block:: sql

    INSERT INTO customers (id,fname,lname) VALUES (:1,:2,:3)

This allows you to build prepared statements, and/or avoid SQL-injection related risks.

Due to the mix of syntax for parameters, depending on connector/driver, it is required that you specify the
parameter token explicitly or use one of the specialized Parameter types per [PEP-0249](https://www.python.org/dev/peps/pep-0249/#paramstyle):
``QmarkParameter()``, ``NumericParameter(int)``,  ``NamedParameter(str)``, ``FormatParameter()``, ``PyformatParameter(str)``

An example of some common SQL parameter styles used in Python drivers are:

PostgreSQL:
    ``$number`` OR ``%s`` + ``:name`` (depending on driver)
MySQL:
    ``%s``
SQLite:
    ``?``
Vertica:
    ``:name``
Oracle:
    ``:number`` + ``:name``
MSSQL:
    ``%(name)s`` OR ``:name`` + ``:number`` (depending on driver)

You can find out what parameter style is needed for DBAPI compliant drivers here: https://www.python.org/dev/peps/pep-0249/#paramstyle or in the DB driver documentation.

Temporal support
^^^^^^^^^^^^^^^^

Temporal criteria can be added to the tables.

Select
""""""

Here is a select using system time.

.. code-block:: python

    t = Table("abc")
    q = Query.from_(t.for_(SYSTEM_TIME.as_of('2020-01-01'))).select("*")

This produces:

.. code-block:: sql

    SELECT * FROM "abc" FOR SYSTEM_TIME AS OF '2020-01-01'

You can also use between.

.. code-block:: python

    t = Table("abc")
    q = Query.from_(
        t.for_(SYSTEM_TIME.between('2020-01-01', '2020-02-01'))
    ).select("*")

This produces:

.. code-block:: sql

    SELECT * FROM "abc" FOR SYSTEM_TIME BETWEEN '2020-01-01' AND '2020-02-01'

You can also use a period range.

.. code-block:: python

    t = Table("abc")
    q = Query.from_(
        t.for_(SYSTEM_TIME.from_to('2020-01-01', '2020-02-01'))
    ).select("*")

This produces:

.. code-block:: sql

    SELECT * FROM "abc" FOR SYSTEM_TIME FROM '2020-01-01' TO '2020-02-01'

Finally you can select for all times:

.. code-block:: python

    t = Table("abc")
    q = Query.from_(t.for_(SYSTEM_TIME.all_())).select("*")

This produces:

.. code-block:: sql

    SELECT * FROM "abc" FOR SYSTEM_TIME ALL

A user defined period can also be used in the following manner.

.. code-block:: python

    t = Table("abc")
    q = Query.from_(
        t.for_(t.valid_period.between('2020-01-01', '2020-02-01'))
    ).select("*")

This produces:

.. code-block:: sql

    SELECT * FROM "abc" FOR "valid_period" BETWEEN '2020-01-01' AND '2020-02-01'

Joins
"""""

With joins, when the table object is used when specifying columns, it is
important to use the table from which the temporal constraint was generated.
This is because `Table("abc")` is not the same table as `Table("abc").for_(...)`.
The following example demonstrates this.

.. code-block:: python

    t0 = Table("abc").for_(SYSTEM_TIME.as_of('2020-01-01'))
    t1 = Table("efg").for_(SYSTEM_TIME.as_of('2020-01-01'))
    query = (
        Query.from_(t0)
        .join(t1)
        .on(t0.foo == t1.bar)
        .select("*")
    )

This produces:

.. code-block:: sql

    SELECT * FROM "abc" FOR SYSTEM_TIME AS OF '2020-01-01'
    JOIN "efg" FOR SYSTEM_TIME AS OF '2020-01-01'
    ON "abc"."foo"="efg"."bar"

Update & Deletes
""""""""""""""""

An update can be written as follows:

.. code-block:: python

    t = Table("abc")
    q = Query.update(
        t.for_portion(
            SYSTEM_TIME.from_to('2020-01-01', '2020-02-01')
        )
    ).set("foo", "bar")

This produces:

.. code-block:: sql

    UPDATE "abc"
    FOR PORTION OF SYSTEM_TIME FROM '2020-01-01' TO '2020-02-01'
    SET "foo"='bar'

Here is a delete:

.. code-block:: python

    t = Table("abc")
    q = Query.from_(
        t.for_portion(t.valid_period.from_to('2020-01-01', '2020-02-01'))
    ).delete()

This produces:

.. code-block:: sql

    DELETE FROM "abc"
    FOR PORTION OF "valid_period" FROM '2020-01-01' TO '2020-02-01'

Creating Tables
^^^^^^^^^^^^^^^

The entry point for creating tables is ``pypika.Query.create_table``, which is used with the class ``pypika.Column``.
As with selecting data, first the table should be specified. This can be either a
string or a `pypika.Table`. Then the columns, and constraints. Here's an example
that demonstrates much of the functionality.

.. code-block:: python

    stmt = Query \
        .create_table("person") \
        .columns(
            Column("id", "INT", nullable=False),
            Column("first_name", "VARCHAR(100)", nullable=False),
            Column("last_name", "VARCHAR(100)", nullable=False),
            Column("phone_number", "VARCHAR(20)", nullable=True),
            Column("status", "VARCHAR(20)", nullable=False, default=ValueWrapper("NEW")),
            Column("date_of_birth", "DATETIME")) \
        .unique("last_name", "first_name") \
        .primary_key("id")

This produces:

.. code-block:: sql

    CREATE TABLE "person" (
        "id" INT NOT NULL,
        "first_name" VARCHAR(100) NOT NULL,
        "last_name" VARCHAR(100) NOT NULL,
        "phone_number" VARCHAR(20) NULL,
        "status" VARCHAR(20) NOT NULL DEFAULT 'NEW',
        "date_of_birth" DATETIME,
        UNIQUE ("last_name","first_name"),
        PRIMARY KEY ("id")
    )

There is also support for creating a table from a query.

.. code-block:: python

    stmt = Query.create_table("names").as_select(
        Query.from_("person").select("last_name", "first_name")
    )

This produces:

.. code-block:: sql

        CREATE TABLE "names" AS (SELECT "last_name","first_name" FROM "person")

.. _tutorial_end:

.. _license_start:

License
-------

Copyright 2020 KAYAK Germany, GmbH

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Crafted with ♥ in Berlin.

.. _license_end:

.. _appendix_start:

.. |Brand| replace:: *PyPika*

.. _appendix_end:

.. _available_badges_start:

.. |BuildStatus| image:: https://github.com/kayak/pypika/workflows/Unit%20Tests/badge.svg
   :target: https://github.com/kayak/pypika/actions
.. |CoverageStatus| image:: https://coveralls.io/repos/kayak/pypika/badge.svg?branch=master
   :target: https://coveralls.io/github/kayak/pypika?branch=master
.. |Codacy| image:: https://api.codacy.com/project/badge/Grade/6d7e44e5628b4839a23da0bd82eaafcf
   :target: https://www.codacy.com/app/twheys/pypika
.. |Docs| image:: https://readthedocs.org/projects/pypika/badge/?version=latest
   :target: http://pypika.readthedocs.io/en/latest/
.. |PyPi| image:: https://img.shields.io/pypi/v/pypika.svg?style=flat
   :target: https://pypi.python.org/pypi/pypika
.. |License| image:: https://img.shields.io/hexpm/l/plug.svg?maxAge=2592000
   :target: http://www.apache.org/licenses/LICENSE-2.0

.. _available_badges_end:


text:
//
//  CDKCoreDataUtilities.m
//  CDKCoreDataManager
//
//  Created by Neill, Michael on 11/7/12.
//  Copyright (c) 2012 Velocitor Solutions. All rights reserved.
//

#import "CDKCoreDataUtilities.h"
#import "CDKCoreDataManager.h"
#import "CDKLogger.h"
#import "PropertyUtilities.h"
#import "Base64.h"

@implementation CDKCoreDataUtilities

/*
 * Method Name: convertObjectToDictionary:usingObjectMapping:
 * @object: An object to convert into a NSDictionary
 * @objectMapping: A ObjectMapping object used to convert the object
 * Description: This method using the supplied ObjectMapping to convert a object to a NSDictionary
 * Note: This method is really only used when it is nessecary to convert an object that is not a subclass of CDKBaseManagedObject
 */
+ (NSDictionary *)convertObjectToDictionary:(id)object usingObjectMapping:(CDKObjectMapping *)objectMapping
{
    NSMutableDictionary *returnDictionary = [[NSMutableDictionary alloc] init];

    NSDictionary *objectProperties = [PropertyUtilities classPropsFor:[object class]];

    // iterate through the object mapping and set each property to the dictionary
    for (id mappingObject in objectMapping.mapping)
    {
        // if this is an attribute, then set it
        if ([mappingObject isKindOfClass:[CDKObjectMappingAttribute class]])
        {
            // snag a proper pointer to the attribute and get the object's value
            CDKObjectMappingAttribute *attribute = (CDKObjectMappingAttribute *)mappingObject;
            id value = [object valueForKey:attribute.attribute];

            // if no value, then set it to an NSNull object
            if (!value)
            {
                // set default value for type of class
                NSString *type = [objectProperties valueForKey:attribute.attribute];
                Class propClass = NSClassFromString(type);
                if (propClass == [NSString class] || propClass == [NSDate class])
                {
                    value = @"";
                }
                else if (propClass == [NSNumber class])
                {
                    value = [NSNumber numberWithInt:0];
                }
                else
                {
                    value = [[NSNull alloc] init];
                }
            }
            // else if this is a NSData object, then convert it to a Base64 string
            else if ([value isKindOfClass:[NSData class]])
            {
                value = [Base64 encode:value];
            }
            // else if this an NSDate, then convert it to a DotNet date/time string
            else if ([value isKindOfClass:[NSDate class]])
            {
                NSLocale *locale = [[NSLocale alloc] initWithLocaleIdentifier:@"en_US_POSIX"];
                NSDateFormatter *dateFormatter = [[NSDateFormatter alloc] init];
                dateFormatter.locale = locale;
                dateFormatter.timeZone = [NSTimeZone defaultTimeZone];
                dateFormatter.dateFormat = [[CDKCoreDataManager sharedManager] dateFormat];
                value = [dateFormatter stringFromDate:value];
            }

            // set the value to the proper keypath
            [returnDictionary setObject:value forKey:attribute.keyPath];
        }
        // else this is a relationship, so call the method again to get it's properties, and set it to a new dictionary
        else
        {
            // snag a proper pointer to the attribute and the object's value
            CDKObjectMappingRelationship *relationship = (CDKObjectMappingRelationship *)mappingObject;
            id value = [object valueForKey:relationship.relationship];

            // if the value is null, then set an NSNull object
            if (!value)
            {
                // set default value for type of class
                NSString *type = [objectProperties valueForKey:relationship.relationship];
                Class propClass = NSClassFromString(type);
                if (propClass == [NSString class] || propClass == [NSDate class])
                {
                    value = @"";
                }
                else if (propClass == [NSNumber class])
                {
                    value = 0;
                }
                else
                {
                    value = [[NSNull alloc] init];
                }

                // set the return type
                [returnDictionary setObject:value forKey:relationship.keyPath];
            }
            // if this is an array or set then iterate through each object, call this method again to get each object's dictionary and add to an array
            else if ([value isKindOfClass:[NSArray class]] || [value isKindOfClass:[NSSet class]])
            {
                NSMutableArray *retArr = [[NSMutableArray alloc] init];
                for (id object in value)
                {
                    NSDictionary *retDict = [CDKBaseManagedObject convertCDKBaseManagedObjectToDictionary:object];
                    [retArr addObject:retDict];
                }

                // set the array to the proper keypath
                [returnDictionary setObject:retArr forKey:relationship.keyPath];
            }
            // else, this is a direct object relationship, so just call this method once to get the object's dictionary
            else
            {
                NSDictionary *retDict = [CDKBaseManagedObject convertCDKBaseManagedObjectToDictionary:value];
                [returnDictionary setObject:retDict forKey:relationship.keyPath];
            }
        }
    }

    return returnDictionary;
}

+ (NSString *)cleanClassName:(NSString *)className
{
    NSString *nClassName = [className copy];

    if ([nClassName rangeOfString:@"?"].location == NSNotFound && [nClassName rangeOfString:@"."].location != NSNotFound)
    {
        NSRange r = [nClassName rangeOfString:@"."];
        nClassName = [nClassName substringFromIndex:r.location + r.length];
    }

    return nClassName;
}

@end


text:
#ifndef zenic_InBinarySerializer_h
#define zenic_InBinarySerializer_h

#include "InSerializer.h"

#include "SerializableAllocation.h"
#include "SerializableVersion.h"

#include "../Storage/Array.h"
#include "../Io/Stream.h"

namespace zenic
{
	class SerializableStructure;
}

namespace zenic
{

class InBinarySerializer : public InSerializer
{
public:

	InBinarySerializer();
	virtual ~InBinarySerializer();

	virtual void reset();
	virtual bool load(Stream& stream);

	void setCustomManager(SerializableManager* manager);

protected:

	enum
	{
		Version1 = 1,
		MaxVersion = Version1
	};

	enum Endian
	{
		LittleEndian,
		BigEndian
	};

	enum { SectorSize = 2048 };

	enum
	{
		MagicCookie = ('Z'<<24)|('N'<<16)|('I'<<8)|'C',
		MagicMarker = ('M'<<24)|('A'<<16)|('R'<<8)|'K'
	};

	virtual void pushGroup(const char* name, const char* type);
	virtual void popGroup();

	virtual void pushStructure(const char* name, SerializableStructure* structure, Pointer* ptr);
	virtual void popStructure();

	virtual void process(SerializableVersion& version);
	virtual void process(SerializableAllocation& allocation);
	virtual void process(const char* name, Serializable*& object);
	virtual void process(const char* name, Pointer* ptr, u32 elementSize);

	virtual void process(const char* name, f32& value);
	virtual void process(const char* name, bool& value);

	virtual void process(const char* name, s8& value);
	virtual void process(const char* name, u8& value);
	virtual void process(const char* name, c8& value);

	virtual void process(const char* name, s16& value);
	virtual void process(const char* name, u16& value);

	virtual void process(const char* name, s32& value);
	virtual void process(const char* name, u32& value);

	virtual void process(const char* name, s64& value);
	virtual void process(const char* name, u64& value);

private:

	class Pool
	{
	public:

		void setData(DataPtr<u8> data);
		DataPtr<u8> data() const;

		void setAllocator(Allocator* allocator);
		Allocator* allocator() const;

		void setHandle(u32 handle);
		u32 handle() const;

		void setSize(u32 size);
		u32 size() const;

	private:

		DataPtr<u8> m_data;
		Allocator* m_allocator;
		u32 m_handle;
		u32 m_size;
	};

	class Object
	{
	public:

		void setFactory(SerializableFactory* factory);
		SerializableFactory* factory() const;

		void setOffset(u32 offset);
		u32 offset() const;

		void setObject(Serializable* object);
		Serializable* object() const;

	private:

		SerializableFactory* m_factory;
		u32 m_offset;
		Serializable* m_object;
	};

	class Block
	{
	public:

		Block() : m_data(0), m_offset(0), m_pool(0), m_structure(0), m_owner(0), m_allocator(0), m_align(0), m_size(0) {}

		bool load(Stream& stream);

		void setData(u8* data);
		u8* data() const;

		void setOffset(u32 offset);
		u32 offset() const;

		void setPool(u32 pool);
		u32 pool() const;

		u32 owner() const;
		SerializableStructure* structure() const;
		u32 allocator() const;
		u32 align() const;
		u32 size() const;

	private:

		u8* m_data;
		u32 m_offset;
		u32 m_pool;

		SerializableStructure* m_structure;
		u32 m_owner;
		u32 m_allocator;
		u32 m_align;
		u32 m_size;

	};

	DataPtr<Pool> m_pools;
	DataPtr<Object> m_objects;
	DataPtr<Block> m_blocks;

	SerializableManager* m_customManager;

	u32 m_endian;
	u32 m_blockIndex;
	u32 m_objectIndex;

	Stream* m_stream;
};

#include "InBinarySerializer.inl"

}

#endif


text:
#include "tensorflow/lite/c/builtin_op_data.h"
#include "tensorflow/lite/c/c_api_internal.h"
#include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h"
#include "tensorflow/lite/kernels/internal/tensor.h"
#include "tensorflow/lite/kernels/kernel_util.h"
#include "tensorflow/lite/kernels/op_macros.h"

namespace tflite {
namespace ops {
namespace builtin {
namespace quantize {

struct OpData {
  int32_t output_multiplier;
  int output_shift;
};

void* Init(TfLiteContext* context, const char* buffer, size_t length) {
  auto* data = new OpData;
  return data;
}

void Free(TfLiteContext* context, void* buffer) {
  delete reinterpret_cast<OpData*>(buffer);
}

struct OpContext {
  OpContext(TfLiteContext* context, TfLiteNode* node) {
    input = GetInput(context, node, 0);
    output = GetOutput(context, node, 0);
  }
  const TfLiteTensor* input;
  TfLiteTensor* output;
};

TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
  OpData* data = reinterpret_cast<OpData*>(node->user_data);
  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);

  OpContext op_context(context, node);

  TF_LITE_ENSURE(context, op_context.output->type == kTfLiteUInt8 ||
                              op_context.output->type == kTfLiteInt8);

  // TODO(b/128934713): Add support for fixed-point per-channel quantization.
  // Currently this only support affine per-layer quantization.
  TF_LITE_ENSURE_EQ(context, op_context.output->quantization.type,
                    kTfLiteAffineQuantization);
  const auto* affine_quantization = reinterpret_cast<TfLiteAffineQuantization*>(
      op_context.output->quantization.params);
  TF_LITE_ENSURE(context, affine_quantization);
  TF_LITE_ENSURE(context, affine_quantization->scale);
  TF_LITE_ENSURE(context, affine_quantization->scale->size == 1);

  // For requantize use case.
  const bool is_requantize = (op_context.input->type == kTfLiteUInt8 ||
                              op_context.input->type == kTfLiteInt8) &&
                             (op_context.output->type == kTfLiteUInt8 ||
                              op_context.output->type == kTfLiteInt8);
  if (is_requantize) {
    const double effective_output_scale =
        static_cast<double>(op_context.input->params.scale) /
        static_cast<double>(op_context.output->params.scale);
    QuantizeMultiplier(effective_output_scale, &data->output_multiplier,
                       &data->output_shift);
  }

  return context->ResizeTensor(context, op_context.output,
                               TfLiteIntArrayCopy(op_context.input->dims));
}

TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
  OpData* data = reinterpret_cast<OpData*>(node->user_data);

  TfLiteTensor* input = &context->tensors[node->inputs->data[0]];
  TfLiteTensor* output = &context->tensors[node->outputs->data[0]];

  switch (input->type) {
    case kTfLiteFloat32: {
      // Float to int8, uint8.
      tflite::QuantizationParams op_params;
      op_params.zero_point = output->params.zero_point;
      op_params.scale = output->params.scale;
      if (output->type == kTfLiteInt8) {
        optimized_ops::AffineQuantize(
            op_params, GetTensorShape(input), GetTensorData<float>(input),
            GetTensorShape(output), GetTensorData<int8_t>(output));
      } else if (output->type == kTfLiteUInt8) {
        optimized_ops::AffineQuantize(
            op_params, GetTensorShape(input), GetTensorData<float>(input),
            GetTensorShape(output), GetTensorData<uint8_t>(output));
      } else {
        context->ReportError(
            context,
            "Input type %d with Output type %d is not currently supported.",
            input->type, output->type);
        return kTfLiteError;
      }
    } break;
    case kTfLiteInt8: {
      // int8 to int8, uint8.
      const int32_t size =
          MatchingFlatSize(GetTensorShape(input), GetTensorShape(output));
      if (output->type == kTfLiteInt8) {
        reference_ops::Requantize<int8_t, int8_t>(
            GetTensorData<int8_t>(input), size, data->output_multiplier,
            data->output_shift, input->params.zero_point,
            output->params.zero_point, GetTensorData<int8_t>(output));
      } else if (output->type == kTfLiteUInt8) {
        reference_ops::Requantize<int8_t, uint8_t>(
            GetTensorData<int8_t>(input), size, data->output_multiplier,
            data->output_shift, input->params.zero_point,
            output->params.zero_point, GetTensorData<uint8_t>(output));
      } else {
        context->ReportError(
            context,
            "Input type %d with Output type %d is not currently supported.",
            input->type, output->type);
        return kTfLiteError;
      }
    } break;
    case kTfLiteUInt8: {
      // uint8 to int8, uint8.
      const int32_t size =
          MatchingFlatSize(GetTensorShape(input), GetTensorShape(output));
      if (output->type == kTfLiteInt8) {
        reference_ops::Requantize<uint8_t, int8_t>(
            GetTensorData<uint8_t>(input), size, data->output_multiplier,
            data->output_shift, input->params.zero_point,
            output->params.zero_point, GetTensorData<int8_t>(output));
      } else if (output->type == kTfLiteUInt8) {
        reference_ops::Requantize<uint8_t, uint8_t>(
            GetTensorData<uint8_t>(input), size, data->output_multiplier,
            data->output_shift, input->params.zero_point,
            output->params.zero_point, GetTensorData<uint8_t>(output));
      } else {
        context->ReportError(
            context,
            "Input type %d with Output type %d is not currently supported.",
            input->type, output->type);
        return kTfLiteError;
      }
    } break;
    default:
      context->ReportError(
          context,
          "Input type %d with Output type %d is not currently supported.",
          input->type, output->type);
      return kTfLiteError;
  }

  return kTfLiteOk;
}

}  // namespace quantize

// This Op (QUANTIZE) quantizes the input and produces quantized output.
// The input can be either float or quantized. If the input is float,
// AffineQuantize takes scale and zero point and quantize the float value to
// quantized output, in int8 or uint8 format. If the input is quantized value,
// the op requantize the input (of a certain type, with a given scale and zero
// point) to the output of the same or different type with a same or different
// scale and zero point.
TfLiteRegistration* Register_QUANTIZE_OPT() {
  static TfLiteRegistration r = {quantize::Init, quantize::Free,
                                 quantize::Prepare, quantize::Eval};
  return &r;
}

TfLiteRegistration* Register_QUANTIZE() { return Register_QUANTIZE_OPT(); }

}  // namespace builtin
}  // namespace ops
}  // namespace tflite


text:
package org.sql2o.converters.joda;

import org.joda.time.LocalTime;
import org.sql2o.converters.Converter;
import org.sql2o.converters.ConverterException;

import java.sql.Timestamp;

/**
 * Created by lars on 12/18/13.
 */
public class LocalTimeConverter implements Converter<LocalTime> {

    public LocalTime convert(Object val) throws ConverterException {
        if (val == null) {
            return null;
        }
        try {
            // Joda has it's own pluggable converters infrastructure
            // it will throw IllegalArgumentException if can't convert
            // look @ org.joda.time.convert.ConverterManager
            return new LocalTime(val);
        } catch (IllegalArgumentException ex) {
            throw new ConverterException("Don't know how to convert from type '" + val.getClass().getName() + "' to type '" + LocalTime.class.getName() + "'", ex);
        }
    }

    public Object toDatabaseParam(LocalTime val) {
        return new Timestamp(val.toDateTimeToday().getMillis());
    }
}


text:
/**
 * @ngdoc directive
 * @name refigureProfile.directive:dashboard
 * @restrict E
 * @description
 * Profile page
 * @example
 * <my-collections></my-collections>
 */
(function (angular) {
    'use strict';

    angular
        .module('refigureProfile')
        .component('dashboard', {
            templateUrl: 'view/dashboard.component.html',
            controller: Controller,
            controllerAs: 'vm'
        });

    Controller.$inject = ['collections', 'rfToast', '$http'];

    function Controller(collections, rfToast, $http) {
        var vm = this;

        vm.stats = null;

        vm.downloadsCSV = downloadsCSV;

        vm.$onInit = activate;

        /////////////////////

        /**
         * @ngdoc method
         * @name refigureProfile.directive:dashboard#activate
         * @methodOf refigureProfile.directive:dashboard
         * @description
         * Activates controller
         */
        function activate() {
            collections.statistics()
                .then(function (resp) {
                    vm.stats = resp;
                });
        }

        /**
         * @ngdoc method
         * @name refigureProfile.directive:dashboard#downloadsCSV
         * @methodOf refigureProfile.directive:dashboard
         * @description
         * Get downloads and generate CSV file
         */
        function downloadsCSV() {
            return $http
                .get('/api/downloads')
                .then(function (res) {
                    var data = utils.get(res, 'data.data') || [];
                    if (!data.length) {
                        rfToast.show('Extension was not downloaded yet');
                    } else {
                        generateCSV(data);
                    }
                });
        }

        function generateCSV(data) {
            var csvArr = [];
            csvArr.push(
                '"' + Object.keys(data[0]).join('","') + '"'
            );
            data.forEach(function (obj) {
                csvArr.push(
                    '"' + Object.keys(obj).map(function (key) {
                        return obj[key];
                    }).join('","') + '"'
                );
            });
            var blob = new Blob([csvArr.join('\n')], {type: 'text/csv;charset=utf-8;'});
            var tempLink = document.createElement('a');
            tempLink.href = window.URL.createObjectURL(blob);
            tempLink.setAttribute('download', 'downloads.csv');
            tempLink.click();
        }

    }
})(window.angular);


text:
/* global getNodeDetails */

import FRGatherer from '../base-gatherer.js';
import {pageFunctions} from '../../lib/page-functions.js';

/* eslint-env browser, node */

/**
 * @return {LH.Artifacts['IFrameElements']}
 */
/* c8 ignore start */
function collectIFrameElements() {
  const realBoundingClientRect = window.__HTMLElementBoundingClientRect ||
    window.HTMLElement.prototype.getBoundingClientRect;

  // @ts-expect-error - put into scope via stringification
  const iFrameElements = getElementsInDocument('iframe'); // eslint-disable-line no-undef
  return iFrameElements.map(/** @param {HTMLIFrameElement} node */ (node) => {
    const clientRect = realBoundingClientRect.call(node);
    const {top, bottom, left, right, width, height} = clientRect;
    return {
      id: node.id,
      src: node.src,
      clientRect: {top, bottom, left, right, width, height},
      // @ts-expect-error - put into scope via stringification
      isPositionFixed: isPositionFixed(node), // eslint-disable-line no-undef
      // @ts-expect-error - getNodeDetails put into scope via stringification
      node: getNodeDetails(node),
    };
  });
}
/* c8 ignore stop */

class IFrameElements extends FRGatherer {
  /** @type {LH.Gatherer.GathererMeta} */
  meta = {
    supportedModes: ['snapshot', 'navigation'],
  };

  /**
   * @param {LH.Gatherer.FRTransitionalContext} passContext
   * @return {Promise<LH.Artifacts['IFrameElements']>}
   * @override
   */
  async getArtifact(passContext) {
    const driver = passContext.driver;

    const iframeElements = await driver.executionContext.evaluate(collectIFrameElements, {
      args: [],
      useIsolation: true,
      deps: [
        pageFunctions.getElementsInDocument,
        pageFunctions.isPositionFixed,
        pageFunctions.getNodeDetails,
      ],
    });
    return iframeElements;
  }
}

export default IFrameElements;