text:
"""Generic base class to implement metadata, compression and chunked data
   operations
"""

import abc
import hashlib
import json
import os

import eventlet
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import excutils
from oslo_utils import units
import six

from cinder.backup import driver
from cinder import exception
from cinder.i18n import _
from cinder import objects
from cinder.objects import fields
from cinder.volume import utils as volume_utils

LOG = logging.getLogger(__name__)

chunkedbackup_service_opts = [
    cfg.StrOpt('backup_compression_algorithm',
               default='zlib',
               choices=['none', 'off', 'no',
                        'zlib', 'gzip',
                        'bz2', 'bzip2'],
               help='Compression algorithm (None to disable)'),
]

CONF = cfg.CONF
CONF.register_opts(chunkedbackup_service_opts)

# Object writer and reader returned by inheriting classes must not have any
# logging calls, as well as the compression libraries, as eventlet has a bug
# (https://github.com/eventlet/eventlet/issues/432) that would result in
# failures.

@six.add_metaclass(abc.ABCMeta)
class ChunkedBackupDriver(driver.BackupDriver):
    """Abstract chunked backup driver.

       Implements common functionality for backup drivers that store volume
       data in multiple "chunks" in a backup repository when the size of
       the backed up cinder volume exceeds the size of a backup repository
       "chunk."

       Provides abstract methods to be implemented in concrete chunking
       drivers.
    """

    DRIVER_VERSION = '1.0.0'
    DRIVER_VERSION_MAPPING = {'1.0.0': '_restore_v1'}

    def _get_compressor(self, algorithm):
        try:
            if algorithm.lower() in ('none', 'off', 'no'):
                return None
            if algorithm.lower() in ('zlib', 'gzip'):
                import zlib as compressor
                result = compressor
            elif algorithm.lower() in ('bz2', 'bzip2'):
                import bz2 as compressor
                result = compressor
            else:
                result = None
            if result:
                # NOTE(geguileo): Compression/Decompression starves
                # greenthreads so we use a native thread instead.
                return eventlet.tpool.Proxy(result)
        except ImportError:
            pass

        err = _('unsupported compression algorithm: %s') % algorithm
        raise ValueError(err)

    def __init__(self, context, chunk_size_bytes, sha_block_size_bytes,
                 backup_default_container, enable_progress_timer,
                 db=None):
        super(ChunkedBackupDriver, self).__init__(context, db)
        self.chunk_size_bytes = chunk_size_bytes
        self.sha_block_size_bytes = sha_block_size_bytes
        self.backup_default_container = backup_default_container
        self.enable_progress_timer = enable_progress_timer

        self.backup_timer_interval = CONF.backup_timer_interval
        self.data_block_num = CONF.backup_object_number_per_notification
        self.az = CONF.storage_availability_zone
        self.backup_compression_algorithm = CONF.backup_compression_algorithm
        self.compressor = \
            self._get_compressor(CONF.backup_compression_algorithm)
        self.support_force_delete = True

    def _get_object_writer(self, container, object_name, extra_metadata=None):
        """Return writer proxy-wrapped to execute methods in native thread."""
        writer = self.get_object_writer(container, object_name, extra_metadata)
        return eventlet.tpool.Proxy(writer)

    def _get_object_reader(self, container, object_name, extra_metadata=None):
        """Return reader proxy-wrapped to execute methods in native thread."""
        reader = self.get_object_reader(container, object_name, extra_metadata)
        return eventlet.tpool.Proxy(reader)

    # To create your own "chunked" backup driver, implement the following
    # abstract methods.

    @abc.abstractmethod
    def put_container(self, container):
        """Create the container if needed. No failure if it pre-exists."""
        return

    @abc.abstractmethod
    def get_container_entries(self, container, prefix):
        """Get container entry names."""
        return

    @abc.abstractmethod
    def get_object_writer(self, container, object_name, extra_metadata=None):
        """Returns a writer object which stores the chunk data in backup repository.

        The object returned should be a context handler that can be used in a
        "with" context.

        The object writer methods must not have any logging calls, as eventlet
        has a bug (https://github.com/eventlet/eventlet/issues/432) that would
        result in failures.
        """
        return

    @abc.abstractmethod
    def get_object_reader(self, container, object_name, extra_metadata=None):
        """Returns a reader object for the backed up chunk.

        The object reader methods must not have any logging calls, as eventlet
        has a bug (https://github.com/eventlet/eventlet/issues/432) that would
        result in failures.
        """
        return

    @abc.abstractmethod
    def delete_object(self, container, object_name):
        """Delete object from container."""
        return

    @abc.abstractmethod
    def _generate_object_name_prefix(self, backup):
        return

    @abc.abstractmethod
    def update_container_name(self, backup, container):
        """Allow sub-classes to override container name.

        This method exists so that sub-classes can override the container name
        as it comes in to the driver in the backup object. Implementations
        should return None if no change to the container name is desired.
        """
        return

    @abc.abstractmethod
    def get_extra_metadata(self, backup, volume):
        """Return extra metadata to use in prepare_backup.

        This method allows for collection of extra metadata in prepare_backup()
        which will be passed to get_object_reader() and get_object_writer().
        Subclass extensions can use this extra information to optimize
        data transfers. Return a json serializable object.
        """
        return

    def _create_container(self, backup):
        # Container's name will be decided by the driver (returned by method
        # update_container_name), if no change is required by the driver then
        # we'll use the one the backup object already has, but if it doesn't
        # have one backup_default_container will be used.
        new_container = self.update_container_name(backup, backup.container)
        if new_container:
            # If the driver is not really changing the name we don't want to
            # dirty the field in the object and save it to the DB with the same
            # value.
            if new_container != backup.container:
                backup.container = new_container
        elif backup.container is None:
            backup.container = self.backup_default_container

        LOG.debug('_create_container started, container: %(container)s,'
                  'backup: %(backup_id)s.',
                  {'container': backup.container, 'backup_id': backup.id})

        backup.save()
        self.put_container(backup.container)
        return backup.container

    def _generate_object_names(self, backup):
        prefix = backup['service_metadata']
        object_names = self.get_container_entries(backup['container'], prefix)
        LOG.debug('generated object list: %s.', object_names)
        return object_names

    def _metadata_filename(self, backup):
        object_name = backup['service_metadata']
        filename = '%s_metadata' % object_name
        return filename

    def _sha256_filename(self, backup):
        object_name = backup['service_metadata']
        filename = '%s_sha256file' % object_name
        return filename

    def _write_metadata(self, backup, volume_id, container, object_list,
                        volume_meta, extra_metadata=None):
        filename = self._metadata_filename(backup)
        LOG.debug('_write_metadata started, container name: %(container)s,'
                  ' metadata filename: %(filename)s.',
                  {'container': container, 'filename': filename})
        metadata = {}
        metadata['version'] = self.DRIVER_VERSION
        metadata['backup_id'] = backup['id']
        metadata['volume_id'] = volume_id
        metadata['backup_name'] = backup['display_name']
        metadata['backup_description'] = backup['display_description']
        metadata['created_at'] = str(backup['created_at'])
        metadata['objects'] = object_list
        metadata['parent_id'] = backup['parent_id']
        metadata['volume_meta'] = volume_meta
        if extra_metadata:
            metadata['extra_metadata'] = extra_metadata
        metadata_json = json.dumps(metadata, sort_keys=True, indent=2)
        if six.PY3:
            metadata_json = metadata_json.encode('utf-8')
        with self._get_object_writer(container, filename) as writer:
            writer.write(metadata_json)
        LOG.debug('_write_metadata finished. Metadata: %s.', metadata_json)

    def _write_sha256file(self, backup, volume_id, container, sha256_list):
        filename = self._sha256_filename(backup)
        LOG.debug('_write_sha256file started, container name: %(container)s,'
                  ' sha256file filename: %(filename)s.',
                  {'container': container, 'filename': filename})
        sha256file = {}
        sha256file['version'] = self.DRIVER_VERSION
        sha256file['backup_id'] = backup['id']
        sha256file['volume_id'] = volume_id
        sha256file['backup_name'] = backup['display_name']
        sha256file['backup_description'] = backup['display_description']
        sha256file['created_at'] = six.text_type(backup['created_at'])
        sha256file['chunk_size'] = self.sha_block_size_bytes
        sha256file['sha256s'] = sha256_list
        sha256file_json = json.dumps(sha256file, sort_keys=True, indent=2)
        if six.PY3:
            sha256file_json = sha256file_json.encode('utf-8')
        with self._get_object_writer(container, filename) as writer:
            writer.write(sha256file_json)
        LOG.debug('_write_sha256file finished.')

    def _read_metadata(self, backup):
        container = backup['container']
        filename = self._metadata_filename(backup)
        LOG.debug('_read_metadata started, container name: %(container)s, '
                  'metadata filename: %(filename)s.',
                  {'container': container, 'filename': filename})
        with self._get_object_reader(container, filename) as reader:
            metadata_json = reader.read()
        if six.PY3:
            metadata_json = metadata_json.decode('utf-8')
        metadata = json.loads(metadata_json)
        LOG.debug('_read_metadata finished. Metadata: %s.', metadata_json)
        return metadata

    def _read_sha256file(self, backup):
        container = backup['container']
        filename = self._sha256_filename(backup)
        LOG.debug('_read_sha256file started, container name: %(container)s, '
                  'sha256 filename: %(filename)s.',
                  {'container': container, 'filename': filename})
        with self._get_object_reader(container, filename) as reader:
            sha256file_json = reader.read()
        if six.PY3:
            sha256file_json = sha256file_json.decode('utf-8')
        sha256file = json.loads(sha256file_json)
        LOG.debug('_read_sha256file finished.')
        return sha256file

    def _prepare_backup(self, backup):
        """Prepare the backup process and return the backup metadata."""
        volume = self.db.volume_get(self.context, backup.volume_id)

        if volume['size'] <= 0:
            err = _('volume size %d is invalid.') % volume['size']
            raise exception.InvalidVolume(reason=err)

        container = self._create_container(backup)

        object_prefix = self._generate_object_name_prefix(backup)
        backup.service_metadata = object_prefix
        backup.save()

        volume_size_bytes = volume['size'] * units.Gi
        availability_zone = self.az
        LOG.debug('starting backup of volume: %(volume_id)s,'
                  ' volume size: %(volume_size_bytes)d, object names'
                  ' prefix %(object_prefix)s, availability zone:'
                  ' %(availability_zone)s',
                  {
                      'volume_id': backup.volume_id,
                      'volume_size_bytes': volume_size_bytes,
                      'object_prefix': object_prefix,
                      'availability_zone': availability_zone,
                  })
        object_meta = {'id': 1, 'list': [], 'prefix': object_prefix,
                       'volume_meta': None}
        object_sha256 = {'id': 1, 'sha256s': [], 'prefix': object_prefix}
        extra_metadata = self.get_extra_metadata(backup, volume)
        if extra_metadata is not None:
            object_meta['extra_metadata'] = extra_metadata

        return (object_meta, object_sha256, extra_metadata, container,
                volume_size_bytes)

    def _backup_chunk(self, backup, container, data, data_offset,
                      object_meta, extra_metadata):
        """Backup data chunk based on the object metadata and offset."""
        object_prefix = object_meta['prefix']
        object_list = object_meta['list']

        object_id = object_meta['id']
        object_name = '%s-%05d' % (object_prefix, object_id)
        obj = {}
        obj[object_name] = {}
        obj[object_name]['offset'] = data_offset
        obj[object_name]['length'] = len(data)
        LOG.debug('Backing up chunk of data from volume.')
        algorithm, output_data = self._prepare_output_data(data)
        obj[object_name]['compression'] = algorithm
        LOG.debug('About to put_object')
        with self._get_object_writer(
                container, object_name, extra_metadata=extra_metadata
        ) as writer:
            writer.write(output_data)
        md5 = eventlet.tpool.execute(hashlib.md5, data).hexdigest()
        obj[object_name]['md5'] = md5
        LOG.debug('backup MD5 for %(object_name)s: %(md5)s',
                  {'object_name': object_name, 'md5': md5})
        object_list.append(obj)
        object_id += 1
        object_meta['list'] = object_list
        object_meta['id'] = object_id

        LOG.debug('Calling eventlet.sleep(0)')
        eventlet.sleep(0)

    def _prepare_output_data(self, data):
        if self.compressor is None:
            return 'none', data
        data_size_bytes = len(data)
        # Execute compression in native thread so it doesn't prevent
        # cooperative greenthread switching.
        compressed_data = self.compressor.compress(data)
        comp_size_bytes = len(compressed_data)
        algorithm = CONF.backup_compression_algorithm.lower()
        if comp_size_bytes >= data_size_bytes:
            LOG.debug('Compression of this chunk was ineffective: '
                      'original length: %(data_size_bytes)d, '
                      'compressed length: %(compressed_size_bytes)d. '
                      'Using original data for this chunk.',
                      {'data_size_bytes': data_size_bytes,
                       'compressed_size_bytes': comp_size_bytes,
                       })
            return 'none', data
        LOG.debug('Compressed %(data_size_bytes)d bytes of data '
                  'to %(comp_size_bytes)d bytes using %(algorithm)s.',
                  {'data_size_bytes': data_size_bytes,
                   'comp_size_bytes': comp_size_bytes,
                   'algorithm': algorithm,
                   })
        return algorithm, compressed_data

    def _finalize_backup(self, backup, container, object_meta, object_sha256):
        """Write the backup's metadata to the backup repository."""
        object_list = object_meta['list']
        object_id = object_meta['id']
        volume_meta = object_meta['volume_meta']
        sha256_list = object_sha256['sha256s']
        extra_metadata = object_meta.get('extra_metadata')
        self._write_sha256file(backup,
                               backup.volume_id,
                               container,
                               sha256_list)
        self._write_metadata(backup,
                             backup.volume_id,
                             container,
                             object_list,
                             volume_meta,
                             extra_metadata)
        backup.object_count = object_id
        backup.save()
        LOG.debug('backup %s finished.', backup['id'])

    def _backup_metadata(self, backup, object_meta):
        """Backup volume metadata.

        NOTE(dosaboy): the metadata we are backing up is obtained from a
                       versioned api so we should not alter it in any way here.
                       We must also be sure that the service that will perform
                       the restore is compatible with version used.
        """
        json_meta = self.get_metadata(backup['volume_id'])
        if not json_meta:
            LOG.debug("No volume metadata to backup.")
            return

        object_meta["volume_meta"] = json_meta

    def _send_progress_end(self, context, backup, object_meta):
        object_meta['backup_percent'] = 100
        volume_utils.notify_about_backup_usage(context,
                                               backup,
                                               "createprogress",
                                               extra_usage_info=
                                               object_meta)

    def _send_progress_notification(self, context, backup, object_meta,
                                    total_block_sent_num, total_volume_size):
        backup_percent = total_block_sent_num * 100 / total_volume_size
        object_meta['backup_percent'] = backup_percent
        volume_utils.notify_about_backup_usage(context,
                                               backup,
                                               "createprogress",
                                               extra_usage_info=
                                               object_meta)

    def _calculate_sha(self, data):
        """Calculate SHA256 of a data chunk.

        This method cannot log anything as it is called on a native thread.
        """
        # NOTE(geguileo): Using memoryview to avoid data copying when slicing
        # for the sha256 call.
        chunk = memoryview(data)
        shalist = []
        off = 0
        datalen = len(chunk)
        while off < datalen:
            chunk_end = min(datalen, off + self.sha_block_size_bytes)
            block = chunk[off:chunk_end]
            sha = hashlib.sha256(block).hexdigest()
            shalist.append(sha)
            off += self.sha_block_size_bytes
        return shalist

    def backup(self, backup, volume_file, backup_metadata=True):
        """Backup the given volume.

           If backup['parent_id'] is given, then an incremental backup
           is performed.
        """
        if self.chunk_size_bytes % self.sha_block_size_bytes:
            err = _('Chunk size is not multiple of '
                    'block size for creating hash.')
            raise exception.InvalidBackup(reason=err)

        # Read the shafile of the parent backup if backup['parent_id']
        # is given.
        parent_backup_shafile = None
        parent_backup = None
        if backup.parent_id:
            parent_backup = objects.Backup.get_by_id(self.context,
                                                     backup.parent_id)
            parent_backup_shafile = self._read_sha256file(parent_backup)
            parent_backup_shalist = parent_backup_shafile['sha256s']
            if (parent_backup_shafile['chunk_size'] !=
                    self.sha_block_size_bytes):
                err = (_('Hash block size has changed since the last '
                         'backup. New hash block size: %(new)s. Old hash '
                         'block size: %(old)s. Do a full backup.')
                       % {'old': parent_backup_shafile['chunk_size'],
                          'new': self.sha_block_size_bytes})
                raise exception.InvalidBackup(reason=err)
            # If the volume size increased since the last backup, fail
            # the incremental backup and ask user to do a full backup.
            if backup.size > parent_backup.size:
                err = _('Volume size increased since the last '
                        'backup. Do a full backup.')
                raise exception.InvalidBackup(reason=err)

        (object_meta, object_sha256, extra_metadata, container,
         volume_size_bytes) = self._prepare_backup(backup)

        counter = 0
        total_block_sent_num = 0

        # There are two mechanisms to send the progress notification.
        # 1. The notifications are periodically sent in a certain interval.
        # 2. The notifications are sent after a certain number of chunks.
        # Both of them are working simultaneously during the volume backup,
        # when "chunked" backup drivers are deployed.
        def _notify_progress():
            self._send_progress_notification(self.context, backup,
                                             object_meta,
                                             total_block_sent_num,
                                             volume_size_bytes)
        timer = loopingcall.FixedIntervalLoopingCall(
            _notify_progress)
        if self.enable_progress_timer:
            timer.start(interval=self.backup_timer_interval)

        sha256_list = object_sha256['sha256s']
        shaindex = 0
        is_backup_canceled = False
        while True:
            # First of all, we check the status of this backup. If it
            # has been changed to delete or has been deleted, we cancel the
            # backup process to do forcing delete.
            with backup.as_read_deleted():
                backup.refresh()
            if backup.status in (fields.BackupStatus.DELETING,
                                 fields.BackupStatus.DELETED):
                is_backup_canceled = True
                # To avoid the chunk left when deletion complete, need to
                # clean up the object of chunk again.
                self.delete_backup(backup)
                LOG.debug('Cancel the backup process of %s.', backup.id)
                break
            data_offset = volume_file.tell()
            data = volume_file.read(self.chunk_size_bytes)
            if data == b'':
                break

            # Calculate new shas with the datablock.
            shalist = eventlet.tpool.execute(self._calculate_sha, data)
            sha256_list.extend(shalist)

            # If parent_backup is not None, that means an incremental
            # backup will be performed.
            if parent_backup:
                # Find the extent that needs to be backed up.
                extent_off = -1
                for idx, sha in enumerate(shalist):
                    if sha != parent_backup_shalist[shaindex]:
                        if extent_off == -1:
                            # Start of new extent.
                            extent_off = idx * self.sha_block_size_bytes
                    else:
                        if extent_off != -1:
                            # We've reached the end of extent.
                            extent_end = idx * self.sha_block_size_bytes
                            segment = data[extent_off:extent_end]
                            self._backup_chunk(backup, container, segment,
                                               data_offset + extent_off,
                                               object_meta,
                                               extra_metadata)
                            extent_off = -1
                    shaindex += 1

                # The last extent extends to the end of data buffer.
                if extent_off != -1:
                    extent_end = len(data)
                    segment = data[extent_off:extent_end]
                    self._backup_chunk(backup, container, segment,
                                       data_offset + extent_off,
                                       object_meta, extra_metadata)
                    extent_off = -1
            else:  # Do a full backup.
                self._backup_chunk(backup, container, data, data_offset,
                                   object_meta, extra_metadata)

            # Notifications
            total_block_sent_num += self.data_block_num
            counter += 1
            if counter == self.data_block_num:
                # Send the notification to Ceilometer when the chunk
                # number reaches the data_block_num.  The backup percentage
                # is put in the metadata as the extra information.
                self._send_progress_notification(self.context, backup,
                                                 object_meta,
                                                 total_block_sent_num,
                                                 volume_size_bytes)
                # Reset the counter
                counter = 0

        # Stop the timer.
        timer.stop()
        # If backup has been cancelled we have nothing more to do
        # but timer.stop().
        if is_backup_canceled:
            return
        # All the data have been sent, the backup_percent reaches 100.
        self._send_progress_end(self.context, backup, object_meta)

        object_sha256['sha256s'] = sha256_list
        if backup_metadata:
            try:
                self._backup_metadata(backup, object_meta)
            # Whatever goes wrong, we want to log, cleanup, and re-raise.
            except Exception:
                with excutils.save_and_reraise_exception():
                    LOG.exception("Backup volume metadata failed.")
                    self.delete_backup(backup)

        self._finalize_backup(backup, container, object_meta, object_sha256)

    def _restore_v1(self, backup, volume_id, metadata, volume_file):
        """Restore a v1 volume backup."""
        backup_id = backup['id']
        LOG.debug('v1 volume backup restore of %s started.', backup_id)
        extra_metadata = metadata.get('extra_metadata')
        container = backup['container']
        metadata_objects = metadata['objects']
        metadata_object_names = []
        for obj in metadata_objects:
            metadata_object_names.extend(obj.keys())
        LOG.debug('metadata_object_names = %s.', metadata_object_names)
        prune_list = [self._metadata_filename(backup),
                      self._sha256_filename(backup)]
        object_names = [object_name for object_name in
                        self._generate_object_names(backup)
                        if object_name not in prune_list]
        if sorted(object_names) != sorted(metadata_object_names):
            err = _('restore_backup aborted, actual object list '
                    'does not match object list stored in metadata.')
            raise exception.InvalidBackup(reason=err)

        for metadata_object in metadata_objects:
            object_name, obj = list(metadata_object.items())[0]
            LOG.debug('restoring object. backup: %(backup_id)s, '
                      'container: %(container)s, object name: '
                      '%(object_name)s, volume: %(volume_id)s.',
                      {
                          'backup_id': backup_id,
                          'container': container,
                          'object_name': object_name,
                          'volume_id': volume_id,
                      })

            with self._get_object_reader(
                    container, object_name,
                    extra_metadata=extra_metadata) as reader:
                body = reader.read()
            compression_algorithm = metadata_object[object_name]['compression']
            decompressor = self._get_compressor(compression_algorithm)
            volume_file.seek(obj['offset'])
            if decompressor is not None:
                LOG.debug('decompressing data using %s algorithm',
                          compression_algorithm)
                decompressed = decompressor.decompress(body)
                volume_file.write(decompressed)
            else:
                volume_file.write(body)

            # force flush every write to avoid long blocking write on close
            volume_file.flush()

            # Be tolerant to IO implementations that do not support fileno()
            try:
                fileno = volume_file.fileno()
            except IOError:
                LOG.info("volume_file does not support fileno() so skipping "
                         "fsync()")
            else:
                os.fsync(fileno)

            # Restoring a backup to a volume can take some time. Yield so other
            # threads can run, allowing for among other things the service
            # status to be updated
            eventlet.sleep(0)
        LOG.debug('v1 volume backup restore of %s finished.',
                  backup_id)

    def restore(self, backup, volume_id, volume_file):
        """Restore the given volume backup from backup repository."""
        backup_id = backup['id']
        container = backup['container']
        object_prefix = backup['service_metadata']
        LOG.debug('starting restore of backup %(object_prefix)s '
                  'container: %(container)s, to volume %(volume_id)s, '
                  'backup: %(backup_id)s.',
                  {
                      'object_prefix': object_prefix,
                      'container': container,
                      'volume_id': volume_id,
                      'backup_id': backup_id,
                  })
        metadata = self._read_metadata(backup)
        metadata_version = metadata['version']
        LOG.debug('Restoring backup version %s', metadata_version)
        try:
            restore_func = getattr(self, self.DRIVER_VERSION_MAPPING.get(
                metadata_version))
        except TypeError:
            err = (_('No support to restore backup version %s')
                   % metadata_version)
            raise exception.InvalidBackup(reason=err)

        # Build a list of backups based on parent_id. A full backup
        # will be the last one in the list.
        backup_list = []
        backup_list.append(backup)
        current_backup = backup
        while current_backup.parent_id:
            prev_backup = objects.Backup.get_by_id(self.context,
                                                   current_backup.parent_id)
            backup_list.append(prev_backup)
            current_backup = prev_backup

        # Do a full restore first, then layer the incremental backups
        # on top of it in order.
        index = len(backup_list) - 1
        while index >= 0:
            backup1 = backup_list[index]
            index = index - 1
            metadata = self._read_metadata(backup1)
            restore_func(backup1, volume_id, metadata, volume_file)

            volume_meta = metadata.get('volume_meta', None)
            try:
                if volume_meta:
                    self.put_metadata(volume_id, volume_meta)
                else:
                    LOG.debug("No volume metadata in this backup.")
            except exception.BackupMetadataUnsupportedVersion:
                msg = _("Metadata restore failed due to incompatible version.")
                LOG.error(msg)
                raise exception.BackupOperationError(msg)

        LOG.debug('restore %(backup_id)s to %(volume_id)s finished.',
                  {'backup_id': backup_id, 'volume_id': volume_id})

    def delete_backup(self, backup):
        """Delete the given backup."""
        container = backup['container']
        object_prefix = backup['service_metadata']
        LOG.debug('delete started, backup: %(id)s, container: %(cont)s, '
                  'prefix: %(pre)s.',
                  {'id': backup['id'],
                   'cont': container,
                   'pre': object_prefix})

        if container is not None and object_prefix is not None:
            object_names = []
            try:
                object_names = self._generate_object_names(backup)
            except Exception:
                LOG.warning('Error while listing objects, continuing'
                            ' with delete.')

            for object_name in object_names:
                self.delete_object(container, object_name)
                LOG.debug('deleted object: %(object_name)s'
                          ' in container: %(container)s.',
                          {
                              'object_name': object_name,
                              'container': container
                          })
                # Deleting a backup's objects can take some time.
                # Yield so other threads can run
                eventlet.sleep(0)

        LOG.debug('delete %s finished.', backup['id'])


text:
(function ($, _, Backbone, Backgrid) {

  "use strict";

  /**
     Paginator is a Footer element that re-renders a large result set in a table
     by splitting it across multiple pages. If the result set is still larger,
     the page handles are rendered within a sliding window, with 10 indexed page
     handles each by default, plus the fast forward, fast backward, previous and
     next page handles. The fast forward, fast backward, previous and next page
     handles can be turned off.

     @class Backgrid.Extension.Paginator
  */
  Backgrid.Extension.Paginator = Backgrid.Footer.extend({

    /** @property */
    className: "paginator",

    /** @property */
    windowSize: 10,

    /** @property */
    fastForwardHandleLabels: {
      first: "《",
      prev: "〈",
      next: "〉",
      last: "》"
    },

    /** @property */
    template: _.template('
<td colspan="<%= colspan %>">
<% _.each(handles, function (handle) { %><li <% if (handle.className) { %>class="<%= handle.className %>"<% } %>><a href="#" <% if (handle.title) {%> title="<%= handle.title %>"<% } %>><%= handle.label %>
<% }); %>

'),

    /** @property */
    events: {
      "click a": "changePage"
    },

    /**
       Initializer.

       @param {Object} options
       @param {Backbone.Collection.<Backgrid.Column>|Array.<Backgrid.Column>|Array.<Object>} options.columns
       Column metadata.
       @param {Backbone.Collection} options.collection
       @param {boolean} [options.fastForwardHandleLabels] Whether to render fast forward buttons.
    */
    initialize: function (options) {
      Backgrid.Footer.prototype.initialize.call(this, options);
      var columns = this.columns;
      this.listenTo(columns, "add", this.render);
      this.listenTo(columns, "remove", this.render);
      this.listenTo(columns, "change:renderable", this.render);
      var collection = this.collection;
      var fullCollection = collection.fullCollection;
      if (fullCollection) {
        this.listenTo(fullCollection, "add", this.render);
        this.listenTo(fullCollection, "remove", this.render);
        this.listenTo(fullCollection, "reset", this.render);
      }
      else {
        this.listenTo(collection, "add", this.render);
        this.listenTo(collection, "remove", this.render);
        this.listenTo(collection, "reset", this.render);
      }
    },

    /**
       jQuery event handler for the page handlers. Goes to the right page upon
       clicking.

       @param {Event} e
     */
    changePage: function (e) {
      e.preventDefault();

      var label = $(e.target).text();
      var ffLabels = this.fastForwardHandleLabels;

      var collection = this.collection;

      if (ffLabels) {
        switch (label) {
        case ffLabels.first:
          collection.getFirstPage();
          return;
        case ffLabels.prev:
          if (collection.hasPrevious()) collection.getPreviousPage();
          return;
        case ffLabels.next:
          if (collection.hasNext()) collection.getNextPage();
          return;
        case ffLabels.last:
          collection.getLastPage();
          return;
        }
      }

      var state = collection.state;
      var pageIndex = $(e.target).text() * 1 - state.firstPage;
      collection.getPage(state.firstPage === 0 ? pageIndex : pageIndex + 1);
    },

    /**
       Internal method to create a list of page handle objects for the template
       to render them.

       @return Array.<Object> an array of page handle objects hashes
     */
    makeHandles: function () {

      var handles = [];
      var collection = this.collection;
      var state = collection.state;

      // convert all indices to 0-based here
      var lastPage = state.lastPage ? state.lastPage : state.firstPage;
      lastPage = state.firstPage === 0 ? lastPage : lastPage - 1;
      var currentPage = state.firstPage === 0 ? state.currentPage : state.currentPage - 1;
      var windowStart = Math.floor(currentPage / this.windowSize) * this.windowSize;
      var windowEnd = windowStart + this.windowSize;
      windowEnd = windowEnd <= lastPage ? windowEnd : lastPage + 1;

      if (collection.mode !== "infinite") {
        for (var i = windowStart; i < windowEnd; i++) {
          handles.push({
            label: i + 1,
            title: "No. " + (i + 1),
            className: currentPage === i ? "active" : undefined
          });
        }
      }

      var ffLabels = this.fastForwardHandleLabels;
      if (ffLabels) {

        if (ffLabels.prev) {
          handles.unshift({
            label: ffLabels.prev,
            className: collection.hasPrevious() ? void 0 : "disabled"
          });
        }

        if (ffLabels.first) {
          handles.unshift({
            label: ffLabels.first,
            className: collection.hasPrevious() ? void 0 : "disabled"
          });
        }

        if (ffLabels.next) {
          handles.push({
            label: ffLabels.next,
            className: collection.hasNext() ? void 0 : "disabled"
          });
        }

        if (ffLabels.last) {
          handles.push({
            label: ffLabels.last,
            className: collection.hasNext() ? void 0 : "disabled"
          });
        }
      }

      return handles;
    },

    /**
       Render the paginator handles inside an unordered list placed inside a
       cell that spans all the columns.
    */
    render: function () {

      this.$el.empty();

      var colspan = _.reduce(
        this.columns.pluck("renderable"),
        function (accum, renderable) {
          return renderable ? accum + 1 : accum;
        }, 0);

      this.$el.append($(this.template({
        colspan: colspan,
        handles: this.makeHandles()
      })));

      return this;
    }

  });

}(jQuery, _, Backbone, Backgrid));

redpajama_set_name: RedPajamaGithub


text:
package bl.beans;

import java.io.Serializable;
import java.util.Date;

import org.bson.types.ObjectId;
import org.mongodb.morphia.annotations.Id;
import org.mongodb.morphia.annotations.Indexed;

import bl.common.BeanContext;

public class Bean implements BeanContext, Cloneable, Serializable {
	@Id
	ObjectId _id;
	@Indexed
	String name;
	@Indexed
	Date createTime = null;
	@Indexed
	Date modifyTime = null;

	Boolean isDeleted = false;

	/**
	 * @return the name
	 */
	public String getName() {
		return name;
	}

	/**
	 * @param name
	 *            the name to set
	 */
	public void setName(String name) {
		this.name = name;
	}

	/**
	 * @return the _id
	 */
	public ObjectId get_id() {
		return _id;
	}

	/**
	 * @return the _id
	 */
	public String getId() {
		return _id != null ? _id.toString() : "";
	}

	/**
	 * @param _id
	 *            the _id to set
	 */
	public void set_id(ObjectId _id) {
		this._id = _id;
	}

	/**
	 * @param _id
	 *            the _id to set
	 */
	public void setId(String _id) {
		if (_id != null && _id.length() > 0) {
			this._id = new ObjectId(_id);
		}
	}

	/**
	 * @return the createTime
	 */
	public Date getCreateTime() {
		return createTime;
	}

	/**
	 * @param createTime
	 *            the createTime to set
	 */
	public void setCreateTime(Date createTime) {
		this.createTime = createTime;
	}

	/**
	 * @return the modifytime
	 */
	public Date getModifyTime() {
		return modifyTime;
	}

	/**
	 * @param modifyTime
	 *            the modifytime to set
	 */
	public void setModifyTime(Date modifyTime) {
		this.modifyTime = modifyTime;
	}

	/**
	 * Clone (create a deep copy of) this instance.
	 *
	 * @return A clone of this instance.
	 */
	public Object clone() {
		// It's necessary to provide a default clone() method in this base class
		// in
		// order to allow public access to it, because Object.clone() is
		// protected.
		try {
			return super.clone();
		} catch (CloneNotSupportedException cnse) {
			throw new RuntimeException("Error cloning in Bean", cnse);
		}
	}

	@Override
	public boolean equals(Object o) {
		if (this == o)
			return true;
		if (o == null || getClass() != o.getClass())
			return false;

		Bean that = (Bean) o;

		if (this.getId() != null ? !this.getId().equals(that.getId()) : that
				.getId() != null)
			return false;

		return true;
	}

	@Override
	public int hashCode() {
		return this.getId() != null ? this.getId().hashCode() : 0;
	}

	public Boolean getIsDeleted() {
		return isDeleted;
	}

	public void setIsDeleted(Boolean isDeleted) {
		this.isDeleted = isDeleted;
	}

}


text:
require 'minitest'

def Minitest.run args = []
  # PORT: removed
  #self.load_plugins

  options = process_args args

  reporter = Minitest::CompositeReporter.new
  reporter << Minitest::SummaryReporter.new(options[:io], options)
  reporter << Minitest::ProgressReporter.new(options[:io], options)

  self.reporter = reporter # this makes it available to plugins
  # PORT: removed
  #self.init_plugins options
  self.reporter = nil # runnables shouldn't depend on the reporter, ever

  reporter.start
  __run reporter, options
  # PORT: removed
  #self.parallel_executor.shutdown
  reporter.report

  # PORT: modified
  reporter.passed?
  # `window.OPAL_TEST_EXIT_STATUS = #{reporter.passed?}`
end

at_exit { exit Minitest.run }


text:
package org.apache.jena.sparql.engine.join;

import java.util.Iterator;
import java.util.List;

import org.apache.jena.atlas.iterator.Iter;
import org.apache.jena.sparql.algebra.Algebra;
import org.apache.jena.sparql.engine.ExecutionContext;
import org.apache.jena.sparql.engine.QueryIterator;
import org.apache.jena.sparql.engine.binding.Binding;
import org.apache.jena.sparql.engine.iterator.QueryIter2;
import org.apache.jena.sparql.expr.ExprList ;

/**
 * Nested Loop left Join (materializing on the right, streaming on the left)
 * A simple, dependable join.
 * <p>
 * See {@link Join#nestedLoopLeftJoinBasic} for a very simple implementation for
 * testing purposes only.
 */
public class QueryIterNestedLoopLeftJoin extends QueryIter2 {
    // XXX Can we materialise left instead?

    private long s_countLHS     = 0;
    private long s_countRHS     = 0;
    private long s_countResults = 0;

    private final ExprList conditions;
    private final List<Binding> rightRows;
    private Iterator<Binding>   right     = null;
    private QueryIterator       left;
    private Binding             rowLeft = null;
    private boolean foundMatch ;

    private Binding slot     = null;
    private boolean finished = false;

    public QueryIterNestedLoopLeftJoin(QueryIterator left, QueryIterator right, ExprList exprList, ExecutionContext cxt) {
        super(left, right, cxt);
        conditions = exprList ;
        rightRows =  Iter.toList(right);
        s_countRHS = rightRows.size();
        this.left = left;
    }

    @Override
    protected boolean hasNextBinding() {
        if ( finished )
            return false;
        if ( slot == null ) {
            slot = moveToNextBindingOrNull();
            if ( slot == null ) {
                close();
                return false;
            }
        }
        return true;
    }

    @Override
    protected Binding moveToNextBinding() {
        Binding r = slot;
        slot = null;
        return r;
    }

    protected Binding moveToNextBindingOrNull() {
        if ( isFinished() )
            return null;

        for ( ;; ) { // For rows from the left
            if ( rowLeft == null ) {
                if ( left.hasNext() ) {
                    rowLeft = left.next();
                    foundMatch = false ;
                    s_countLHS++;
                    right = rightRows.iterator();
                } else
                    return null;
            }

            while (right.hasNext()) {
                Binding rowRight = right.next();
                Binding r = Algebra.merge(rowLeft, rowRight);
                if ( r != null && applyConditions(r) ) {
                    s_countResults++;
                    foundMatch = true ;
                    return r;
                }
            }
            if ( ! foundMatch ) {
                s_countResults++;
                Binding r = rowLeft ;
                rowLeft = null;
                return r ;
            }
            rowLeft = null;
        }
    }

    private boolean applyConditions(Binding binding) {
        if ( conditions == null )
            return true ;
        return conditions.isSatisfied(binding, getExecContext()) ;
    }

    @Override
    protected void requestSubCancel() {}

    @Override
    protected void closeSubIterator() {
        if ( JoinLib.JOIN_EXPLAIN ) {
            String x = String.format("InnerLoopJoin: LHS=%d RHS=%d Results=%d", s_countLHS, s_countRHS, s_countResults);
            System.out.println(x);
        }
    }
}


text:
namespace Microsoft.Azure.Internal.Subscriptions
{
    using System;
    using System.Collections.Generic;
    using System.Net.Http;
    using System.Threading;
    using System.Threading.Tasks;
    using Newtonsoft.Json;
    using Microsoft.Rest;
    using Microsoft.Rest.Azure;
    using Models;

    /// <summary>
    /// All resource groups and resources exist within subscriptions. These
    /// operation enable you get information about your subscriptions and
    /// tenants. A tenant is a dedicated instance of Azure Active Directory
    /// (Azure AD) for your organization.
    /// </summary>
    public partial interface ISubscriptionClient : IDisposable
    {
        /// <summary>
        /// The base URI of the service.
        /// </summary>
        Uri BaseUri { get; set; }

        /// <summary>
        /// Gets or sets json serialization settings.
        /// </summary>
        JsonSerializerSettings SerializationSettings { get; }

        /// <summary>
        /// Gets or sets json deserialization settings.
        /// </summary>
        JsonSerializerSettings DeserializationSettings { get; }

        /// <summary>
        /// Gets Azure subscription credentials.
        /// </summary>
        ServiceClientCredentials Credentials { get; }

        /// <summary>
        /// The API version to use for the operation.
        /// </summary>
        string ApiVersion { get; }

        /// <summary>
        /// Gets or sets the preferred language for the response.
        /// </summary>
        string AcceptLanguage { get; set; }

        /// <summary>
        /// Gets or sets the retry timeout in seconds for Long Running
        /// Operations. Default value is 30.
        /// </summary>
        int? LongRunningOperationRetryTimeout { get; set; }

        /// <summary>
        /// When set to true a unique x-ms-client-request-id value is
        /// generated and included in each request. Default is true.
        /// </summary>
        bool? GenerateClientRequestId { get; set; }

        /// <summary>
        /// Gets the ISubscriptionsOperations.
        /// </summary>
        ISubscriptionsOperations Subscriptions { get; }

        /// <summary>
        /// Gets the ITenantsOperations.
        /// </summary>
        ITenantsOperations Tenants { get; }

    }
}


text:
PLEASE NOTE: This document applies to the HEAD of the source tree

If you are using a released version of Kubernetes, you should
refer to the docs that go with that version.

The latest 1.0.x release of this document can be found
[here](http://releases.k8s.io/release-1.0/docs/admin/kube-proxy.md).

Documentation for other releases can be found at
[releases.k8s.io](http://releases.k8s.io).

--

## kube-proxy

### Synopsis

The kubernetes network proxy runs on each node. This
reflects services as defined in the Kubernetes API on each node and can do simple
TCP,UDP stream forwarding or round robin TCP,UDP forwarding across a set of backends.
Service cluster ips and ports are currently found through Docker-links-compatible
environment variables specifying ports opened by the service proxy. There is an optional
addon that provides cluster DNS for these cluster IPs. The user must create a service
with the apiserver API to configure the proxy.

### Options

```
      --bind-address=<nil>: The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)
      --healthz-bind-address=<nil>: The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)
      --healthz-port=0: The port to bind the health check server. Use 0 to disable.
  -h, --help=false: help for kube-proxy
      --kubeconfig="": Path to kubeconfig file with authorization information (the master location is set by the master flag).
      --master="": The address of the Kubernetes API server (overrides any value in kubeconfig)
      --oom-score-adj=0: The oom_score_adj value for kube-proxy process. Values must be within the range [-1000, 1000]
      --proxy-port-range=: Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.
      --resource-container="": Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).
```

###### Auto generated by spf13/cobra at 2015-07-06 18:03:37.657112759 +0000 UTC

[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/admin/kube-proxy.md?pixel)]()


text:
/* Select */

var Select = (function () {

	/**
	 * Select constructor
	 * @constructor
	 */
	function Select(element, fallback) {

		this.element = element;
		this.fallback = fallback;

		this.config = {
			innerElementSelector: '.Select-inner',
			valueElementSelector: '.Select-value',
			selectElementSelector: '.Select-select'
		};

		if (this.element)
			this.init();

	}

	Select.prototype.updateValue = function () {

		var self = this;

		try {

			this.value = this.selectElement.value;

			this.valueElement.innerHTML = '';

			var valueTextElement = document.createElement('span');
			valueTextElement.innerText = this.selectElement.value;

			this.valueElement.appendChild(valueTextElement);

			setTimeout(function () {

				if (self.fallback)
					self.fallback(self.value);

			}, 1);

		} catch (e) { }

	};

	/**
	 * Função que adiciona eventos de focus, blur, keydown e change.
	 * A função keydown serve para pegar os eventos de tecla que não
	 * aparecem na hora que o usuário tecla as setas 'down' e 'up'.
	 * A função change serve para trocar o evento.
	 * Todos eventos que trocam o valor do select executam a função 'updateValue'.
	 */
	Select.prototype.addListeners = function () {

		var self = this;

		try {

			this.selectElement.addEventListener('focus', function () {

				self.element.classList.add('is-focus');

			});

			this.selectElement.addEventListener('blur', function () {

				self.element.classList.remove('is-focus');

			});

			this.selectElement.addEventListener('keydown', function () {

				setTimeout(function () {

					self.updateValue();

				}, 1);

			});

			this.selectElement.addEventListener('change', function () {

				self.updateValue();

			});

		} catch (e) {

			console.log(e);

		}

	};

	Select.prototype.getElements = function () {

		this.innerElement = this.element.querySelector(this.config.innerElementSelector);
		this.valueElement = this.element.querySelector(this.config.valueElementSelector);
		this.selectElement = this.element.querySelector(this.config.selectElementSelector);

	};

	Select.prototype.init = function () {

		this.getElements();
		this.addListeners();
		this.updateValue();

	};

	return Select;

})();


text:
package org.docksidestage.remote.fortress.members.index;

import org.lastaflute.core.util.Lato;
import org.lastaflute.web.validation.Required;

/**
 * The bean class as param for remote API of DELETE /members/{memberId}.
 * @author FreeGen
 */
public class RemoteMembersDeleteParam {

    /** The property of memberId. */
    @Required
    public Integer memberId;

    /** The property of memberName. */
    @Required
    public String memberName;

    /** The property of memberAccount. */
    @Required
    public String memberAccount;

    /** The property of birthdate. (NullAllowed) */
    public java.time.LocalDate birthdate;

    /** The property of typeFailureInteger. (NullAllowed) */
    public Integer typeFailureInteger;

    /** The property of memberStatus. (enumValue=[FML, WDL, PRV]) ( * `FML` - Formalized. * `WDL` - Withdrawal. * `PRV` - Provisional. :: fromCls(CDef$MemberStatus)) */
    @Required
    public org.docksidestage.dbflute.allcommon.CDef.MemberStatus memberStatus;

    /** The property of latestLoginDatetime. (NullAllowed) */
    public java.time.LocalDateTime latestLoginDatetime;

    /** The property of updateDatetime. (NullAllowed) */
    public java.time.LocalDateTime updateDatetime;

    /** The property of previousStatus. (enumValue=[FML, WDL, PRV]) ( * `FML` - Formalized. * `WDL` - Withdrawal. * `PRV` - Provisional. :: fromCls(CDef$MemberStatus)) */
    @Required
    public String previousStatus;

    /** The property of versionNo. */
    @Required
    public Long versionNo;

    @Override
    public String toString() {
        return Lato.string(this);
    }
}


text:
#ifndef CPPMATH_LINE2_HPP
#define CPPMATH_LINE2_HPP

#include "Vector.hpp"
#include "Point2.hpp"
#include "../compat.hpp"

/*
 * A 2D Line implementation, based on vectors.
 * p is the starting point, and d is the direction vector.
 * If d is a zero vector, every function has undefined behaviour.
 * A line can have different types:
 *  Line:       Goes to infinity on both ends
 *  Ray:        Goes to infinity in the direction of d
 *  Segment:    Has a fixed size d
 */

namespace math
{
    template <class T>
    class AABB;

    enum LineType
    {
        Line,
        Ray,
        Segment
    };

    enum NormalDirection
    {
        NormalBoth,
        NormalLeft,
        NormalRight
    };

    template <class T>
    class Line2
    {
        public:
            Line2(LineType type_ = Line);
            Line2(const Point2<T>& p_, const Vec2<T>& d_, LineType type_ = Line);
            Line2(const Point2<T>& p1, const Point2<T>& p2, LineType type_ = Segment);

        public:
            auto isParallel(const Line2<T>& line) const -> bool;

            auto closestPoint(const Point2<T>& point, bool clamp = true) const -> Point2<T>;
            auto distance(const Point2<T>& point, bool clamp = true) const     -> T;

            // Only useful for segments
            auto getBBox() const -> AABB<T>;

        public:
            Point2<T> p; // start point
            Vec2<T> d; // direction (not normalized)
            LineType type;
    };

    typedef Line2<float> Line2f;
    typedef Line2<double> Line2d;
    typedef Line2<int> Line2i;
}

#include "Line2.inl"

#endif


text:
package Paws::Config::Source;
  use Moose;
  has Owner => (is => 'ro', isa => 'Str', required => 1);
  has SourceDetails => (is => 'ro', isa => 'ArrayRef[Paws::Config::SourceDetail]');
  has SourceIdentifier => (is => 'ro', isa => 'Str', required => 1);
1;

### main pod documentation begin ###

=head1 NAME

Paws::Config::Source

=head1 USAGE

This class represents one of two things:

=head3 Arguments in a call to a service

Use the attributes of this class as arguments to methods. You shouldn't make instances of this class.
Each attribute should be used as a named argument in the calls that expect this type of object.

As an example, if Att1 is expected to be a Paws::Config::Source object:

  $service_obj->Method(Att1 => { Owner => $value, ..., SourceIdentifier => $value  });

=head3 Results returned from an API call

Use accessors for each attribute. If Att1 is expected to be an Paws::Config::Source object:

  $result = $service_obj->Method(...);
  $result->Att1->Owner

=head1 DESCRIPTION

Provides the AWS Config rule owner (AWS or customer), the rule
identifier, and the events that trigger the evaluation of your AWS
resources.

=head1 ATTRIBUTES

=head2 B<REQUIRED> Owner => Str

  Indicates whether AWS or the customer owns and manages the AWS Config
rule.

=head2 SourceDetails => ArrayRef[L<Paws::Config::SourceDetail>]

  Provides the source and type of the event that causes AWS Config to
evaluate your AWS resources.

=head2 B<REQUIRED> SourceIdentifier => Str

  For AWS Config managed rules, a predefined identifier from a list. For
example, C<IAM_PASSWORD_POLICY> is a managed rule. To reference a
managed rule, see Using AWS Managed Config Rules.

For custom rules, the identifier is the Amazon Resource Name (ARN) of
the rule's AWS Lambda function, such as
C<arn:aws:lambda:us-east-2:123456789012:function:custom_rule_name>.

=head1 SEE ALSO

This class forms part of L<Paws>, describing an object used in L<Paws::Config>

=head1 BUGS and CONTRIBUTIONS

The source code is located here: https://github.com/pplu/aws-sdk-perl

Please report bugs to: https://github.com/pplu/aws-sdk-perl/issues

=cut


text:
set(REQUIRED_PLUGINS
  org.mitk.example.gui.minimalapplication
)

set(DESCRIPTION
"This BlueBerry example plugin demonstrates a minimal implementation of an application consisting only of an empty perspective.

The following features are demonstrated:

 creating a new application

 creating a perspective

See the main documentation for details."
)


text:
package io.druid.query.aggregation;

import io.druid.query.monomorphicprocessing.CalledFromHotLoop;
import io.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import io.druid.segment.FloatColumnSelector;

import java.nio.ByteBuffer;

public abstract class SimpleDoubleBufferAggregator implements BufferAggregator
{
  protected final FloatColumnSelector selector;

  SimpleDoubleBufferAggregator(FloatColumnSelector selector)
  {
    this.selector = selector;
  }

  public FloatColumnSelector getSelector()
  {
    return selector;
  }

  /**
   * Faster equivalent to
   * aggregator.init(buf, position);
   * aggregator.aggregate(buf, position, value);
   */
  @CalledFromHotLoop
  public abstract void putFirst(ByteBuffer buf, int position, double value);

  @CalledFromHotLoop
  public abstract void aggregate(ByteBuffer buf, int position, double value);

  @Override
  public final void aggregate(ByteBuffer buf, int position)
  {
    aggregate(buf, position, (double) selector.get());
  }

  @Override
  public final Object get(ByteBuffer buf, int position)
  {
    return buf.getDouble(position);
  }

  @Override
  public final float getFloat(ByteBuffer buf, int position)
  {
    return (float) buf.getDouble(position);
  }

  @Override
  public final long getLong(ByteBuffer buf, int position)
  {
    return (long) buf.getDouble(position);
  }

  @Override
  public void close()
  {
    // no resources to cleanup
  }

  @Override
  public void inspectRuntimeShape(RuntimeShapeInspector inspector)
  {
    inspector.visit("selector", selector);
  }
}


text:
package nl.knaw.huygens.timbuctoo.v5.serializable.serializations;

import nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.Direction;
import nl.knaw.huygens.timbuctoo.v5.serializable.SerializableResult;
import nl.knaw.huygens.timbuctoo.v5.serializable.dto.Value;

import java.io.IOException;

import static com.google.common.collect.ImmutableMap.of;
import static com.google.common.collect.Lists.newArrayList;
import static nl.knaw.huygens.timbuctoo.v5.serializable.dto.Entity.entity;
import static nl.knaw.huygens.timbuctoo.v5.serializable.dto.PredicateInfo.predicateInfo;
import static nl.knaw.huygens.timbuctoo.v5.serializable.dto.QueryContainer.queryContainer;
import static nl.knaw.huygens.timbuctoo.v5.serializable.dto.SerializableList.serializableList;
import static nl.knaw.huygens.timbuctoo.v5.serializable.dto.Value.fromRawJavaType;

public class SourceData {
  public static SerializableResult simpleResult() throws IOException {
    return new SerializableResult(queryContainer(of(
      "Persons", serializableList(null, null,
        newArrayList(
          entity("http://example.com/1", "http://example.com/Person", of(
            predicateInfo("a", "http://example.org/b"), fromRawJavaType(1), //Note: predicate and mapped name differ!
            predicateInfo("b", "http://example.org/b", Direction.IN), serializableList("next", null,
              entity("http://example.com/11", "http://example.com/SubItem", of(
                predicateInfo("c", "http://example.org/c"), fromRawJavaType("2"),
                predicateInfo("d", "http://example.org/d"),
                  serializableList(null, null, fromRawJavaType("3"), fromRawJavaType("4"), null) //note: Null!
              )),
              entity("http://example.com/12", "http://example.com/SubItem", of(
                predicateInfo("c", "http://example.org/c"), fromRawJavaType("5"),
                predicateInfo("d", "http://example.org/d"),
                  serializableList(null, null, fromRawJavaType("6"), fromRawJavaType("7"))
              ))
            )
          )),
          entity("http://example.com/2", "http://example.com/Person", of(
            predicateInfo("a", "http://example.org/a"), fromRawJavaType("8"),
            predicateInfo("b", "http://example.org/b"), serializableList(null, null,
              entity("http://example.com/11", "http://example.com/SubItem", of(
                predicateInfo("c", "http://example.org/c"), Value.fromRawJavaType("9"),
                predicateInfo("d", "http://example.org/d"),
                  serializableList(null, null, fromRawJavaType(10), fromRawJavaType("11"))
              )),
              entity("http://example.com/12", "http://example.com/SubItem", of(
                predicateInfo("c", "http://example.org/c"), fromRawJavaType("12"),
                predicateInfo("d", "http://example.org/d"),
                  serializableList(null, null, fromRawJavaType(13.0), fromRawJavaType("14")) //note: double!
              )),
              entity("http://example.com/13", "http://example.com/SubItem", of(
                predicateInfo("c", "http://example.org/c"), Value.fromRawJavaType("15"),
                predicateInfo("d", "http://example.org/d"),
                  serializableList(null, null, fromRawJavaType("16"), fromRawJavaType("17"), fromRawJavaType("18"))
              ))
            )
          )),
          entity("http://example.com/3", "http://example.com/Person", of(
            predicateInfo("a", "http://example.org/a"), fromRawJavaType("19"),
            predicateInfo("b", "http://example.org/b"), entity("http://example.com/21", "http://example.com/OtherSubItem", of(
              predicateInfo("e", "http://example.org/e"), fromRawJavaType("20"),
              predicateInfo("f", "http://example.org/f"), fromRawJavaType("21")
            ))
          ))
        )
      )
    )));
  }

}


text:
void Method(const v8::FunctionCallbackInfo<v8::Value>& args) {
  v8::Isolate* isolate = args.GetIsolate();
  args.GetReturnValue().Set(v8::String::NewFromUtf8(
        isolate, "world").ToLocalChecked());
}

void init(v8::Local<v8::Object> exports) {
  NODE_SET_METHOD(exports, "hello", Method);
}

NODE_MODULE(NODE_GYP_MODULE_NAME, init)


text:
<?php

namespace ResultSystems\Storehouse\Product\Repositories;

interface iProductCategoryRepository
{
    public function all(array $with = []);
    public function findById($id);
    public function search(array $request);
    public function store(array $request, $beginTransaction=true);
    public function update($id, array $request);
    public function destroy($ids);
}


text:
package ml.dmlc.xgboost4j.java;

import hex.tree.xgboost.BoosterParms;
import hex.tree.xgboost.XGBoostModel;
import water.*;
import water.nbhm.NonBlockingHashMap;
import water.util.Log;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;

public class XGBoostUpdater extends Thread {

  private static long WORK_START_TIMEOUT_SECS = 5 * 60; // Each Booster task should start before this timer expires
  private static long INACTIVE_CHECK_INTERVAL_SECS = 60;

  private static final NonBlockingHashMap<Key<XGBoostModel>, XGBoostUpdater> updaters = new NonBlockingHashMap<>();

  private final Key<XGBoostModel> _modelKey;
  private final DMatrix _trainMat;
  private final BoosterParms _boosterParms;
  private final Map<String, String> _rabitEnv;

  private volatile SynchronousQueue<BoosterCallable<?>> _in;
  private volatile SynchronousQueue<Object> _out;

  private Booster _booster;

  private XGBoostUpdater(Key<XGBoostModel> modelKey, DMatrix trainMat, BoosterParms boosterParms,
                         Map<String, String> rabitEnv) {
    super("XGBoostUpdater-" + modelKey);
    _modelKey = modelKey;
    _trainMat = trainMat;
    _boosterParms = boosterParms;
    _rabitEnv = rabitEnv;
    _in = new SynchronousQueue<>();
    _out = new SynchronousQueue<>();
  }

  @Override
  public void run() {
    try {
      Rabit.init(_rabitEnv);

      while (! interrupted()) {
        BoosterCallable<?> task = _in.take();
        Object result = task.call();
        _out.put(result);
      }
    } catch (InterruptedException e) {
      XGBoostUpdater self = updaters.get(_modelKey);
      if (self != null) {
        throw new IllegalStateException("Updater thread was interrupted while it was still registered, name=" + self.getName());
      }
    } catch (XGBoostError e) {
      throw new IllegalStateException("XGBoost training iteration failed", e);
    } finally {
      _in = null; // Will throw NPE if used wrong
      _out = null;
      _trainMat.dispose();
      if (_booster != null)
        _booster.dispose();
      updaters.remove(_modelKey);
      try {
        Rabit.shutdown();
      } catch (XGBoostError xgBoostError) {
        Log.warn("Rabit shutdown during update failed", xgBoostError);
      }
    }
  }

  @SuppressWarnings("unchecked")
  private <T> T invoke(BoosterCallable<T> callable) throws InterruptedException {
    final SynchronousQueue<BoosterCallable<?>> inQ = _in;
    if (inQ == null)
      throw new IllegalStateException("Updater is inactive on node " + H2O.SELF);
    if (! inQ.offer(callable, WORK_START_TIMEOUT_SECS, TimeUnit.SECONDS))
      throw new IllegalStateException("XGBoostUpdater couldn't start work on task " + callable + " in "  + WORK_START_TIMEOUT_SECS + "s.");
    SynchronousQueue<?> outQ;
    int i = 0;
    while ((outQ = _out) != null) {
      i++;
      T result = (T) outQ.poll(INACTIVE_CHECK_INTERVAL_SECS, TimeUnit.SECONDS);
      if (result != null) {
        return result;
      } else if (i > 5) {
        Log.warn(String.format("Exceeded waiting interval of %d seconds for a task of type '%s' to finish on node '%s'. ",
                INACTIVE_CHECK_INTERVAL_SECS * i, callable, H2O.SELF));
      }
    }
    throw new IllegalStateException("Cannot perform booster operation: updater is inactive on node " + H2O.SELF);
  }

  private class UpdateBooster implements BoosterCallable<Booster> {
    private final int _tid;

    private UpdateBooster(int tid) { _tid = tid; }

    @Override
    public Booster call() throws XGBoostError {
      if ((_booster == null) && _tid == 0) {
        HashMap<String, DMatrix> watches = new HashMap<>();
        // Create empty Booster
        _booster = ml.dmlc.xgboost4j.java.XGBoost.train(_trainMat,
                _boosterParms.get(),
                0,
                watches,
                null,
                null);
        // Force Booster initialization; we can call any method that does "lazy init"
        byte[] boosterBytes = _booster.toByteArray();
        Log.info("Initial (0 tree) Booster created, size=" + boosterBytes.length);
      } else {
        // Do one iteration
        assert _booster != null;
        _booster.update(_trainMat, _tid);
      }
      return _booster;
    }

    @Override
    public String toString() {
      return "Boosting Iteration (tid=" + _tid + ")";
    }
  }

  private class SerializeBooster implements BoosterCallable<byte[]> {
    @Override
    public byte[] call() throws XGBoostError {
      return _booster.toByteArray();
    }
    @Override
    public String toString() {
      return "SerializeBooster";
    }
  }

  Booster getBooster() {
    return _booster;
  }

  byte[] getBoosterBytes() {
    try {
      return invoke(new SerializeBooster());
    } catch (InterruptedException e) {
      throw new IllegalStateException("Failed to serialize Booster - operation was interrupted", e);
    }
  }

  Booster doUpdate(int tid) {
    try {
      return invoke(new UpdateBooster(tid));
    } catch (InterruptedException e) {
      throw new IllegalStateException("Boosting iteration failed - operation was interrupted", e);
    }
  }

  static XGBoostUpdater make(Key<XGBoostModel> modelKey, DMatrix trainMat, BoosterParms boosterParms,
                             Map<String, String> rabitEnv) {
    XGBoostUpdater updater = new XGBoostUpdater(modelKey, trainMat, boosterParms, rabitEnv);
    if (updaters.putIfAbsent(modelKey, updater) != null)
      throw new IllegalStateException("XGBoostUpdater for modelKey=" + modelKey + " already exists!");
    return updater;
  }

  static void terminate(Key<XGBoostModel> modelKey) {
    XGBoostUpdater updater = updaters.remove(modelKey);
    if (updater == null)
      Log.debug("XGBoostUpdater for modelKey=" + modelKey + " was already clean-up on node " + H2O.SELF);
    else
      updater.interrupt();
  }

  static XGBoostUpdater getUpdater(Key<XGBoostModel> modelKey) {
    XGBoostUpdater updater = updaters.get(modelKey);
    if (updater == null) {
      throw new IllegalStateException("XGBoostUpdater for modelKey=" + modelKey + " was not found!");
    }
    return updater;
  }

  private interface BoosterCallable<E> {
    E call() throws XGBoostError;
  }

}


text:
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE topic
  PUBLIC "-//OASIS//DTD DITA Composite//EN" "ditabase.dtd">
<topic id="topic1" xml:lang="en">
  <title id="nk110126">About This Guide</title>

This guide describes system and database administration tasks for Greenplum Database. The
      guide consists of five sections:

<xref href="intro/partI.xml#topic1" type="topic" format="dita"/> describes
        Greenplum Database architecture and components. It introduces administration topics such as
        mirroring, parallel data loading, and Greenplum management and monitoring utilities.

<xref href="managing/partII.xml#topic1" type="topic" format="dita"/>
        contains information about everyday Greenplum Database system administration tasks. Topics
        include starting and stopping the server, client front-ends to access the database,
        configuring Greenplum, enabling high availability features, backing up and restoring
        databases, expanding the system by adding nodes, monitoring the system, and regular
        maintenance tasks.

<xref href="partIII.xml#topic1" type="topic" format="dita"/> covers
        configuring Greenplum Database authentication, managing roles and privileges, and setting up
        Kerberos athentication.

<xref href="partIV.xml#topic1" type="topic" format="dita"/> contains
        information about creating and managing databases, schemas, tables and other database
        objects. It describes how to view database metadata, insert, update, and delete data in
        tables, load data from external files, and run queries in a database.

<xref href="./partV.xml#topic1" type="topic" format="dita"/> describes how
        to monitor and manage system performance. It discusses how to define performance in a
        parallel environment, how to diagnose performance problems, workload and resource
        administration, and performance troubleshooting.

This guide assumes knowledge of Linux/UNIX system administration and database management
      systems. Familiarity with structured query language (SQL) is helpful.

Because Greenplum Database is based on PostgreSQL 8.3.23, this guide assumes some familiarity
      with PostgreSQL. References to <xref
        href="https://www.postgresql.org/docs/8.3/static/index.html" scope="external" format="html"
          ><ph>PostgreSQL documentation</ph></xref> are provided throughout this guide for features
      that are similar to those in Greenplum Database.

This guide provides information for system administrators responsible for administering a
      Greenplum Database system.

        <xref href="#topic3" type="topic" format="dita"/>

        <xref href="#topic4" type="topic" format="dita"/>

        <xref href="#topic7" type="topic" format="dita"/>

  <topic id="topic3" xml:lang="en">
    <title>About the Greenplum Database Documentation Set</title>

The Greenplum Database 4.3 documentation set consists of the following guides.

        <title>Greenplum Database documentation set</title>
        <tgroup cols="2">
          <colspec colnum="1" colname="col1" colwidth="130pt"/>
          <colspec colnum="2" colname="col2" colwidth="243pt"/>

            <row>
              <entry colname="col1">Guide Name</entry>
              <entry colname="col2">Description</entry>
            </row>

            <row>
              <entry colname="col1">Greenplum Database Administrator Guide</entry>
              <entry colname="col2">Describes the Greenplum Database architecture and concepts such
                as parallel processing, and system administration and database administration tasks
                for Greenplum Database. System administraiton topics include configuring the server,
                monitoring system activity, enabling high-availability, backing up and restoring
                databases, and expanding the system. Database administration topics include creating
                databases and database objects, loading and manipulating data, writing queries, and
                monitoring and managing database performance. </entry>
            </row>
            <row>
              <entry colname="col1">Greenplum Database Reference Guide</entry>
              <entry colname="col2">Reference information for Greenplum Database systems: SQL
                commands, system catalogs, environment variables, character set support, datatypes,
                the Greenplum MapReduce specification, postGIS extension, server parameters, the
                gp_toolkit administrative schema, and SQL 2008 support.</entry>
            </row>
            <row>
              <entry colname="col1">Greenplum Database Utility Guide</entry>
              <entry colname="col2">Reference information for command-line utilities, client
                programs, and Oracle compatibility functions.</entry>
            </row>
            <row>
              <entry colname="col1">Greenplum Database Installation Guide</entry>
              <entry colname="col2">Information and instructions for installing and initializing a
                Greenplum Database system.</entry>
            </row>

        </tgroup>

  </topic>
  <topic id="topic4" xml:lang="en">
    <title id="nk164282">Document Conventions</title>

The following conventions are used throughout the Greenplum Database documentation to help
        you identify certain types of information.

          <xref href="#topic5" type="topic" format="dita"/>

          <xref href="#topic6" type="topic" format="dita"/>

    <topic id="topic5" xml:lang="en">
      <title id="nk164370">Text Conventions</title>

          <title>Text Conventions</title>
          <tgroup cols="3">
            <colspec colnum="1" colname="col1" colwidth="110pt"/>
            <colspec colnum="2" colname="col2" colwidth="165pt"/>
            <colspec colnum="3" colname="col3" colwidth="174pt"/>

              <row>
                <entry colname="col1">Text Convention</entry>
                <entry colname="col2">Usage</entry>
                <entry colname="col3">Examples</entry>
              </row>

              <row>
                <entry colname="col1">
                  bold
                </entry>
                <entry colname="col2">
                  <ph>Button, menu, tab, page, and field names in GUI applications</ph>
                </entry>
                <entry colname="col3">
                  <ph>Click </ph>
                  Cancel
                  <ph> to exit the page without saving your changes.</ph>
                </entry>
              </row>
              <row>
                <entry colname="col1">
                  italics
                </entry>
                <entry colname="col2">
                  <ph>New terms where they are defined</ph>

                    <ph>Database objects, such as schema, table, or column names</ph>

                </entry>
                <entry colname="col3">
                  <ph>The </ph>
                  master instance
                  <ph>is the </ph>
                  <codeph>postgres</codeph>
                  <ph> process that accepts client connections.</ph>

                    <ph>Catalog information for Greenplum Database resides in the </ph>
                    pg_catalog
                    <ph> schema.</ph>

                </entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>monospace</codeph>
                </entry>
                <entry colname="col2">
                  <ph>File names and path names</ph>

                    <ph>Programs and executables</ph>

                    <ph>Command names and syntax</ph>

                    <ph>Parameter names</ph>

                </entry>
                <entry colname="col3">
                  <ph>Edit the </ph>
                  <codeph>postgresql.conf</codeph>
                  <ph> file.</ph>

Use <codeph>gpstart</codeph> to start Greenplum Database.

                </entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>
                    monospace italics
                  </codeph>
                </entry>
                <entry colname="col2">
                  <ph>Variable information within file paths and file names</ph>

                    <ph>Variable information within command syntax</ph>

                </entry>
                <entry colname="col3"
                      ><codeph>/home/</codeph><codeph>gpadmin</codeph>/<codeph>config_file</codeph>
<codeph>COPY
                        </codeph><codeph>tablename</codeph><codeph> FROM
                        '</codeph><codeph>filename</codeph><codeph>'</codeph>
</entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>
                    monospace bold
                  </codeph>
                </entry>
                <entry colname="col2">Used to call attention to a particular part of a command,
                  parameter, or code snippet.</entry>
                <entry colname="col3">Change the host name, port, and database name in the JDBC
                  connection
                      URL:
<codeph>jdbc:postgresql://</codeph><codeph>host</codeph><codeph>:</codeph><codeph>5432</codeph><codeph>/</codeph><codeph>mydb</codeph>
</entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>UPPERCASE</codeph>
                </entry>
                <entry colname="col2">
                  <ph>Environment variables</ph>

                    <ph>SQL commands</ph>

                    <ph>Keyboard keys</ph>

                </entry>
                <entry colname="col3">
                  <ph>Make sure that the Java </ph>
                  <codeph>/bin</codeph>
                  <ph> directory is in your </ph>
                  <codeph>$PATH</codeph>
                  <ph>. </ph>

                    <codeph>SELECT * FROM </codeph>
                    <codeph>my_table</codeph>
                    <codeph>;</codeph>

                    <ph>Press </ph>
                    <codeph>CTRL+C</codeph>
                    <ph> to escape.</ph>

                </entry>
              </row>

          </tgroup>

    </topic>
    <topic id="topic6" xml:lang="en">
      <title id="nk164424">Command Syntax Conventions</title>

          <title>Command Syntax Conventions</title>
          <tgroup cols="3">
            <colspec colnum="1" colname="col1" colwidth="124pt"/>
            <colspec colnum="2" colname="col2" colwidth="162pt"/>
            <colspec colnum="3" colname="col3" colwidth="162pt"/>

              <row>
                <entry colname="col1">Text Convention</entry>
                <entry colname="col2">Usage</entry>
                <entry colname="col3">Examples</entry>
              </row>

              <row>
                <entry colname="col1">
                  <codeph>{ }</codeph>
                </entry>
                <entry colname="col2">Within command syntax, curly braces group related command
                  options. Do not type the curly braces.</entry>
                <entry colname="col3">
                  <codeph>FROM </codeph>
                  <codeph>{</codeph>
                  <codeph> '</codeph>
                  <codeph>filename</codeph>
                  <codeph>' | STDIN </codeph>
                  <codeph>}</codeph>
                </entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>[ ]</codeph>
                </entry>
                <entry colname="col2">Within command syntax, square brackets denote optional
                  arguments. Do not type the brackets.</entry>
                <entry colname="col3">
                  <codeph>TRUNCATE </codeph>
                  <codeph>[</codeph>
                  <codeph> TABLE </codeph>
                  <codeph>]</codeph>
                  <codeph>name</codeph>
                </entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>...</codeph>
                </entry>
                <entry colname="col2">Within command syntax, an ellipsis denotes repetition of a
                  command, variable, or option. Do not type the ellipsis.</entry>
                <entry colname="col3">
                  <codeph>DROP TABLE </codeph>
                  <codeph>name</codeph>
                  <codeph> [,</codeph>
                  <codeph> ...</codeph>
                  <codeph>]</codeph>
                </entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>|</codeph>
                </entry>
                <entry colname="col2">Within command syntax, the pipe symbol denotes an "OR"
                  relationship. Do not type the pipe symbol.</entry>
                <entry colname="col3">
                  <codeph>VACUUM [ FULL </codeph>
                  <codeph>|</codeph>
                  <codeph> FREEZE ]</codeph>
                </entry>
              </row>
              <row>
                <entry colname="col1">
                  <codeph>$ </codeph>
                  <ph>system_command</ph>

                    <codeph># </codeph>
                    <ph>root_system_command</ph>

                    <codeph>=> </codeph>
                    <ph>gpdb_command</ph>

                    <codeph>=# </codeph>
                    <ph>su_gpdb_command</ph>

                </entry>
                <entry colname="col2">Denotes a command prompt - do not type the prompt symbol.
                    <codeph>$</codeph> and <codeph>#</codeph> denote terminal command prompts.
                    <codeph>=></codeph> and <codeph>=#</codeph> denote Greenplum Database
                  interactive program command prompts (<codeph>psql</codeph> or
                    <codeph>gpssh</codeph>, for example).</entry>
                <entry colname="col3">
                  <ph>$</ph>
                  <codeph> createdb mydatabase</codeph>

                    <ph>#</ph>
                    <codeph> chown gpadmin -R /datadir</codeph>

                    <ph>=></ph>
                    <codeph> SELECT * FROM mytable;</codeph>

                    <ph>=#</ph>
                    <codeph> SELECT * FROM pg_database;</codeph>

                </entry>
              </row>

          </tgroup>

    </topic>
  </topic>
  <topic id="topic7" xml:lang="en">
    <title id="nk164426">Getting Support</title>

For technical support, documentation, release notes, software updates, or for information
          about Pivotal products, licensing, and services, go to <xref
            href="https://www.pivotal.io/" scope="external" format="html"
              ><ph>www.pivotal.io</ph></xref>.

    </topic>
  </topic>


text:
package com.shapesecurity.bandolier.es2018.bundlers;

import com.shapesecurity.bandolier.es2018.ModuleWrapper;
import com.shapesecurity.bandolier.es2018.transformations.DeadCodeElimination;
import com.shapesecurity.bandolier.es2018.transformations.ImportExportConnector;
import com.shapesecurity.bandolier.es2018.transformations.VariableCollisionResolver;
import com.shapesecurity.functional.Pair;
import com.shapesecurity.functional.data.HashTable;
import com.shapesecurity.functional.data.ImmutableList;
import com.shapesecurity.functional.data.Maybe;
import com.shapesecurity.shift.es2018.ast.BindingIdentifier;
import com.shapesecurity.shift.es2018.ast.CallExpression;
import com.shapesecurity.shift.es2018.ast.ClassDeclaration;
import com.shapesecurity.shift.es2018.ast.Directive;
import com.shapesecurity.shift.es2018.ast.Export;
import com.shapesecurity.shift.es2018.ast.ExpressionStatement;
import com.shapesecurity.shift.es2018.ast.FormalParameters;
import com.shapesecurity.shift.es2018.ast.FunctionBody;
import com.shapesecurity.shift.es2018.ast.FunctionDeclaration;
import com.shapesecurity.shift.es2018.ast.FunctionExpression;
import com.shapesecurity.shift.es2018.ast.Module;
import com.shapesecurity.shift.es2018.ast.Script;
import com.shapesecurity.shift.es2018.ast.ThisExpression;
import com.shapesecurity.shift.es2018.ast.VariableDeclarationStatement;
import com.shapesecurity.shift.es2018.parser.EarlyError;
import com.shapesecurity.shift.es2018.parser.EarlyErrorChecker;

import javax.annotation.Nonnull;
import java.util.Map;
import java.util.stream.Collectors;

// merges modules by resolving variable collision, scheduling, and merging modules.
public class PiercedModuleBundler implements IModuleBundler {

	@Override
	@Nonnull
	public Script bundleEntrypoint(BundlerOptions options, String entry, Map<String, Module> modules) {
		HashTable<String, ModuleWrapper> newModules = HashTable.emptyUsingEquality();
		for (Map.Entry<String, Module> mapEntry : modules.entrySet()) {
			Module module = mapEntry.getValue();
			if (options.exportStrategy == BundlerOptions.ExportStrategy.ALL_GLOBALS) {
				module = new Module(module.directives, module.items.map(item -> {
					if (item instanceof VariableDeclarationStatement) {
						return new Export(((VariableDeclarationStatement) item).declaration);
					} else if (item instanceof FunctionDeclaration) {
						return new Export((FunctionDeclaration) item);
					} else if (item instanceof ClassDeclaration) {
						return new Export((ClassDeclaration) item);
					}
					return item;
				}));
			}
			newModules = newModules.put(mapEntry.getKey(), new ModuleWrapper(module));
		}
		VariableCollisionResolver.ResolvedResult result = VariableCollisionResolver.resolveCollisions(newModules);
		HashTable<String, ModuleWrapper> specifierToModule = newModules.map(module -> result.moduleMap.get(module).fromJust());
		Pair<Script, String> scriptAndGlobalParameter = ImportExportConnector.combineModules(options, result.moduleMap.get(newModules.get(entry).fromJust()).fromJust(), result, specifierToModule);
		Script combined = scriptAndGlobalParameter.left;
		combined = DeadCodeElimination.removeAllUnusedDeclarations(combined);
		return new Script(ImmutableList.empty(), ImmutableList.of(
				new ExpressionStatement(new CallExpression(
						new FunctionExpression(false, false, Maybe.empty(), new FormalParameters(ImmutableList.of(new BindingIdentifier(scriptAndGlobalParameter.right)), Maybe.empty()),
								new FunctionBody(ImmutableList.of(new Directive("use strict")), combined.statements)
						),
						ImmutableList.of(new ThisExpression())
				))
		));
	}

	@Override
	@Nonnull
	public Pair<Script, ImmutableList<EarlyError>> bundleEntrypointWithEarlyErrors(BundlerOptions options, String entry, Map<String, Module> modules) {
		return Pair.of(bundleEntrypoint(options, entry, modules), ImmutableList.from(modules.values().stream().map(EarlyErrorChecker::validate).collect(Collectors.toList())).foldLeft(ImmutableList::append, ImmutableList.empty()));
	}
}


text:
namespace blink {

TEST(TextTrackListTest, InvalidateTrackIndexes) {
  // Create and fill the list
  auto* list = MakeGarbageCollected<TextTrackList>(
      MakeGarbageCollected<HTMLVideoElement>(
          std::make_unique<DummyPageHolder>()->GetDocument()));
  const size_t kNumTextTracks = 4;
  TextTrack* text_tracks[kNumTextTracks];
  for (size_t i = 0; i < kNumTextTracks; ++i) {
    text_tracks[i] =
        MakeGarbageCollected<TextTrack>("subtitles", "", "", *list->Owner());
    list->Append(text_tracks[i]);
  }

  EXPECT_EQ(4u, list->length());
  EXPECT_EQ(0, text_tracks[0]->TrackIndex());
  EXPECT_EQ(1, text_tracks[1]->TrackIndex());
  EXPECT_EQ(2, text_tracks[2]->TrackIndex());
  EXPECT_EQ(3, text_tracks[3]->TrackIndex());

  // Remove element from the middle of the list
  list->Remove(text_tracks[1]);

  EXPECT_EQ(3u, list->length());
  EXPECT_EQ(nullptr, text_tracks[1]->TrackList());
  EXPECT_EQ(0, text_tracks[0]->TrackIndex());
  EXPECT_EQ(1, text_tracks[2]->TrackIndex());
  EXPECT_EQ(2, text_tracks[3]->TrackIndex());
}

}  // namespace blink


text:
//---------------------------------------------------------------------
// <copyright file="NamedStreamResourceAttribute.cs" company="Microsoft">
//      Copyright (C) Microsoft Corporation. All rights reserved. See License.txt in the project root for license information.
// </copyright>
//---------------------------------------------------------------------

namespace System.Data.Test.Astoria
{
    using System.Xml;
    using System.Collections.Generic;
    using System.Linq;

    public class NamedStreamResourceAttribute : ResourceAttribute
    {
        private ResourceType type;
        private string name;

        public NamedStreamResourceAttribute(ResourceType type, string name)
            : base("NamedStream")
        {
            this.type = type;
            this.name = name;
            this.orderedParams = new List<string>() { '"' + name + '"' };
            type.Facets.NamedStreams.Add(name);

            if (type.HasDerivedTypes)
            {
                foreach (ResourceType derived in type.DerivedTypes)
                {
                    derived.Facets.NamedStreams.Add(name);
                }
            }
        }

        //---------------------------------------------------------------------
        // Modifies CSDL.
        //---------------------------------------------------------------------
        public override void Apply(XmlDocument csdl)
        {
            throw new NotSupportedException("Named streams aren't supported on EF in the old framework");
        }
    }
}


text:
package org.datanucleus.tests;

import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.assertj.core.api.Assertions.assertThat;

import java.beans.Introspector;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Stream;

import javax.jdo.JDOException;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
import javax.jdo.Transaction;

import org.datanucleus.samples.types.basic.BasicTypeHolder;
import org.datanucleus.samples.types.basic.DecimalHolder;
import org.datanucleus.samples.types.basic.FloatHolder;
import org.junit.Test;

public class JDOQLAvgTest extends JDOPersistenceTestCase
{
    @Test(expected = JDOException.class)
    public void testNoArg()
    {
        PersistenceManager pm = pmf.getPersistenceManager();
        Transaction tx = pm.currentTransaction();

        try
        {
            Query query = pm.newQuery(BasicTypeHolder.class);
            query.setResult("AVG()");

            tx.begin();
            query.execute();
        }
        finally
        {
            if (tx.isActive())
            {
                tx.rollback();
            }
            pm.close();
        }
    }

    @Test(expected = JDOException.class)
    public void testMultipleArgs()
    {
        PersistenceManager pm = pmf.getPersistenceManager();
        Transaction tx = pm.currentTransaction();

        try
        {
            Query query = pm.newQuery(BasicTypeHolder.class);
            query.setResult("AVG(shortField,intField)");

            tx.begin();
            query.execute();
        }
        finally
        {
            if (tx.isActive())
            {
                tx.rollback();
            }
            pm.close();
        }
    }

    public void testAvgOnIntegralTypes() throws Exception
    {
        testAvgTypes(this::newBasicTypeHolderWith, 7.5, 10, 5);
    }

    public void testAvgOnFloatingPointTypes() throws Exception
    {
        testAvgTypes(this::newFloatHolderWith, 7.5, 10, 5);
    }

    public void testAvgOnArbritaryPrecisionTypes() throws Exception
    {
        testAvgTypes(this::newDecimalHolderWith, 7.5, 10, 5);
    }

    public void testAvgTypesSubquery() throws Exception
    {
        PersistenceManager pm = pmf.getPersistenceManager();
        Transaction tx = pm.currentTransaction();

        // Create sample holders for each value that will avg to a decimal
        BasicTypeHolder holder1 = newBasicTypeHolderWith(10);
        BasicTypeHolder holder2 = newBasicTypeHolderWith(5);
        holder1.setCharField('a');
        holder2.setCharField('a');

        // Additional entry that to be filtered by the where clause
        BasicTypeHolder holder3 = newBasicTypeHolderWith(3);
        holder3.setCharField('b');

        try
        {
            tx.begin();
            pm.makePersistentAll(holder1, holder2);
            tx.commit();

            // Do a query for each numeric field to check all the types
            for (String field : extractNumericFields(holder1))
            {
                Query query = pm.newQuery(
                        "SELECT charField FROM org.datanucleus.samples.types.basic.BasicTypeHolder WHERE avg("
                                + field
                                + ") == 7.5 GROUP BY charField ");

                tx.begin();

                List<Character> result = (List<Character>) query.execute();
                assertThat(result).as("Should filter based on AVG decimal value").containsExactly('a');

                tx.commit();
            }
        }
        finally
        {
            if (tx.isActive())
            {
                tx.rollback();
            }

            pm.close();

            clean(BasicTypeHolder.class);
        }
    }

    private <T extends TestObject> void testAvgTypes(Function<Integer, T> holderFactory, double expectedResult, Integer... values)
        throws Exception
    {
        PersistenceManager pm = pmf.getPersistenceManager();
        Transaction tx = pm.currentTransaction();

        // Create sample holders for each value
        List<T> holders = Stream.of(values).map(holderFactory).collect(toList());
        T sampleHolder = holders.get(0);
        Class<?> holderClass = sampleHolder.getClass();

        try
        {
            tx.begin();
            pm.makePersistentAll(holders);
            tx.commit();

            Set<String> fields = extractNumericFields(sampleHolder);

            // Create the result query: avg(fieldName1),avg(fieldName2)...
            String avgResultQuery = fields.stream()
                    .map(fieldName -> "AVG(" + fieldName + ")")
                    .collect(joining(","));

            tx.begin();

            // Query the avg for the given fields on the holderClass
            Query query = pm.newQuery(holderClass);
            query.setResult(avgResultQuery);
            Object[] result = (Object[]) query.execute();

            assertThat(result)
                    .as("Avg return type for numeric values should be Double")
                    .hasSameSizeAs(fields)
                    .containsOnly(expectedResult)
                    .extracting("class")
                    .containsOnly(Double.class);

            tx.commit();
        }
        finally
        {
            if (tx.isActive())
            {
                tx.rollback();
            }

            pm.close();

            // Clean out our data
            clean(holderClass);
        }
    }

    private <T extends TestObject> Set<String> extractNumericFields(T holder) throws Exception
    {
        return Stream.of(Introspector.getBeanInfo(holder.getClass()).getPropertyDescriptors())
                .filter(pd -> pd.getName().matches("(?i).*(short|int|long|float|double|integer|decimal).*"))
                .map(pd -> pd.getName())
                .collect(toSet());
    }

    private DecimalHolder newDecimalHolderWith(int value)
    {
        DecimalHolder decimalHolder = new DecimalHolder();
        decimalHolder.setBigIntegerField(BigInteger.valueOf(value));
        decimalHolder.setBigDecimalField(new BigDecimal(value));

        return decimalHolder;
    }

    private FloatHolder newFloatHolderWith(int value)
    {
        FloatHolder floatHolder = new FloatHolder();
        floatHolder.setFloatField(value);
        floatHolder.setFloatObjField(Float.valueOf(value));
        floatHolder.setDoubleField(value);
        floatHolder.setDoubleObjField((double) value);

        return floatHolder;
    }

    private BasicTypeHolder newBasicTypeHolderWith(int value)
    {
        short shortValue = (short) value;

        BasicTypeHolder typeHolder = new BasicTypeHolder();
        typeHolder.setCharField((char)value);
        typeHolder.setShortField(shortValue);
        typeHolder.setShortObjField(shortValue);
        typeHolder.setIntField(value);
        typeHolder.setIntObjField(value);
        typeHolder.setLongField(value);
        typeHolder.setLongObjField((long) value);

        return typeHolder;
    }
}