text:
<?php

namespace Unimatrix\Cake\Model\Behavior;

use Cake\Database\Type;
use Cake\Event\Event;
use Cake\Filesystem\File;
use Cake\Filesystem\Folder;
use Cake\ORM\Behavior;
use Cake\ORM\Entity;
use Cake\Utility\Text;
use Unimatrix\Cake\Database\Type\FileType;
use Unimatrix\Cake\Validation\UploadValidation;
use RuntimeException;
use ArrayObject;

/**
 * Uploadable
 * Easy file upload for each defined field
 *
 * Configuration:
 * -------------------------------------------------------
 * Add this behavior in the model table:
 * $this->addBehavior('Unimatrix/Cake.Uploadable', [
 *     'fields' => [
 *         'file' => 'img/:model/:uuid'
 *     ]
 * ]);
 *
 * Field identifiers:
 * -------------------------------------------------------
 * :model: The model name
 * :uuid: A random and unique identifier UUID type
 * :md5: A random and unique identifier with 32 characters.
 *
 * Validation:
 * ---------------------------------------------------------------------------------
 * $validator
 *     ->requirePresence('file', 'create')
 *     ->allowEmpty('file', 'update');
 *
 * @author Flavius
 * @version 1.2
 */
class UploadableBehavior extends Behavior
{
    /**
     * Default config.
     * @var array
     */
    protected $_defaultConfig = [
        'root' => WWW_ROOT,
        'fields' => []
    ];

    /**
     * Check if configuration is set
     * @throws RuntimeException
     */
    private function check() {
        if(empty($this->_config['fields']))
            throw new RuntimeException('Configuration missing for UploadableBehavior');
    }

    /**
     * Build the behaviour
     * @param array $config Passed configuration
     * @return void
     */
    public function initialize(array $config) {
        // configuration set?
        $this->check();

        // load the file type & schema
        Type::map('unimatrix.file', FileType::class);
        $schema = $this->_table->getSchema();

        // go through each field and change the column type to our file type
        foreach($this->_config['fields'] as $field => $path)
            $schema->setColumnType($field, 'unimatrix.file');

        // update schema
        $this->_table->setSchema($schema);
    }

    /**
     * If a field is allowed to be empty as defined in the validation it should be unset to prevent processing
     * @param \Cake\Event\Event $event Event instance
     * @param ArrayObject $data Data to process
     * @return void
     */
    public function beforeMarshal(Event $event, ArrayObject $data, ArrayObject $options) {
        // configuration set?
        $this->check();

        // load validator and add our custom upload validator
        $validator = $this->_table->getValidator();
        $validator->setProvider('upload', UploadValidation::class);

        // go through each field
        foreach($this->_config['fields'] as $field => $path) {
            // add validators
            $validator->add($field, [
                'isUnderPhpSizeLimit' => ['rule' => 'isUnderPhpSizeLimit', 'provider' => 'upload', 'message' => __d('Unimatrix/cake', 'This file is too large')],
                'isUnderFormSizeLimit' => ['rule' => 'isUnderFormSizeLimit', 'provider' => 'upload', 'message' => __d('Unimatrix/cake', 'This file is too large')],
                'isCompletedUpload' => ['rule' => 'isCompletedUpload', 'provider' => 'upload', 'message' => __d('Unimatrix/cake', 'This file was only partially uploaded')],
                'isTemporaryDirectory' => ['rule' => 'isTemporaryDirectory', 'provider' => 'upload', 'message' => __d('Unimatrix/cake', 'Missing a temporary folder')],
                'isSuccessfulWrite' => ['rule' => 'isSuccessfulWrite', 'provider' => 'upload', 'message' => __d('Unimatrix/cake', 'Failed to write file to disk')],
                'isNotStoppedByExtension' => ['rule' => 'isNotStoppedByExtension', 'provider' => 'upload', 'message' => __d('Unimatrix/cake', 'Upload was stopped by extension')],
            ]);

            // empty allowed? && no file uploaded? unset field
            if($validator->isEmptyAllowed($field, false)
                && isset($data[$field]['error']) && $data[$field]['error'] === UPLOAD_ERR_NO_FILE)
                    unset($data[$field]);
        }
    }

    /**
     * beforeSave handle
     * @param \Cake\Event\Event  $event The beforeSave event that was fired.
     * @param \Cake\ORM\Entity $entity The entity that is going to be saved.
     * @return void
     */
    public function beforeSave(Event $event, Entity $entity) {
        // configuration set?
        $this->check();

        // go through each field
        foreach($this->_config['fields'] as $field => $path) {
            // get uploaded file info
            $file = $entity->get($field);

            // no file or not array? problem...
            if(!($file && is_array($entity[$field])))
                continue;

            // file not ok? skip
            if($file['error'] !== UPLOAD_ERR_OK) {
                $entity->set($field, '');
                continue;
            }

            // get the final name
            $final = $this->formatFile($entity, $file, $path);

            // create the folder
            $folder = new Folder($this->_config['root']);
            $folder->create($this->_config['root'] . dirname($final));

            // copy file, delete old file and set new file in entity
            $file = new File($file['tmp_name']);
            if($file->copy($this->_config['root'] . $final)) {
                $this->deleteFile($entity->extractOriginalChanged([$field])[$field] ?? null);
                $entity->set($field, '/' . str_replace(DS, '/', $final));
            } else $entity->set($field, '');
        }
    }

    /**
     * afterDelete handle
     * @param \Cake\Event\Event  $event The beforeSave event that was fired.
     * @param \Cake\ORM\Entity $entity The entity that is going to be saved.
     * @return void
     */
    public function afterDelete(Event $event, Entity $entity) {
        // configuration set?
        $this->check();

        // delete existing files linked to this entity
        foreach($this->_config['fields'] as $field => $path)
            $this->deleteFile($entity->get($field));
    }

    /**
     * Delete a file
     * @param string $file
     * @return boolean
     */
    private function deleteFile($file = null) {
        // no file
        if(!$file || is_array($file))
            return false;

        // get a file and delete it
        $file = new File($this->_config['root'] . trim(str_replace('/', DS, $file), DS));
        if($file->exists())
            return $file->delete();

        // nothing found
        return false;
    }

    /**
     * Get the path formatted without its identifiers to upload the file.
     * @param \Cake\ORM\Entity $entity The entity that is going to be saved.
     * @param array $file The file array
     * @param string $path The path
     * @return string
     */
    private function formatFile(Entity $entity, $file = [], $path = false) {
        // get extension & path
        $ext = (new File($file['name']))->ext();
        $path = trim(str_replace(['/', '\\'], DS, $path), DS);

        // handle identifiers
        $identifiers = [
            ':model' => strtolower($entity->getSource()),
            ':uuid' => Text::uuid(),
            ':md5' => md5(rand() . uniqid() . time()),
        ];

        // output
        return strtr($path, $identifiers) . '.' . strtolower($ext);
    }
}


text:
"""Ceph Backup Service Implementation.

This driver supports backing up volumes of any type to a Ceph object store. It
is also capable of detecting whether the volume to be backed up is a Ceph RBD
volume and, if so, attempts to perform incremental/differential backups.

Support is also included for the following in the case of a source volume being
a Ceph RBD volume:

    * backing up within the same Ceph pool (not recommended)
    * backing up between different Ceph pools
    * backing up between different Ceph clusters

At the time of writing, differential backup support in Ceph/librbd was quite
new so this driver accounts for this by first attempting differential backup
and falling back to full backup/copy if the former fails. It is recommended
that you upgrade to Ceph Dumpling (>= v0.67) or above to get the best results.

If incremental backups are used, multiple backups of the same volume are stored
as snapshots so that minimal space is consumed in the object store and
restoring the volume takes a far reduced amount of time compared to a full
copy.

Note that Cinder supports restoring to a new volume or the original volume the
backup was taken from. For the latter case, a full copy is enforced since this
was deemed the safest action to take. It is therefore recommended to always
restore to a new volume (default).
"""

import fcntl
import os
import re
import subprocess
import time

import eventlet
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import units
from six.moves import range

from cinder.backup import driver
from cinder import exception
from cinder.i18n import _, _LE, _LI, _LW
from cinder import utils
import cinder.volume.drivers.rbd as rbd_driver

try:
    import rados
    import rbd
except ImportError:
    rados = None
    rbd = None

LOG = logging.getLogger(__name__)

service_opts = [
    cfg.StrOpt('backup_ceph_conf', default='/etc/ceph/ceph.conf',
               help='Ceph configuration file to use.'),
    cfg.StrOpt('backup_ceph_user', default='cinder',
               help='The Ceph user to connect with. Default here is to use '
                    'the same user as for Cinder volumes. If not using cephx '
                    'this should be set to None.'),
    cfg.IntOpt('backup_ceph_chunk_size', default=(units.Mi * 128),
               help='The chunk size, in bytes, that a backup is broken into '
                    'before transfer to the Ceph object store.'),
    cfg.StrOpt('backup_ceph_pool', default='backups',
               help='The Ceph pool where volume backups are stored.'),
    cfg.IntOpt('backup_ceph_stripe_unit', default=0,
               help='RBD stripe unit to use when creating a backup image.'),
    cfg.IntOpt('backup_ceph_stripe_count', default=0,
               help='RBD stripe count to use when creating a backup image.'),
    cfg.BoolOpt('restore_discard_excess_bytes', default=True,
                help='If True, always discard excess bytes when restoring '
                     'volumes i.e. pad with zeroes.')
]

CONF = cfg.CONF
CONF.register_opts(service_opts)

class VolumeMetadataBackup(object):

    def __init__(self, client, backup_id):
        self._client = client
        self._backup_id = backup_id

    @property
    def name(self):
        return utils.convert_str("backup.%s.meta" % self._backup_id)

    @property
    def exists(self):
        meta_obj = rados.Object(self._client.ioctx, self.name)
        return self._exists(meta_obj)

    def _exists(self, obj):
        try:
            obj.stat()
        except rados.ObjectNotFound:
            return False
        else:
            return True

    def set(self, json_meta):
        """Write JSON metadata to a new object.

        This should only be called once per backup. Raises
        VolumeMetadataBackupExists if the object already exists.
        """
        meta_obj = rados.Object(self._client.ioctx, self.name)
        if self._exists(meta_obj):
            msg = _("Metadata backup object '%s' already exists") % self.name
            raise exception.VolumeMetadataBackupExists(msg)

        meta_obj.write(json_meta)

    def get(self):
        """Get metadata backup object.

        Returns None if the object does not exist.
        """
        meta_obj = rados.Object(self._client.ioctx, self.name)
        if not self._exists(meta_obj):
            LOG.debug("Metadata backup object %s does not exist", self.name)
            return None

        return meta_obj.read()

    def remove_if_exists(self):
        meta_obj = rados.Object(self._client.ioctx, self.name)
        try:
            meta_obj.remove()
        except rados.ObjectNotFound:
            LOG.debug("Metadata backup object '%s' not found - ignoring",
                      self.name)

class CephBackupDriver(driver.BackupDriver):
    """Backup Cinder volumes to Ceph Object Store.

    This class enables backing up Cinder volumes to a Ceph object store.
    Backups may be stored in their own pool or even cluster. Store location is
    defined by the Ceph conf file and service config options supplied.

    If the source volume is itself an RBD volume, the backup will be performed
    using incremental differential backups which *should* give a performance
    gain.
    """

    def __init__(self, context, db_driver=None, execute=None):
        super(CephBackupDriver, self).__init__(context, db_driver)
        self.rbd = rbd
        self.rados = rados
        self.chunk_size = CONF.backup_ceph_chunk_size
        self._execute = execute or utils.execute

        if self._supports_stripingv2:
            self.rbd_stripe_unit = CONF.backup_ceph_stripe_unit
            self.rbd_stripe_count = CONF.backup_ceph_stripe_count
        else:
            LOG.info(_LI("RBD striping not supported - ignoring configuration "
                         "settings for rbd striping"))
            self.rbd_stripe_count = 0
            self.rbd_stripe_unit = 0

        self._ceph_backup_user = utils.convert_str(CONF.backup_ceph_user)
        self._ceph_backup_pool = utils.convert_str(CONF.backup_ceph_pool)
        self._ceph_backup_conf = utils.convert_str(CONF.backup_ceph_conf)

    def _validate_string_args(self, *args):
        """Ensure all args are non-None and non-empty."""
        return all(args)

    def _ceph_args(self, user, conf=None, pool=None):
        """Create default ceph args for executing rbd commands.

        If no --conf is provided, rbd will look in the default locations e.g.
        /etc/ceph/ceph.conf
        """

        # Make sure user arg is valid since rbd command may not fail if
        # invalid/no user provided, resulting in unexpected behaviour.
        if not self._validate_string_args(user):
            raise exception.BackupInvalidCephArgs(_("invalid user '%s'") %
                                                  user)

        args = ['--id', user]
        if conf:
            args.extend(['--conf', conf])
        if pool:
            args.extend(['--pool', pool])

        return args

    @property
    def _supports_layering(self):
        """Determine if copy-on-write is supported by our version of librbd."""
        return hasattr(self.rbd, 'RBD_FEATURE_LAYERING')

    @property
    def _supports_stripingv2(self):
        """Determine if striping is supported by our version of librbd."""
        return hasattr(self.rbd, 'RBD_FEATURE_STRIPINGV2')

    def _get_rbd_support(self):
        """Determine RBD features supported by our version of librbd."""
        old_format = True
        features = 0
        if self._supports_layering:
            old_format = False
            features |= self.rbd.RBD_FEATURE_LAYERING
        if self._supports_stripingv2:
            old_format = False
            features |= self.rbd.RBD_FEATURE_STRIPINGV2

        return (old_format, features)

    def _connect_to_rados(self, pool=None):
        """Establish connection to the backup Ceph cluster."""
        client = self.rados.Rados(rados_id=self._ceph_backup_user,
                                  conffile=self._ceph_backup_conf)
        try:
            client.connect()
            pool_to_open = utils.convert_str(pool or self._ceph_backup_pool)
            ioctx = client.open_ioctx(pool_to_open)
            return client, ioctx
        except self.rados.Error:
            # shutdown cannot raise an exception
            client.shutdown()
            raise

    def _disconnect_from_rados(self, client, ioctx):
        """Terminate connection with the backup Ceph cluster."""
        # closing an ioctx cannot raise an exception
        ioctx.close()
        client.shutdown()

    def _get_backup_base_name(self, volume_id, backup_id=None,
                              diff_format=False):
        """Return name of base image used for backup.

        Incremental backups use a new base name so we support old and new style
        format.
        """
        # Ensure no unicode
        if diff_format:
            return utils.convert_str("volume-%s.backup.base" % volume_id)
        else:
            if backup_id is None:
                msg = _("Backup id required")
                raise exception.InvalidParameterValue(msg)
            return utils.convert_str("volume-%s.backup.%s"
                                     % (volume_id, backup_id))

    def _discard_bytes(self, volume, offset, length):
        """Trim length bytes from offset.

        If the volume is an rbd do a discard() otherwise assume it is a file
        and pad with zeroes.
        """
        if length:
            LOG.debug("Discarding %(length)s bytes from offset %(offset)s",
                      {'length': length, 'offset': offset})
            if self._file_is_rbd(volume):
                volume.rbd_image.discard(offset, length)
            else:
                zeroes = '\0' * length
                chunks = int(length / self.chunk_size)
                for chunk in range(0, chunks):
                    LOG.debug("Writing zeroes chunk %d", chunk)
                    volume.write(zeroes)
                    volume.flush()
                    # yield to any other pending backups
                    eventlet.sleep(0)

                rem = int(length % self.chunk_size)
                if rem:
                    zeroes = '\0' * rem
                    volume.write(zeroes)
                    volume.flush()

    def _transfer_data(self, src, src_name, dest, dest_name, length):
        """Transfer data between files (Python IO objects)."""
        LOG.debug("Transferring data between '%(src)s' and '%(dest)s'",
                  {'src': src_name, 'dest': dest_name})

        chunks = int(length / self.chunk_size)
        LOG.debug("%(chunks)s chunks of %(bytes)s bytes to be transferred",
                  {'chunks': chunks, 'bytes': self.chunk_size})

        for chunk in range(0, chunks):
            before = time.time()
            data = src.read(self.chunk_size)
            # If we have reach end of source, discard any extraneous bytes from
            # destination volume if trim is enabled and stop writing.
            if data == '':
                if CONF.restore_discard_excess_bytes:
                    self._discard_bytes(dest, dest.tell(),
                                        length - dest.tell())

                return

            dest.write(data)
            dest.flush()
            delta = (time.time() - before)
            rate = (self.chunk_size / delta) / 1024
            LOG.debug("Transferred chunk %(chunk)s of %(chunks)s "
                      "(%(rate)dK/s)",
                      {'chunk': chunk + 1,
                       'chunks': chunks,
                       'rate': rate})

            # yield to any other pending backups
            eventlet.sleep(0)

        rem = int(length % self.chunk_size)
        if rem:
            LOG.debug("Transferring remaining %s bytes", rem)
            data = src.read(rem)
            if data == '':
                if CONF.restore_discard_excess_bytes:
                    self._discard_bytes(dest, dest.tell(), rem)
            else:
                dest.write(data)
                dest.flush()
                # yield to any other pending backups
                eventlet.sleep(0)

    def _create_base_image(self, name, size, rados_client):
        """Create a base backup image.

        This will be the base image used for storing differential exports.
        """
        LOG.debug("Creating base image '%s'", name)
        old_format, features = self._get_rbd_support()
        self.rbd.RBD().create(ioctx=rados_client.ioctx,
                              name=name,
                              size=size,
                              old_format=old_format,
                              features=features,
                              stripe_unit=self.rbd_stripe_unit,
                              stripe_count=self.rbd_stripe_count)

    def _delete_backup_snapshot(self, rados_client, base_name, backup_id):
        """Delete snapshot associated with this backup if one exists.

        A backup should have at most ONE associated snapshot.

        This is required before attempting to delete the base image. The
        snapshot on the original volume can be left as it will be purged when
        the volume is deleted.

        Returns tuple(deleted_snap_name, num_of_remaining_snaps).
        """
        remaining_snaps = 0
        base_rbd = self.rbd.Image(rados_client.ioctx, base_name)
        try:
            snap_name = self._get_backup_snap_name(base_rbd, base_name,
                                                   backup_id)
            if snap_name:
                LOG.debug("Deleting backup snapshot='%s'", snap_name)
                base_rbd.remove_snap(snap_name)
            else:
                LOG.debug("No backup snapshot to delete")

            # Now check whether any snapshots remain on the base image
            backup_snaps = self.get_backup_snaps(base_rbd)
            if backup_snaps:
                remaining_snaps = len(backup_snaps)
        finally:
            base_rbd.close()

        return snap_name, remaining_snaps

    def _try_delete_base_image(self, backup_id, volume_id, base_name=None):
        """Try to delete backup RBD image.

        If the rbd image is a base image for incremental backups, it may have
        snapshots. Delete the snapshot associated with backup_id and if the
        image has no more snapshots, delete it. Otherwise return.

        If no base name is provided try normal (full) format then diff format
        image name.

        If a base name is provided but does not exist, ImageNotFound will be
        raised.

        If the image is busy, a number of retries will be performed if
        ImageBusy is received, after which the exception will be propagated to
        the caller.
        """
        retries = 3
        delay = 5
        try_diff_format = False

        if base_name is None:
            try_diff_format = True

            base_name = self._get_backup_base_name(volume_id, backup_id)
            LOG.debug("Trying diff format basename='%(basename)s' for "
                      "backup base image of volume %(volume)s.",
                      {'basename': base_name, 'volume': volume_id})

        with rbd_driver.RADOSClient(self) as client:
            rbd_exists, base_name = \
                self._rbd_image_exists(base_name, volume_id, client,
                                       try_diff_format=try_diff_format)
            if not rbd_exists:
                raise self.rbd.ImageNotFound(_("image %s not found") %
                                             base_name)

            while retries >= 0:
                # First delete associated snapshot from base image (if exists)
                snap, rem = self._delete_backup_snapshot(client, base_name,
                                                         backup_id)
                if rem:
                    LOG.info(
                        _LI("Backup base image of volume %(volume)s still "
                            "has %(snapshots)s snapshots so skipping base "
                            "image delete."),
                        {'snapshots': rem, 'volume': volume_id})
                    return

                LOG.info(_LI("Deleting backup base image='%(basename)s' of "
                             "volume %(volume)s."),
                         {'basename': base_name, 'volume': volume_id})
                # Delete base if no more snapshots
                try:
                    self.rbd.RBD().remove(client.ioctx, base_name)
                except self.rbd.ImageBusy:
                    # Allow a retry if the image is busy
                    if retries > 0:
                        LOG.info(_LI("Backup image of volume %(volume)s is "
                                     "busy, retrying %(retries)s more time(s) "
                                     "in %(delay)ss."),
                                 {'retries': retries,
                                  'delay': delay,
                                  'volume': volume_id})
                        eventlet.sleep(delay)
                    else:
                        LOG.error(_LE("Max retries reached deleting backup "
                                      "%(basename)s image of volume "
                                      "%(volume)s."),
                                  {'volume': volume_id,
                                   'basename': base_name})
                        raise
                else:
                    LOG.debug("Base backup image='%(basename)s' of volume "
                              "%(volume)s deleted.",
                              {'basename': base_name, 'volume': volume_id})
                    retries = 0
                finally:
                    retries -= 1

            # Since we have deleted the base image we can delete the source
            # volume backup snapshot.
            src_name = utils.convert_str(volume_id)
            if src_name in self.rbd.RBD().list(client.ioctx):
                LOG.debug("Deleting source volume snapshot '%(snapshot)s' "
                          "for backup %(basename)s.",
                          {'snapshot': snap, 'basename': base_name})
                src_rbd = self.rbd.Image(client.ioctx, src_name)
                try:
                    src_rbd.remove_snap(snap)
                finally:
                    src_rbd.close()

    def _piped_execute(self, cmd1, cmd2):
        """Pipe output of cmd1 into cmd2."""
        LOG.debug("Piping cmd1='%s' into...", ' '.join(cmd1))
        LOG.debug("cmd2='%s'", ' '.join(cmd2))

        try:
            p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
        except OSError as e:
            LOG.error(_LE("Pipe1 failed - %s "), e)
            raise

        # NOTE(dosaboy): ensure that the pipe is blocking. This is to work
        # around the case where evenlet.green.subprocess is used which seems to
        # use a non-blocking pipe.
        flags = fcntl.fcntl(p1.stdout, fcntl.F_GETFL) & (~os.O_NONBLOCK)
        fcntl.fcntl(p1.stdout, fcntl.F_SETFL, flags)

        try:
            p2 = subprocess.Popen(cmd2, stdin=p1.stdout,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE)
        except OSError as e:
            LOG.error(_LE("Pipe2 failed - %s "), e)
            raise

        p1.stdout.close()
        stdout, stderr = p2.communicate()
        return p2.returncode, stderr

    def _rbd_diff_transfer(self, src_name, src_pool, dest_name, dest_pool,
                           src_user, src_conf, dest_user, dest_conf,
                           src_snap=None, from_snap=None):
        """Copy only extents changed between two points.

        If no snapshot is provided, the diff extents will be all those changed
        since the rbd volume/base was created, otherwise it will be those
        changed since the snapshot was created.
        """
        LOG.debug("Performing differential transfer from '%(src)s' to "
                  "'%(dest)s'",
                  {'src': src_name, 'dest': dest_name})

        # NOTE(dosaboy): Need to be tolerant of clusters/clients that do
        # not support these operations since at the time of writing they
        # were very new.

        src_ceph_args = self._ceph_args(src_user, src_conf, pool=src_pool)
        dest_ceph_args = self._ceph_args(dest_user, dest_conf, pool=dest_pool)

        cmd1 = ['rbd', 'export-diff'] + src_ceph_args
        if from_snap is not None:
            cmd1.extend(['--from-snap', from_snap])
        if src_snap:
            path = utils.convert_str("%s/%s@%s"
                                     % (src_pool, src_name, src_snap))
        else:
            path = utils.convert_str("%s/%s" % (src_pool, src_name))
        cmd1.extend([path, '-'])

        cmd2 = ['rbd', 'import-diff'] + dest_ceph_args
        rbd_path = utils.convert_str("%s/%s" % (dest_pool, dest_name))
        cmd2.extend(['-', rbd_path])

        ret, stderr = self._piped_execute(cmd1, cmd2)
        if ret:
            msg = (_("RBD diff op failed - (ret=%(ret)s stderr=%(stderr)s)") %
                   {'ret': ret, 'stderr': stderr})
            LOG.info(msg)
            raise exception.BackupRBDOperationFailed(msg)

    def _rbd_image_exists(self, name, volume_id, client,
                          try_diff_format=False):
        """Return tuple (exists, name)."""
        rbds = self.rbd.RBD().list(client.ioctx)
        if name not in rbds:
            LOG.debug("Image '%s' not found - trying diff format name", name)
            if try_diff_format:
                name = self._get_backup_base_name(volume_id, diff_format=True)
                if name not in rbds:
                    LOG.debug("Diff format image '%s' not found", name)
                    return False, name
            else:
                return False, name

        return True, name

    def _snap_exists(self, base_name, snap_name, client):
        """Return True if snapshot exists in base image."""
        base_rbd = self.rbd.Image(client.ioctx, base_name, read_only=True)
        try:
            snaps = base_rbd.list_snaps()
        finally:
            base_rbd.close()

        if snaps is None:
            return False

        for snap in snaps:
            if snap['name'] == snap_name:
                return True

        return False

    def _backup_rbd(self, backup_id, volume_id, volume_file, volume_name,
                    length):
        """Create an incremental backup from an RBD image."""
        rbd_user = volume_file.rbd_user
        rbd_pool = volume_file.rbd_pool
        rbd_conf = volume_file.rbd_conf
        source_rbd_image = volume_file.rbd_image

        # Identify our --from-snap point (if one exists)
        from_snap = self._get_most_recent_snap(source_rbd_image)
        LOG.debug("Using --from-snap '%(snap)s' for incremental backup of "
                  "volume %(volume)s.",
                  {'snap': from_snap, 'volume': volume_id})

        base_name = self._get_backup_base_name(volume_id, diff_format=True)
        image_created = False
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            # If from_snap does not exist at the destination (and the
            # destination exists), this implies a previous backup has failed.
            # In this case we will force a full backup.
            #
            # TODO(dosaboy): find a way to repair the broken backup
            #
            if base_name not in self.rbd.RBD().list(ioctx=client.ioctx):
                # If a from_snap is defined but the base does not exist, we
                # ignore it since it is stale and waiting to be cleaned up.
                if from_snap:
                    LOG.debug("Source snapshot '%(snapshot)s' of volume "
                              "%(volume)s is stale so deleting.",
                              {'snapshot': from_snap, 'volume': volume_id})
                    source_rbd_image.remove_snap(from_snap)
                    from_snap = None

                # Create new base image
                self._create_base_image(base_name, length, client)
                image_created = True
            else:
                # If a from_snap is defined but does not exist in the back base
                # then we cannot proceed (see above)
                if not self._snap_exists(base_name, from_snap, client):
                    errmsg = (_("Snapshot='%(snap)s' does not exist in base "
                                "image='%(base)s' - aborting incremental "
                                "backup") %
                              {'snap': from_snap, 'base': base_name})
                    LOG.info(errmsg)
                    # Raise this exception so that caller can try another
                    # approach
                    raise exception.BackupRBDOperationFailed(errmsg)

        # Snapshot source volume so that we have a new point-in-time
        new_snap = self._get_new_snap_name(backup_id)
        LOG.debug("Creating backup snapshot='%s'", new_snap)
        source_rbd_image.create_snap(new_snap)

        # Attempt differential backup. If this fails, perhaps because librbd
        # or Ceph cluster version does not support it, do a full backup
        # instead.
        #
        # TODO(dosaboy): find a way to determine if the operation is supported
        #                rather than brute force approach.
        try:
            before = time.time()
            self._rbd_diff_transfer(volume_name, rbd_pool, base_name,
                                    self._ceph_backup_pool,
                                    src_user=rbd_user,
                                    src_conf=rbd_conf,
                                    dest_user=self._ceph_backup_user,
                                    dest_conf=self._ceph_backup_conf,
                                    src_snap=new_snap,
                                    from_snap=from_snap)

            LOG.debug("Differential backup transfer completed in %.4fs",
                      (time.time() - before))

            # We don't need the previous snapshot (if there was one) anymore so
            # delete it.
            if from_snap:
                source_rbd_image.remove_snap(from_snap)

        except exception.BackupRBDOperationFailed:
            with excutils.save_and_reraise_exception():
                LOG.debug("Differential backup transfer failed")

                # Clean up if image was created as part of this operation
                if image_created:
                    self._try_delete_base_image(backup_id, volume_id,
                                                base_name=base_name)

                # Delete snapshot
                LOG.debug("Deleting diff backup snapshot='%(snapshot)s' of "
                          "source volume='%(volume)s'.",
                          {'snapshot': new_snap, 'volume': volume_id})
                source_rbd_image.remove_snap(new_snap)

    def _file_is_rbd(self, volume_file):
        """Returns True if the volume_file is actually an RBD image."""
        return hasattr(volume_file, 'rbd_image')

    def _full_backup(self, backup_id, volume_id, src_volume, src_name, length):
        """Perform a full backup of src volume.

        First creates a base backup image in our backup location then performs
        an chunked copy of all data from source volume to a new backup rbd
        image.
        """
        backup_name = self._get_backup_base_name(volume_id, backup_id)

        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            # First create base backup image
            old_format, features = self._get_rbd_support()
            LOG.debug("Creating backup base image='%(name)s' for volume "
                      "%(volume)s.",
                      {'name': backup_name, 'volume': volume_id})
            self.rbd.RBD().create(ioctx=client.ioctx,
                                  name=backup_name,
                                  size=length,
                                  old_format=old_format,
                                  features=features,
                                  stripe_unit=self.rbd_stripe_unit,
                                  stripe_count=self.rbd_stripe_count)

            LOG.debug("Copying data from volume %s.", volume_id)
            dest_rbd = self.rbd.Image(client.ioctx, backup_name)
            try:
                rbd_meta = rbd_driver.RBDImageMetadata(dest_rbd,
                                                       self._ceph_backup_pool,
                                                       self._ceph_backup_user,
                                                       self._ceph_backup_conf)
                rbd_fd = rbd_driver.RBDImageIOWrapper(rbd_meta)
                self._transfer_data(src_volume, src_name, rbd_fd, backup_name,
                                    length)
            finally:
                dest_rbd.close()

    @staticmethod
    def backup_snapshot_name_pattern():
        """Returns the pattern used to match backup snapshots.

        It is essential that snapshots created for purposes other than backups
        do not have this name format.
        """
        return r"^backup\.([a-z0-9\-]+?)\.snap\.(.+)$"

    @classmethod
    def get_backup_snaps(cls, rbd_image, sort=False):
        """Get all backup snapshots for the given rbd image.

        NOTE: this call is made public since these snapshots must be deleted
              before the base volume can be deleted.
        """
        snaps = rbd_image.list_snaps()

        backup_snaps = []
        for snap in snaps:
            search_key = cls.backup_snapshot_name_pattern()
            result = re.search(search_key, snap['name'])
            if result:
                backup_snaps.append({'name': result.group(0),
                                     'backup_id': result.group(1),
                                     'timestamp': result.group(2)})

        if sort:
            # Sort into ascending order of timestamp
            backup_snaps.sort(key=lambda x: x['timestamp'], reverse=True)

        return backup_snaps

    def _get_new_snap_name(self, backup_id):
        return utils.convert_str("backup.%s.snap.%s"
                                 % (backup_id, time.time()))

    def _get_backup_snap_name(self, rbd_image, name, backup_id):
        """Return the name of the snapshot associated with backup_id.

        The rbd image provided must be the base image used for an incremental
        backup.

        A backup is only allowed ONE associated snapshot. If more are found,
        exception.BackupOperationError is raised.
        """
        snaps = self.get_backup_snaps(rbd_image)

        LOG.debug("Looking for snapshot of backup base '%s'", name)

        if not snaps:
            LOG.debug("Backup base '%s' has no snapshots", name)
            return None

        snaps = [snap['name'] for snap in snaps
                 if snap['backup_id'] == backup_id]

        if not snaps:
            LOG.debug("Backup '%s' has no snapshot", backup_id)
            return None

        if len(snaps) > 1:
            msg = (_("Backup should only have one snapshot but instead has %s")
                   % len(snaps))
            LOG.error(msg)
            raise exception.BackupOperationError(msg)

        LOG.debug("Found snapshot '%s'", snaps[0])
        return snaps[0]

    def _get_most_recent_snap(self, rbd_image):
        """Get the most recent backup snapshot of the provided image.

        Returns name of most recent backup snapshot or None if there are no
        backup snapshots.
        """
        backup_snaps = self.get_backup_snaps(rbd_image, sort=True)
        if not backup_snaps:
            return None

        return backup_snaps[0]['name']

    def _get_volume_size_gb(self, volume):
        """Return the size in gigabytes of the given volume.

        Raises exception.InvalidParameterValue if volume size is 0.
        """
        if int(volume['size']) == 0:
            errmsg = _("Need non-zero volume size")
            raise exception.InvalidParameterValue(errmsg)

        return int(volume['size']) * units.Gi

    def _backup_metadata(self, backup):
        """Backup volume metadata.

        NOTE(dosaboy): the metadata we are backing up is obtained from a
                       versioned api so we should not alter it in any way here.
                       We must also be sure that the service that will perform
                       the restore is compatible with version used.
        """
        json_meta = self.get_metadata(backup['volume_id'])
        if not json_meta:
            LOG.debug("No metadata to backup for volume %s.",
                      backup['volume_id'])
            return

        LOG.debug("Backing up metadata for volume %s.",
                  backup['volume_id'])
        try:
            with rbd_driver.RADOSClient(self) as client:
                vol_meta_backup = VolumeMetadataBackup(client, backup['id'])
                vol_meta_backup.set(json_meta)
        except exception.VolumeMetadataBackupExists as e:
            msg = (_("Failed to backup volume metadata - %s") % e)
            raise exception.BackupOperationError(msg)

    def backup(self, backup, volume_file, backup_metadata=True):
        """Backup volume and metadata (if available) to Ceph object store.

        If the source volume is an RBD we will attempt to do an
        incremental/differential backup, otherwise a full copy is performed.
        If this fails we will attempt to fall back to full copy.
        """
        backup_id = backup['id']
        volume = self.db.volume_get(self.context, backup['volume_id'])
        volume_id = volume['id']
        volume_name = volume['name']

        LOG.debug("Starting backup of volume='%s'.", volume_id)

        # Ensure we are at the beginning of the volume
        volume_file.seek(0)
        length = self._get_volume_size_gb(volume)

        do_full_backup = False
        if self._file_is_rbd(volume_file):
            # If volume an RBD, attempt incremental backup.
            try:
                self._backup_rbd(backup_id, volume_id, volume_file,
                                 volume_name, length)
            except exception.BackupRBDOperationFailed:
                LOG.debug("Forcing full backup of volume %s.", volume_id)
                do_full_backup = True
        else:
            do_full_backup = True

        if do_full_backup:
            self._full_backup(backup_id, volume_id, volume_file,
                              volume_name, length)

        backup.container = self._ceph_backup_pool
        backup.save()

        if backup_metadata:
            try:
                self._backup_metadata(backup)
            except exception.BackupOperationError:
                with excutils.save_and_reraise_exception():
                    # Cleanup.
                    self.delete(backup)

        LOG.debug("Backup '%(backup_id)s' of volume %(volume_id)s finished.",
                  {'backup_id': backup_id, 'volume_id': volume_id})

    def _full_restore(self, backup_id, volume_id, dest_file, dest_name,
                      length, src_snap=None):
        """Restore volume using full copy i.e. all extents.

        This will result in all extents being copied from source to
        destination.
        """
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            # If a source snapshot is provided we assume the base is diff
            # format.
            if src_snap:
                diff_format = True
            else:
                diff_format = False

            backup_name = self._get_backup_base_name(volume_id,
                                                     backup_id=backup_id,
                                                     diff_format=diff_format)

            # Retrieve backup volume
            src_rbd = self.rbd.Image(client.ioctx, backup_name,
                                     snapshot=src_snap, read_only=True)
            try:
                rbd_meta = rbd_driver.RBDImageMetadata(src_rbd,
                                                       self._ceph_backup_pool,
                                                       self._ceph_backup_user,
                                                       self._ceph_backup_conf)
                rbd_fd = rbd_driver.RBDImageIOWrapper(rbd_meta)
                self._transfer_data(rbd_fd, backup_name, dest_file, dest_name,
                                    length)
            finally:
                src_rbd.close()

    def _check_restore_vol_size(self, backup_base, restore_vol, restore_length,
                                src_pool):
        """Ensure that the restore volume is the correct size.

        If the restore volume was bigger than the backup, the diff restore will
        shrink it to the size of the original backup so we need to
        post-process and resize it back to its expected size.
        """
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            adjust_size = 0
            base_image = self.rbd.Image(client.ioctx,
                                        utils.convert_str(backup_base),
                                        read_only=True)
            try:
                if restore_length != base_image.size():
                    adjust_size = restore_length
            finally:
                base_image.close()

        if adjust_size:
            with rbd_driver.RADOSClient(self, src_pool) as client:
                restore_vol_encode = utils.convert_str(restore_vol)
                dest_image = self.rbd.Image(client.ioctx, restore_vol_encode)
                try:
                    LOG.debug("Adjusting restore vol size")
                    dest_image.resize(adjust_size)
                finally:
                    dest_image.close()

    def _diff_restore_rbd(self, base_name, restore_file, restore_name,
                          restore_point, restore_length):
        """Attempt restore rbd volume from backup using diff transfer."""
        rbd_user = restore_file.rbd_user
        rbd_pool = restore_file.rbd_pool
        rbd_conf = restore_file.rbd_conf

        LOG.debug("Attempting incremental restore from base='%(base)s' "
                  "snap='%(snap)s'",
                  {'base': base_name, 'snap': restore_point})
        before = time.time()
        try:
            self._rbd_diff_transfer(base_name, self._ceph_backup_pool,
                                    restore_name, rbd_pool,
                                    src_user=self._ceph_backup_user,
                                    src_conf=self._ceph_backup_conf,
                                    dest_user=rbd_user, dest_conf=rbd_conf,
                                    src_snap=restore_point)
        except exception.BackupRBDOperationFailed:
            LOG.exception(_LE("Differential restore failed, trying full "
                              "restore"))
            raise

        # If the volume we are restoring to is larger than the backup volume,
        # we will need to resize it after the diff import since import-diff
        # appears to shrink the target rbd volume to the size of the original
        # backup volume.
        self._check_restore_vol_size(base_name, restore_name, restore_length,
                                     rbd_pool)

        LOG.debug("Restore transfer completed in %.4fs",
                  (time.time() - before))

    def _num_backup_snaps(self, backup_base_name):
        """Return the number of snapshots that exist on the base image."""
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            base_rbd = self.rbd.Image(client.ioctx, backup_base_name,
                                      read_only=True)
            try:
                snaps = self.get_backup_snaps(base_rbd)
            finally:
                base_rbd.close()

        if snaps:
            return len(snaps)
        else:
            return 0

    def _get_restore_point(self, base_name, backup_id):
        """Get restore point snapshot name for incremental backup.

        If the backup was not incremental (determined by the fact that the
        base has no snapshots/restore points), None is returned. Otherwise, the
        restore point associated with backup_id is returned.
        """
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            base_rbd = self.rbd.Image(client.ioctx, base_name, read_only=True)
            try:
                restore_point = self._get_backup_snap_name(base_rbd, base_name,
                                                           backup_id)
            finally:
                base_rbd.close()

        return restore_point

    def _rbd_has_extents(self, rbd_volume):
        """Check whether the given rbd volume has extents.

        Return True if has extents, otherwise False.
        """
        extents = []

        def iter_cb(offset, length, exists):
            if exists:
                extents.append(length)

        rbd_volume.diff_iterate(0, rbd_volume.size(), None, iter_cb)

        if extents:
            LOG.debug("RBD has %s extents", sum(extents))
            return True

        return False

    def _diff_restore_allowed(self, base_name, backup, volume, volume_file,
                              rados_client):
        """Determine if differential restore is possible and restore point.

        Determine whether a differential restore is possible/allowed,
        and find out the restore point if backup base is diff-format.

        In order for a differential restore to be performed we need:
            * destination volume must be RBD
            * destination volume must have zero extents
            * backup base image must exist
            * backup must have a restore point
            * target volume is different from source volume of backup

        Returns True if differential restore is allowed, False otherwise.
        Return the restore point if back base is diff-format.
        """
        # NOTE(dosaboy): base_name here must be diff format.
        rbd_exists, base_name = self._rbd_image_exists(base_name,
                                                       backup['volume_id'],
                                                       rados_client)

        if not rbd_exists:
            return False, None

        # Get the restore point. If no restore point is found, we assume
        # that the backup was not performed using diff/incremental methods
        # so we enforce full copy.
        restore_point = self._get_restore_point(base_name, backup['id'])

        if restore_point:
            if self._file_is_rbd(volume_file):

                # If the volume we are restoring to is the volume the backup
                # was made from, force a full restore since a diff will not
                # work in this case.
                if volume['id'] == backup['volume_id']:
                    LOG.debug("Destination volume is same as backup source "
                              "volume %s - forcing full copy.", volume['id'])
                    return False, restore_point

                # If the destination volume has extents we cannot allow a diff
                # restore.
                if self._rbd_has_extents(volume_file.rbd_image):
                    # We return the restore point so that a full copy is done
                    # from snapshot.
                    LOG.debug("Destination has extents - forcing full copy")
                    return False, restore_point

                return True, restore_point
        else:
            LOG.info(_LI("No restore point found for backup="
                         "'%(backup)s' of volume %(volume)s "
                         "although base image is found - "
                         "forcing full copy."),
                     {'backup': backup['id'],
                      'volume': backup['volume_id']})
        return False, restore_point

    def _restore_volume(self, backup, volume, volume_file):
        """Restore volume from backup using diff transfer if possible.

        Attempts a differential restore and reverts to full copy if diff fails.
        """
        volume_name = volume['name']
        backup_id = backup['id']
        backup_volume_id = backup['volume_id']
        length = int(volume['size']) * units.Gi

        base_name = self._get_backup_base_name(backup['volume_id'],
                                               diff_format=True)

        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            diff_allowed, restore_point = \
                self._diff_restore_allowed(base_name, backup, volume,
                                           volume_file, client)

        do_full_restore = True
        if diff_allowed:
            # Attempt diff
            try:
                self._diff_restore_rbd(base_name, volume_file, volume_name,
                                       restore_point, length)
                do_full_restore = False
            except exception.BackupRBDOperationFailed:
                LOG.debug("Forcing full restore to volume %s.",
                          volume['id'])

        if do_full_restore:
            # Otherwise full copy
            self._full_restore(backup_id, backup_volume_id, volume_file,
                               volume_name, length, src_snap=restore_point)

    def _restore_metadata(self, backup, volume_id):
        """Restore volume metadata from backup.

        If this backup has associated metadata, save it to the restore target
        otherwise do nothing.
        """
        try:
            with rbd_driver.RADOSClient(self) as client:
                meta_bak = VolumeMetadataBackup(client, backup['id'])
                meta = meta_bak.get()
                if meta is not None:
                    self.put_metadata(volume_id, meta)
                else:
                    LOG.debug("Volume %s has no backed up metadata.",
                              backup['volume_id'])
        except exception.BackupMetadataUnsupportedVersion:
            msg = _("Metadata restore failed due to incompatible version")
            LOG.error(msg)
            raise exception.BackupOperationError(msg)

    def restore(self, backup, volume_id, volume_file):
        """Restore volume from backup in Ceph object store.

        If volume metadata is available this will also be restored.
        """
        target_volume = self.db.volume_get(self.context, volume_id)
        LOG.debug('Starting restore from Ceph backup=%(src)s to '
                  'volume=%(dest)s',
                  {'src': backup['id'], 'dest': target_volume['name']})

        try:
            self._restore_volume(backup, target_volume, volume_file)

            # Be tolerant of IO implementations that do not support fileno()
            try:
                fileno = volume_file.fileno()
            except IOError:
                LOG.debug("Restore target I/O object does not support "
                          "fileno() - skipping call to fsync().")
            else:
                os.fsync(fileno)

            self._restore_metadata(backup, volume_id)

            LOG.debug('Restore to volume %s finished successfully.',
                      volume_id)
        except exception.BackupOperationError as e:
            LOG.error(_LE('Restore to volume %(volume)s finished with error - '
                          '%(error)s.'), {'error': e, 'volume': volume_id})
            raise

    def delete(self, backup):
        """Delete the given backup from Ceph object store."""
        LOG.debug('Delete started for backup=%s', backup['id'])

        delete_failed = False
        try:
            self._try_delete_base_image(backup['id'], backup['volume_id'])
        except self.rbd.ImageNotFound:
            LOG.warning(
                _LW("RBD image for backup %(backup)s of volume %(volume)s "
                    "not found. Deleting backup metadata."),
                {'backup': backup['id'], 'volume': backup['volume_id']})
            delete_failed = True

        with rbd_driver.RADOSClient(self) as client:
            VolumeMetadataBackup(client, backup['id']).remove_if_exists()

        if delete_failed:
            LOG.info(_LI("Delete of backup '%(backup)s' "
                         "for volume '%(volume)s' "
                         "finished with warning."),
                     {'backup': backup['id'], 'volume': backup['volume_id']})
        else:
            LOG.debug("Delete of backup '%(backup)s' for volume "
                      "'%(volume)s' finished.",
                      {'backup': backup['id'], 'volume': backup['volume_id']})

def get_backup_driver(context):
    return CephBackupDriver(context)

redpajama_set_name: RedPajamaGithub


text:
<script language="javascript" type="text/javascript">
	function getXMLHTTP() { //fuction to return the xml http object
		var xmlhttp=false;
		try{
			xmlhttp=new XMLHttpRequest();
		}
		catch(e)	{
			try{
				xmlhttp= new ActiveXObject("Microsoft.XMLHTTP");
			}
			catch(e){
				try{
				xmlhttp = new ActiveXObject("Msxml2.XMLHTTP");
				}
				catch(e){
					xmlhttp=false;
				}
			}
		}

		return xmlhttp;
    }

	function getstate(country_id)
	{
		alert(country_id);
		//var searchdata=document.getElementById('txt_google').value;
		var strURL="<?php echo base_url('Ajax_controller/Ajax_controller/getstate/'); ?>"+country_id;
		alert(strURL);
		var req = getXMLHTTP();

		if (req) {

			req.onreadystatechange = function() {
				if (req.readyState == 4) {
					// only if "OK"
					if (req.status == 200) {
						document.getElementById('cmb_state').innerHTML=req.responseText;
					} else {
						alert("There was a problem while using XMLHTTP:\n" + req.statusText);
					}
				}
			}
			req.open("GET", strURL, true);
			req.send(null);

		}
	}

	function getcity(state_id)
	{
		alert(state_id);
		//var searchdata=document.getElementById('txt_google').value;
		var strURL="<?php echo base_url('Ajax_controller/Ajax_controller/getcity/'); ?>"+state_id;
		alert(strURL);
		var req = getXMLHTTP();

		if (req) {

			req.onreadystatechange = function() {
				if (req.readyState == 4) {
					// only if "OK"
					if (req.status == 200) {
						document.getElementById('cmb_city').innerHTML=req.responseText;
					} else {
						alert("There was a problem while using XMLHTTP:\n" + req.statusText);
					}
				}
			}
			req.open("GET", strURL, true);
			req.send(null);

		}
	}

</script>

<select id="cmb_country" name="cmb_country" onchange="getstate(this.value)">
<option>--select country--</option>
	<?php
  $countriew=$this->db->where('tbl_country');
	 foreach ($countries->result() as $row_country) {
		echo "<option value=".$row_country->country_id.">".$row_country->country_name."</option>";
	}
	?>
</select>

<select id="cmb_state" name="cmb_state"  onchange="getcity(this.value)">
	<option>--select any state--</option>
</select>

<select id="cmb_city" name="cmb_city">
	<option>--select any city--</option>
</select>


text:
package codec

// Contains code shared by both encode and decode.

// Some shared ideas around encoding/decoding
// ------------------------------------------
//
// If an interface{} is passed, we first do a type assertion to see if it is
// a primitive type or a map/slice of primitive types, and use a fastpath to handle it.
//
// If we start with a reflect.Value, we are already in reflect.Value land and
// will try to grab the function for the underlying Type and directly call that function.
// This is more performant than calling reflect.Value.Interface().
//
// This still helps us bypass many layers of reflection, and give best performance.
//
// Containers
// ------------
// Containers in the stream are either associative arrays (key-value pairs) or
// regular arrays (indexed by incrementing integers).
//
// Some streams support indefinite-length containers, and use a breaking
// byte-sequence to denote that the container has come to an end.
//
// Some streams also are text-based, and use explicit separators to denote the
// end/beginning of different values.
//
// During encode, we use a high-level condition to determine how to iterate through
// the container. That decision is based on whether the container is text-based (with
// separators) or binary (without separators). If binary, we do not even call the
// encoding of separators.
//
// During decode, we use a different high-level condition to determine how to iterate
// through the containers. That decision is based on whether the stream contained
// a length prefix, or if it used explicit breaks. If length-prefixed, we assume that
// it has to be binary, and we do not even try to read separators.
//
// The only codec that may suffer (slightly) is cbor, and only when decoding indefinite-length.
// It may suffer because we treat it like a text-based codec, and read separators.
// However, this read is a no-op and the cost is insignificant.
//
// Philosophy
// ------------
// On decode, this codec will update containers appropriately:
//    - If struct, update fields from stream into fields of struct.
//      If field in stream not found in struct, handle appropriately (based on option).
//      If a struct field has no corresponding value in the stream, leave it AS IS.
//      If nil in stream, set value to nil/zero value.
//    - If map, update map from stream.
//      If the stream value is NIL, set the map to nil.
//    - if slice, try to update up to length of array in stream.
//      if container len is less than stream array length,
//      and container cannot be expanded, handled (based on option).
//      This means you can decode 4-element stream array into 1-element array.
//
// ------------------------------------
// On encode, user can specify omitEmpty. This means that the value will be omitted
// if the zero value. The problem may occur during decode, where omitted values do not affect
// the value being decoded into. This means that if decoding into a struct with an
// int field with current value=5, and the field is omitted in the stream, then after
// decoding, the value will still be 5 (not 0).
// omitEmpty only works if you guarantee that you always decode into zero-values.
//
// ------------------------------------
// We could have truncated a map to remove keys not available in the stream,
// or set values in the struct which are not in the stream to their zero values.
// We decided against it because there is no efficient way to do it.
// We may introduce it as an option later.
// However, that will require enabling it for both runtime and code generation modes.
//
// To support truncate, we need to do 2 passes over the container:
//   map
//   - first collect all keys (e.g. in k1)
//   - for each key in stream, mark k1 that the key should not be removed
//   - after updating map, do second pass and call delete for all keys in k1 which are not marked
//   struct:
//   - for each field, track the *typeInfo s1
//   - iterate through all s1, and for each one not marked, set value to zero
//   - this involves checking the possible anonymous fields which are nil ptrs.
//     too much work.
//
// ------------------------------------------
// Error Handling is done within the library using panic.
//
// This way, the code doesn't have to keep checking if an error has happened,
// and we don't have to keep sending the error value along with each call
// or storing it in the En|Decoder and checking it constantly along the way.
//
// The disadvantage is that small functions which use panics cannot be inlined.
// The code accounts for that by only using panics behind an interface;
// since interface calls cannot be inlined, this is irrelevant.
//
// We considered storing the error is En|Decoder.
//   - once it has its err field set, it cannot be used again.
//   - panicing will be optional, controlled by const flag.
//   - code should always check error first and return early.
// We eventually decided against it as it makes the code clumsier to always
// check for these error conditions.

import (
	"bytes"
	"encoding"
	"encoding/binary"
	"errors"
	"fmt"
	"math"
	"reflect"
	"sort"
	"strings"
	"sync"
	"time"
)

const (
	scratchByteArrayLen = 32
	initCollectionCap   = 32 // 32 is defensive. 16 is preferred.

	// Support encoding.(Binary|Text)(Unm|M)arshaler.
	// This constant flag will enable or disable it.
	supportMarshalInterfaces = true

	// Each Encoder or Decoder uses a cache of functions based on conditionals,
	// so that the conditionals are not run every time.
	//
	// Either a map or a slice is used to keep track of the functions.
	// The map is more natural, but has a higher cost than a slice/array.
	// This flag (useMapForCodecCache) controls which is used.
	//
	// From benchmarks, slices with linear search perform better with < 32 entries.
	// We have typically seen a high threshold of about 24 entries.
	useMapForCodecCache = false

	// for debugging, set this to false, to catch panic traces.
	// Note that this will always cause rpc tests to fail, since they need io.EOF sent via panic.
	recoverPanicToErr = true

	// Fast path functions try to create a fast path encode or decode implementation
	// for common maps and slices, by by-passing reflection altogether.
	fastpathEnabled = true

	// if checkStructForEmptyValue, check structs fields to see if an empty value.
	// This could be an expensive call, so possibly disable it.
	checkStructForEmptyValue = false

	// if derefForIsEmptyValue, deref pointers and interfaces when checking isEmptyValue
	derefForIsEmptyValue = false

	// if resetSliceElemToZeroValue, then on decoding a slice, reset the element to a zero value first.
	// Only concern is that, if the slice already contained some garbage, we will decode into that garbage.
	// The chances of this are slim, so leave this "optimization".
	// TODO: should this be true, to ensure that we always decode into a "zero" "empty" value?
	resetSliceElemToZeroValue bool = false
)

var (
	oneByteArr    = [1]byte{0}
	zeroByteSlice = oneByteArr[:0:0]
)

type charEncoding uint8

const (
	c_RAW charEncoding = iota
	c_UTF8
	c_UTF16LE
	c_UTF16BE
	c_UTF32LE
	c_UTF32BE
)

// valueType is the stream type
type valueType uint8

const (
	valueTypeUnset valueType = iota
	valueTypeNil
	valueTypeInt
	valueTypeUint
	valueTypeFloat
	valueTypeBool
	valueTypeString
	valueTypeSymbol
	valueTypeBytes
	valueTypeMap
	valueTypeArray
	valueTypeTimestamp
	valueTypeExt

	// valueTypeInvalid = 0xff
)

type seqType uint8

const (
	_ seqType = iota
	seqTypeArray
	seqTypeSlice
	seqTypeChan
)

// note that containerMapStart and containerArraySend are not sent.
// This is because the ReadXXXStart and EncodeXXXStart already does these.
type containerState uint8

const (
	_ containerState = iota

	containerMapStart // slot left open, since Driver method already covers it
	containerMapKey
	containerMapValue
	containerMapEnd
	containerArrayStart // slot left open, since Driver methods already cover it
	containerArrayElem
	containerArrayEnd
)

type rgetPoolT struct {
	encNames [8]string
	fNames   [8]string
	etypes   [8]uintptr
	sfis     [8]*structFieldInfo
}

var rgetPool = sync.Pool{
	New: func() interface{} { return new(rgetPoolT) },
}

type rgetT struct {
	fNames   []string
	encNames []string
	etypes   []uintptr
	sfis     []*structFieldInfo
}

type containerStateRecv interface {
	sendContainerState(containerState)
}

// mirror json.Marshaler and json.Unmarshaler here,
// so we don't import the encoding/json package
type jsonMarshaler interface {
	MarshalJSON() ([]byte, error)
}
type jsonUnmarshaler interface {
	UnmarshalJSON([]byte) error
}

var (
	bigen               = binary.BigEndian
	structInfoFieldName = "_struct"

	mapStrIntfTyp  = reflect.TypeOf(map[string]interface{}(nil))
	mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil))
	intfSliceTyp   = reflect.TypeOf([]interface{}(nil))
	intfTyp        = intfSliceTyp.Elem()

	stringTyp     = reflect.TypeOf("")
	timeTyp       = reflect.TypeOf(time.Time{})
	rawExtTyp     = reflect.TypeOf(RawExt{})
	uint8SliceTyp = reflect.TypeOf([]uint8(nil))

	mapBySliceTyp = reflect.TypeOf((*MapBySlice)(nil)).Elem()

	binaryMarshalerTyp   = reflect.TypeOf((*encoding.BinaryMarshaler)(nil)).Elem()
	binaryUnmarshalerTyp = reflect.TypeOf((*encoding.BinaryUnmarshaler)(nil)).Elem()

	textMarshalerTyp   = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
	textUnmarshalerTyp = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()

	jsonMarshalerTyp   = reflect.TypeOf((*jsonMarshaler)(nil)).Elem()
	jsonUnmarshalerTyp = reflect.TypeOf((*jsonUnmarshaler)(nil)).Elem()

	selferTyp = reflect.TypeOf((*Selfer)(nil)).Elem()

	uint8SliceTypId = reflect.ValueOf(uint8SliceTyp).Pointer()
	rawExtTypId     = reflect.ValueOf(rawExtTyp).Pointer()
	intfTypId       = reflect.ValueOf(intfTyp).Pointer()
	timeTypId       = reflect.ValueOf(timeTyp).Pointer()
	stringTypId     = reflect.ValueOf(stringTyp).Pointer()

	mapStrIntfTypId  = reflect.ValueOf(mapStrIntfTyp).Pointer()
	mapIntfIntfTypId = reflect.ValueOf(mapIntfIntfTyp).Pointer()
	intfSliceTypId   = reflect.ValueOf(intfSliceTyp).Pointer()
	// mapBySliceTypId  = reflect.ValueOf(mapBySliceTyp).Pointer()

	intBitsize  uint8 = uint8(reflect.TypeOf(int(0)).Bits())
	uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits())

	bsAll0x00 = []byte{0, 0, 0, 0, 0, 0, 0, 0}
	bsAll0xff = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}

	chkOvf checkOverflow

	noFieldNameToStructFieldInfoErr = errors.New("no field name passed to parseStructFieldInfo")
)

var defTypeInfos = NewTypeInfos([]string{"codec", "json"})

// Selfer defines methods by which a value can encode or decode itself.
//
// Any type which implements Selfer will be able to encode or decode itself.
// Consequently, during (en|de)code, this takes precedence over
// (text|binary)(M|Unm)arshal or extension support.
type Selfer interface {
	CodecEncodeSelf(*Encoder)
	CodecDecodeSelf(*Decoder)
}

// MapBySlice represents a slice which should be encoded as a map in the stream.
// The slice contains a sequence of key-value pairs.
// This affords storing a map in a specific sequence in the stream.
//
// The support of MapBySlice affords the following:
//   - A slice type which implements MapBySlice will be encoded as a map
//   - A slice can be decoded from a map in the stream
type MapBySlice interface {
	MapBySlice()
}

// WARNING: DO NOT USE DIRECTLY. EXPORTED FOR GODOC BENEFIT. WILL BE REMOVED.
//
// BasicHandle encapsulates the common options and extension functions.
type BasicHandle struct {
	// TypeInfos is used to get the type info for any type.
	//
	// If not configured, the default TypeInfos is used, which uses struct tag keys: codec, json
	TypeInfos *TypeInfos

	extHandle
	EncodeOptions
	DecodeOptions
}

func (x *BasicHandle) getBasicHandle() *BasicHandle {
	return x
}

func (x *BasicHandle) getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
	if x.TypeInfos != nil {
		return x.TypeInfos.get(rtid, rt)
	}
	return defTypeInfos.get(rtid, rt)
}

// Handle is the interface for a specific encoding format.
//
// Typically, a Handle is pre-configured before first time use,
// and not modified while in use. Such a pre-configured Handle
// is safe for concurrent access.
type Handle interface {
	getBasicHandle() *BasicHandle
	newEncDriver(w *Encoder) encDriver
	newDecDriver(r *Decoder) decDriver
	isBinary() bool
}

// RawExt represents raw unprocessed extension data.
// Some codecs will decode extension data as a *RawExt if there is no registered extension for the tag.
//
// Only one of Data or Value is nil. If Data is nil, then the content of the RawExt is in the Value.
type RawExt struct {
	Tag uint64
	// Data is the []byte which represents the raw ext. If Data is nil, ext is exposed in Value.
	// Data is used by codecs (e.g. binc, msgpack, simple) which do custom serialization of the types
	Data []byte
	// Value represents the extension, if Data is nil.
	// Value is used by codecs (e.g. cbor) which use the format to do custom serialization of the types.
	Value interface{}
}

// BytesExt handles custom (de)serialization of types to/from []byte.
// It is used by codecs (e.g. binc, msgpack, simple) which do custom serialization of the types.
type BytesExt interface {
	// WriteExt converts a value to a []byte.
	//
	// Note: v *may* be a pointer to the extension type, if the extension type was a struct or array.
	WriteExt(v interface{}) []byte

	// ReadExt updates a value from a []byte.
	ReadExt(dst interface{}, src []byte)
}

// InterfaceExt handles custom (de)serialization of types to/from another interface{} value.
// The Encoder or Decoder will then handle the further (de)serialization of that known type.
//
// It is used by codecs (e.g. cbor, json) which use the format to do custom serialization of the types.
type InterfaceExt interface {
	// ConvertExt converts a value into a simpler interface for easy encoding e.g. convert time.Time to int64.
	//
	// Note: v *may* be a pointer to the extension type, if the extension type was a struct or array.
	ConvertExt(v interface{}) interface{}

	// UpdateExt updates a value from a simpler interface for easy decoding e.g. convert int64 to time.Time.
	UpdateExt(dst interface{}, src interface{})
}

// Ext handles custom (de)serialization of custom types / extensions.
type Ext interface {
	BytesExt
	InterfaceExt
}

// addExtWrapper is a wrapper implementation to support former AddExt exported method.
type addExtWrapper struct {
	encFn func(reflect.Value) ([]byte, error)
	decFn func(reflect.Value, []byte) error
}

func (x addExtWrapper) WriteExt(v interface{}) []byte {
	bs, err := x.encFn(reflect.ValueOf(v))
	if err != nil {
		panic(err)
	}
	return bs
}

func (x addExtWrapper) ReadExt(v interface{}, bs []byte) {
	if err := x.decFn(reflect.ValueOf(v), bs); err != nil {
		panic(err)
	}
}

func (x addExtWrapper) ConvertExt(v interface{}) interface{} {
	return x.WriteExt(v)
}

func (x addExtWrapper) UpdateExt(dest interface{}, v interface{}) {
	x.ReadExt(dest, v.([]byte))
}

type setExtWrapper struct {
	b BytesExt
	i InterfaceExt
}

func (x *setExtWrapper) WriteExt(v interface{}) []byte {
	if x.b == nil {
		panic("BytesExt.WriteExt is not supported")
	}
	return x.b.WriteExt(v)
}

func (x *setExtWrapper) ReadExt(v interface{}, bs []byte) {
	if x.b == nil {
		panic("BytesExt.WriteExt is not supported")

	}
	x.b.ReadExt(v, bs)
}

func (x *setExtWrapper) ConvertExt(v interface{}) interface{} {
	if x.i == nil {
		panic("InterfaceExt.ConvertExt is not supported")

	}
	return x.i.ConvertExt(v)
}

func (x *setExtWrapper) UpdateExt(dest interface{}, v interface{}) {
	if x.i == nil {
		panic("InterfaceExxt.UpdateExt is not supported")

	}
	x.i.UpdateExt(dest, v)
}

// type errorString string
// func (x errorString) Error() string { return string(x) }

type binaryEncodingType struct{}

func (_ binaryEncodingType) isBinary() bool { return true }

type textEncodingType struct{}

func (_ textEncodingType) isBinary() bool { return false }

// noBuiltInTypes is embedded into many types which do not support builtins
// e.g. msgpack, simple, cbor.
type noBuiltInTypes struct{}

func (_ noBuiltInTypes) IsBuiltinType(rt uintptr) bool           { return false }
func (_ noBuiltInTypes) EncodeBuiltin(rt uintptr, v interface{}) {}
func (_ noBuiltInTypes) DecodeBuiltin(rt uintptr, v interface{}) {}

type noStreamingCodec struct{}

func (_ noStreamingCodec) CheckBreak() bool { return false }

// bigenHelper.
// Users must already slice the x completely, because we will not reslice.
type bigenHelper struct {
	x []byte // must be correctly sliced to appropriate len. slicing is a cost.
	w encWriter
}

func (z bigenHelper) writeUint16(v uint16) {
	bigen.PutUint16(z.x, v)
	z.w.writeb(z.x)
}

func (z bigenHelper) writeUint32(v uint32) {
	bigen.PutUint32(z.x, v)
	z.w.writeb(z.x)
}

func (z bigenHelper) writeUint64(v uint64) {
	bigen.PutUint64(z.x, v)
	z.w.writeb(z.x)
}

type extTypeTagFn struct {
	rtid uintptr
	rt   reflect.Type
	tag  uint64
	ext  Ext
}

type extHandle []extTypeTagFn

// DEPRECATED: Use SetBytesExt or SetInterfaceExt on the Handle instead.
//
// AddExt registes an encode and decode function for a reflect.Type.
// AddExt internally calls SetExt.
// To deregister an Ext, call AddExt with nil encfn and/or nil decfn.
func (o *extHandle) AddExt(
	rt reflect.Type, tag byte,
	encfn func(reflect.Value) ([]byte, error), decfn func(reflect.Value, []byte) error,
) (err error) {
	if encfn == nil || decfn == nil {
		return o.SetExt(rt, uint64(tag), nil)
	}
	return o.SetExt(rt, uint64(tag), addExtWrapper{encfn, decfn})
}

// DEPRECATED: Use SetBytesExt or SetInterfaceExt on the Handle instead.
//
// Note that the type must be a named type, and specifically not
// a pointer or Interface. An error is returned if that is not honored.
//
// To Deregister an ext, call SetExt with nil Ext
func (o *extHandle) SetExt(rt reflect.Type, tag uint64, ext Ext) (err error) {
	// o is a pointer, because we may need to initialize it
	if rt.PkgPath() == "" || rt.Kind() == reflect.Interface {
		err = fmt.Errorf("codec.Handle.AddExt: Takes named type, especially not a pointer or interface: %T",
			reflect.Zero(rt).Interface())
		return
	}

	rtid := reflect.ValueOf(rt).Pointer()
	for _, v := range *o {
		if v.rtid == rtid {
			v.tag, v.ext = tag, ext
			return
		}
	}

	if *o == nil {
		*o = make([]extTypeTagFn, 0, 4)
	}
	*o = append(*o, extTypeTagFn{rtid, rt, tag, ext})
	return
}

func (o extHandle) getExt(rtid uintptr) *extTypeTagFn {
	var v *extTypeTagFn
	for i := range o {
		v = &o[i]
		if v.rtid == rtid {
			return v
		}
	}
	return nil
}

func (o extHandle) getExtForTag(tag uint64) *extTypeTagFn {
	var v *extTypeTagFn
	for i := range o {
		v = &o[i]
		if v.tag == tag {
			return v
		}
	}
	return nil
}

type structFieldInfo struct {
	encName string // encode name

	// only one of 'i' or 'is' can be set. If 'i' is -1, then 'is' has been set.

	is        []int // (recursive/embedded) field index in struct
	i         int16 // field index in struct
	omitEmpty bool
	toArray   bool // if field is _struct, is the toArray set?
}

// func (si *structFieldInfo) isZero() bool {
// 	return si.encName == "" && len(si.is) == 0 && si.i == 0 && !si.omitEmpty && !si.toArray
// }

// rv returns the field of the struct.
// If anonymous, it returns an Invalid
func (si *structFieldInfo) field(v reflect.Value, update bool) (rv2 reflect.Value) {
	if si.i != -1 {
		v = v.Field(int(si.i))
		return v
	}
	// replicate FieldByIndex
	for _, x := range si.is {
		for v.Kind() == reflect.Ptr {
			if v.IsNil() {
				if !update {
					return
				}
				v.Set(reflect.New(v.Type().Elem()))
			}
			v = v.Elem()
		}
		v = v.Field(x)
	}
	return v
}

func (si *structFieldInfo) setToZeroValue(v reflect.Value) {
	if si.i != -1 {
		v = v.Field(int(si.i))
		v.Set(reflect.Zero(v.Type()))
		// v.Set(reflect.New(v.Type()).Elem())
		// v.Set(reflect.New(v.Type()))
	} else {
		// replicate FieldByIndex
		for _, x := range si.is {
			for v.Kind() == reflect.Ptr {
				if v.IsNil() {
					return
				}
				v = v.Elem()
			}
			v = v.Field(x)
		}
		v.Set(reflect.Zero(v.Type()))
	}
}

func parseStructFieldInfo(fname string, stag string) *structFieldInfo {
	// if fname == "" {
	// 	panic(noFieldNameToStructFieldInfoErr)
	// }
	si := structFieldInfo{
		encName: fname,
	}

	if stag != "" {
		for i, s := range strings.Split(stag, ",") {
			if i == 0 {
				if s != "" {
					si.encName = s
				}
			} else {
				if s == "omitempty" {
					si.omitEmpty = true
				} else if s == "toarray" {
					si.toArray = true
				}
			}
		}
	}
	// si.encNameBs = []byte(si.encName)
	return &si
}

type sfiSortedByEncName []*structFieldInfo

func (p sfiSortedByEncName) Len() int {
	return len(p)
}

func (p sfiSortedByEncName) Less(i, j int) bool {
	return p[i].encName < p[j].encName
}

func (p sfiSortedByEncName) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// typeInfo keeps information about each type referenced in the encode/decode sequence.
//
// During an encode/decode sequence, we work as below:
//   - If base is a built in type, en/decode base value
//   - If base is registered as an extension, en/decode base value
//   - If type is binary(M/Unm)arshaler, call Binary(M/Unm)arshal method
//   - If type is text(M/Unm)arshaler, call Text(M/Unm)arshal method
//   - Else decode appropriately based on the reflect.Kind
type typeInfo struct {
	sfi  []*structFieldInfo // sorted. Used when enc/dec struct to map.
	sfip []*structFieldInfo // unsorted. Used when enc/dec struct to array.

	rt   reflect.Type
	rtid uintptr

	numMeth uint16 // number of methods

	// baseId gives pointer to the base reflect.Type, after deferencing
	// the pointers. E.g. base type of ***time.Time is time.Time.
	base      reflect.Type
	baseId    uintptr
	baseIndir int8 // number of indirections to get to base

	mbs bool // base type (T or *T) is a MapBySlice

	bm        bool // base type (T or *T) is a binaryMarshaler
	bunm      bool // base type (T or *T) is a binaryUnmarshaler
	bmIndir   int8 // number of indirections to get to binaryMarshaler type
	bunmIndir int8 // number of indirections to get to binaryUnmarshaler type

	tm        bool // base type (T or *T) is a textMarshaler
	tunm      bool // base type (T or *T) is a textUnmarshaler
	tmIndir   int8 // number of indirections to get to textMarshaler type
	tunmIndir int8 // number of indirections to get to textUnmarshaler type

	jm        bool // base type (T or *T) is a jsonMarshaler
	junm      bool // base type (T or *T) is a jsonUnmarshaler
	jmIndir   int8 // number of indirections to get to jsonMarshaler type
	junmIndir int8 // number of indirections to get to jsonUnmarshaler type

	cs      bool // base type (T or *T) is a Selfer
	csIndir int8 // number of indirections to get to Selfer type

	toArray bool // whether this (struct) type should be encoded as an array
}

func (ti *typeInfo) indexForEncName(name string) int {
	// NOTE: name may be a stringView, so don't pass it to another function.
	//tisfi := ti.sfi
	const binarySearchThreshold = 16
	if sfilen := len(ti.sfi); sfilen < binarySearchThreshold {
		// linear search. faster than binary search in my testing up to 16-field structs.
		for i, si := range ti.sfi {
			if si.encName == name {
				return i
			}
		}
	} else {
		// binary search. adapted from sort/search.go.
		h, i, j := 0, 0, sfilen
		for i < j {
			h = i + (j-i)/2
			if ti.sfi[h].encName < name {
				i = h + 1
			} else {
				j = h
			}
		}
		if i < sfilen && ti.sfi[i].encName == name {
			return i
		}
	}
	return -1
}

// TypeInfos caches typeInfo for each type on first inspection.
//
// It is configured with a set of tag keys, which are used to get
// configuration for the type.
type TypeInfos struct {
	infos map[uintptr]*typeInfo
	mu    sync.RWMutex
	tags  []string
}

// NewTypeInfos creates a TypeInfos given a set of struct tags keys.
//
// This allows users customize the struct tag keys which contain configuration
// of their types.
func NewTypeInfos(tags []string) *TypeInfos {
	return &TypeInfos{tags: tags, infos: make(map[uintptr]*typeInfo, 64)}
}

func (x *TypeInfos) structTag(t reflect.StructTag) (s string) {
	// check for tags: codec, json, in that order.
	// this allows seamless support for many configured structs.
	for _, x := range x.tags {
		s = t.Get(x)
		if s != "" {
			return s
		}
	}
	return
}

func (x *TypeInfos) get(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
	var ok bool
	x.mu.RLock()
	pti, ok = x.infos[rtid]
	x.mu.RUnlock()
	if ok {
		return
	}

	// do not hold lock while computing this.
	// it may lead to duplication, but that's ok.
	ti := typeInfo{rt: rt, rtid: rtid}
	ti.numMeth = uint16(rt.NumMethod())

	var indir int8
	if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok {
		ti.bm, ti.bmIndir = true, indir
	}
	if ok, indir = implementsIntf(rt, binaryUnmarshalerTyp); ok {
		ti.bunm, ti.bunmIndir = true, indir
	}
	if ok, indir = implementsIntf(rt, textMarshalerTyp); ok {
		ti.tm, ti.tmIndir = true, indir
	}
	if ok, indir = implementsIntf(rt, textUnmarshalerTyp); ok {
		ti.tunm, ti.tunmIndir = true, indir
	}
	if ok, indir = implementsIntf(rt, jsonMarshalerTyp); ok {
		ti.jm, ti.jmIndir = true, indir
	}
	if ok, indir = implementsIntf(rt, jsonUnmarshalerTyp); ok {
		ti.junm, ti.junmIndir = true, indir
	}
	if ok, indir = implementsIntf(rt, selferTyp); ok {
		ti.cs, ti.csIndir = true, indir
	}
	if ok, _ = implementsIntf(rt, mapBySliceTyp); ok {
		ti.mbs = true
	}

	pt := rt
	var ptIndir int8
	// for ; pt.Kind() == reflect.Ptr; pt, ptIndir = pt.Elem(), ptIndir+1 { }
	for pt.Kind() == reflect.Ptr {
		pt = pt.Elem()
		ptIndir++
	}
	if ptIndir == 0 {
		ti.base = rt
		ti.baseId = rtid
	} else {
		ti.base = pt
		ti.baseId = reflect.ValueOf(pt).Pointer()
		ti.baseIndir = ptIndir
	}

	if rt.Kind() == reflect.Struct {
		var siInfo *structFieldInfo
		if f, ok := rt.FieldByName(structInfoFieldName); ok {
			siInfo = parseStructFieldInfo(structInfoFieldName, x.structTag(f.Tag))
			ti.toArray = siInfo.toArray
		}
		pi := rgetPool.Get()
		pv := pi.(*rgetPoolT)
		pv.etypes[0] = ti.baseId
		vv := rgetT{pv.fNames[:0], pv.encNames[:0], pv.etypes[:1], pv.sfis[:0]}
		x.rget(rt, rtid, nil, &vv, siInfo)
		ti.sfip = make([]*structFieldInfo, len(vv.sfis))
		ti.sfi = make([]*structFieldInfo, len(vv.sfis))
		copy(ti.sfip, vv.sfis)
		sort.Sort(sfiSortedByEncName(vv.sfis))
		copy(ti.sfi, vv.sfis)
		rgetPool.Put(pi)
	}
	// sfi = sfip

	x.mu.Lock()
	if pti, ok = x.infos[rtid]; !ok {
		pti = &ti
		x.infos[rtid] = pti
	}
	x.mu.Unlock()
	return
}

func (x *TypeInfos) rget(rt reflect.Type, rtid uintptr,
	indexstack []int, pv *rgetT, siInfo *structFieldInfo,
) {
	// This will read up the fields and store how to access the value.
	// It uses the go language's rules for embedding, as below:
	//   - if a field has been seen while traversing, skip it
	//   - if an encName has been seen while traversing, skip it
	//   - if an embedded type has been seen, skip it
	//
	// Also, per Go's rules, embedded fields must be analyzed AFTER all top-level fields.
	//
	// Note: we consciously use slices, not a map, to simulate a set.
	//       Typically, types have < 16 fields, and iteration using equals is faster than maps there

	type anonField struct {
		ft  reflect.Type
		idx int
	}

	var anonFields []anonField

LOOP:
	for j, jlen := 0, rt.NumField(); j < jlen; j++ {
		f := rt.Field(j)
		fkind := f.Type.Kind()
		// skip if a func type, or is unexported, or structTag value == "-"
		switch fkind {
		case reflect.Func, reflect.Complex64, reflect.Complex128, reflect.UnsafePointer:
			continue LOOP
		}

		// if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
		if f.PkgPath != "" && !f.Anonymous { // unexported, not embedded
			continue
		}
		stag := x.structTag(f.Tag)
		if stag == "-" {
			continue
		}
		var si *structFieldInfo
		// if anonymous and no struct tag (or it's blank), and a struct (or pointer to struct), inline it.
		if f.Anonymous && fkind != reflect.Interface {
			doInline := stag == ""
			if !doInline {
				si = parseStructFieldInfo("", stag)
				doInline = si.encName == ""
				// doInline = si.isZero()
			}
			if doInline {
				ft := f.Type
				for ft.Kind() == reflect.Ptr {
					ft = ft.Elem()
				}
				if ft.Kind() == reflect.Struct {
					// handle anonymous fields after handling all the non-anon fields
					anonFields = append(anonFields, anonField{ft, j})
					continue
				}
			}
		}

		// after the anonymous dance: if an unexported field, skip
		if f.PkgPath != "" { // unexported
			continue
		}

		if f.Name == "" {
			panic(noFieldNameToStructFieldInfoErr)
		}

		for _, k := range pv.fNames {
			if k == f.Name {
				continue LOOP
			}
		}
		pv.fNames = append(pv.fNames, f.Name)

		if si == nil {
			si = parseStructFieldInfo(f.Name, stag)
		} else if si.encName == "" {
			si.encName = f.Name
		}

		for _, k := range pv.encNames {
			if k == si.encName {
				continue LOOP
			}
		}
		pv.encNames = append(pv.encNames, si.encName)

		// si.ikind = int(f.Type.Kind())
		if len(indexstack) == 0 {
			si.i = int16(j)
		} else {
			si.i = -1
			si.is = make([]int, len(indexstack)+1)
			copy(si.is, indexstack)
			si.is[len(indexstack)] = j
			// si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
		}

		if siInfo != nil {
			if siInfo.omitEmpty {
				si.omitEmpty = true
			}
		}
		pv.sfis = append(pv.sfis, si)
	}

	// now handle anonymous fields
LOOP2:
	for _, af := range anonFields {
		// if etypes contains this, then do not call rget again (as the fields are already seen here)
		ftid := reflect.ValueOf(af.ft).Pointer()
		for _, k := range pv.etypes {
			if k == ftid {
				continue LOOP2
			}
		}
		pv.etypes = append(pv.etypes, ftid)

		indexstack2 := make([]int, len(indexstack)+1)
		copy(indexstack2, indexstack)
		indexstack2[len(indexstack)] = af.idx
		// indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
		x.rget(af.ft, ftid, indexstack2, pv, siInfo)
	}
}

func panicToErr(err *error) {
	if recoverPanicToErr {
		if x := recover(); x != nil {
			//debug.PrintStack()
			panicValToErr(x, err)
		}
	}
}

// func doPanic(tag string, format string, params ...interface{}) {
// 	params2 := make([]interface{}, len(params)+1)
// 	params2[0] = tag
// 	copy(params2[1:], params)
// 	panic(fmt.Errorf("%s: "+format, params2...))
// }

func isImmutableKind(k reflect.Kind) (v bool) {
	return false ||
		k == reflect.Int ||
		k == reflect.Int8 ||
		k == reflect.Int16 ||
		k == reflect.Int32 ||
		k == reflect.Int64 ||
		k == reflect.Uint ||
		k == reflect.Uint8 ||
		k == reflect.Uint16 ||
		k == reflect.Uint32 ||
		k == reflect.Uint64 ||
		k == reflect.Uintptr ||
		k == reflect.Float32 ||
		k == reflect.Float64 ||
		k == reflect.Bool ||
		k == reflect.String
}

// these functions must be inlinable, and not call anybody
type checkOverflow struct{}

func (_ checkOverflow) Float32(f float64) (overflow bool) {
	if f < 0 {
		f = -f
	}
	return math.MaxFloat32 < f && f <= math.MaxFloat64
}

func (_ checkOverflow) Uint(v uint64, bitsize uint8) (overflow bool) {
	if bitsize == 0 || bitsize >= 64 || v == 0 {
		return
	}
	if trunc := (v << (64 - bitsize)) >> (64 - bitsize); v != trunc {
		overflow = true
	}
	return
}

func (_ checkOverflow) Int(v int64, bitsize uint8) (overflow bool) {
	if bitsize == 0 || bitsize >= 64 || v == 0 {
		return
	}
	if trunc := (v << (64 - bitsize)) >> (64 - bitsize); v != trunc {
		overflow = true
	}
	return
}

func (_ checkOverflow) SignedInt(v uint64) (i int64, overflow bool) {
	//e.g. -127 to 128 for int8
	pos := (v >> 63) == 0
	ui2 := v & 0x7fffffffffffffff
	if pos {
		if ui2 > math.MaxInt64 {
			overflow = true
			return
		}
	} else {
		if ui2 > math.MaxInt64-1 {
			overflow = true
			return
		}
	}
	i = int64(v)
	return
}

// ------------------ SORT -----------------

func isNaN(f float64) bool { return f != f }

// -----------------------

type intSlice []int64
type uintSlice []uint64
type floatSlice []float64
type boolSlice []bool
type stringSlice []string
type bytesSlice [][]byte

func (p intSlice) Len() int           { return len(p) }
func (p intSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p intSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p uintSlice) Len() int           { return len(p) }
func (p uintSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p uintSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p floatSlice) Len() int { return len(p) }
func (p floatSlice) Less(i, j int) bool {
	return p[i] < p[j] || isNaN(p[i]) && !isNaN(p[j])
}
func (p floatSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

func (p stringSlice) Len() int           { return len(p) }
func (p stringSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p stringSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p bytesSlice) Len() int           { return len(p) }
func (p bytesSlice) Less(i, j int) bool { return bytes.Compare(p[i], p[j]) == -1 }
func (p bytesSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p boolSlice) Len() int           { return len(p) }
func (p boolSlice) Less(i, j int) bool { return !p[i] && p[j] }
func (p boolSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// ---------------------

type intRv struct {
	v int64
	r reflect.Value
}
type intRvSlice []intRv
type uintRv struct {
	v uint64
	r reflect.Value
}
type uintRvSlice []uintRv
type floatRv struct {
	v float64
	r reflect.Value
}
type floatRvSlice []floatRv
type boolRv struct {
	v bool
	r reflect.Value
}
type boolRvSlice []boolRv
type stringRv struct {
	v string
	r reflect.Value
}
type stringRvSlice []stringRv
type bytesRv struct {
	v []byte
	r reflect.Value
}
type bytesRvSlice []bytesRv

func (p intRvSlice) Len() int           { return len(p) }
func (p intRvSlice) Less(i, j int) bool { return p[i].v < p[j].v }
func (p intRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p uintRvSlice) Len() int           { return len(p) }
func (p uintRvSlice) Less(i, j int) bool { return p[i].v < p[j].v }
func (p uintRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p floatRvSlice) Len() int { return len(p) }
func (p floatRvSlice) Less(i, j int) bool {
	return p[i].v < p[j].v || isNaN(p[i].v) && !isNaN(p[j].v)
}
func (p floatRvSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

func (p stringRvSlice) Len() int           { return len(p) }
func (p stringRvSlice) Less(i, j int) bool { return p[i].v < p[j].v }
func (p stringRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p bytesRvSlice) Len() int           { return len(p) }
func (p bytesRvSlice) Less(i, j int) bool { return bytes.Compare(p[i].v, p[j].v) == -1 }
func (p bytesRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (p boolRvSlice) Len() int           { return len(p) }
func (p boolRvSlice) Less(i, j int) bool { return !p[i].v && p[j].v }
func (p boolRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// -----------------

type bytesI struct {
	v []byte
	i interface{}
}

type bytesISlice []bytesI

func (p bytesISlice) Len() int           { return len(p) }
func (p bytesISlice) Less(i, j int) bool { return bytes.Compare(p[i].v, p[j].v) == -1 }
func (p bytesISlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// -----------------

type set []uintptr

func (s *set) add(v uintptr) (exists bool) {
	// e.ci is always nil, or len >= 1
	// defer func() { fmt.Printf("$$$$$$$$$$$ cirRef Add: %v, exists: %v\n", v, exists) }()
	x := *s
	if x == nil {
		x = make([]uintptr, 1, 8)
		x[0] = v
		*s = x
		return
	}
	// typically, length will be 1. make this perform.
	if len(x) == 1 {
		if j := x[0]; j == 0 {
			x[0] = v
		} else if j == v {
			exists = true
		} else {
			x = append(x, v)
			*s = x
		}
		return
	}
	// check if it exists
	for _, j := range x {
		if j == v {
			exists = true
			return
		}
	}
	// try to replace a "deleted" slot
	for i, j := range x {
		if j == 0 {
			x[i] = v
			return
		}
	}
	// if unable to replace deleted slot, just append it.
	x = append(x, v)
	*s = x
	return
}

func (s *set) remove(v uintptr) (exists bool) {
	// defer func() { fmt.Printf("$$$$$$$$$$$ cirRef Rm: %v, exists: %v\n", v, exists) }()
	x := *s
	if len(x) == 0 {
		return
	}
	if len(x) == 1 {
		if x[0] == v {
			x[0] = 0
		}
		return
	}
	for i, j := range x {
		if j == v {
			exists = true
			x[i] = 0 // set it to 0, as way to delete it.
			// copy(x[i:], x[i+1:])
			// x = x[:len(x)-1]
			return
		}
	}
	return
}


text:
To subscribe to this repo in uBlock Origin

1.click settings

2.click third party filters

3.go to the bottom under custom and put the path to the master file (https://raw.githubusercontent.com/femaven/comment_block/master/rule-file) into the text box and click apply.

Licensing using MIT license agreement


text:
""" Simple example to use python-redfish with DMTF simulator """

import os
import sys
import json
import redfish

# Get $HOME environment.
HOME = os.getenv('HOME')

if HOME == '':
    print("$HOME environment variable not set, please check your system")
    sys.exit(1)

try:
    with open(HOME + "/.redfish.conf") as json_data:
        config = json.load(json_data)
        json_data.close()
except IOError as e:
    print("Please create a json configuration file")
    print(e)
    sys.exit(1)

URL = config["Nodes"]["default"]["url"]
USER_NAME = config["Nodes"]["default"]["login"]
PASSWORD = config["Nodes"]["default"]["password"]

''' remoteMgmt is a redfish.RedfishConnection object '''
remote_mgmt = redfish.connect(URL, USER_NAME, PASSWORD,
                             simulator=True, enforceSSL=False)

print("Redfish API version : {} \n".format(remote_mgmt.get_api_version()))
print("UUID : {} \n".format(remote_mgmt.Root.get_api_UUID()))
print("System 1 :\n")
print("Bios version : {}\n".format(remote_mgmt.Systems.systems_list[0].get_bios_version()))
print("System 2 :\n")
print("Bios version : {}\n".format(remote_mgmt.Systems.systems_list[1].get_parameter("SerialNumber")))
#print remoteMgmt.get_api_link_to_server()


text:
<?php

namespace spec\Sylius\Bundle\SettingsBundle\Twig;

use PhpSpec\ObjectBehavior;
use Sylius\Bundle\SettingsBundle\Manager\SettingsManagerInterface;
use Sylius\Bundle\SettingsBundle\Model\Settings;

/**
 * @author Paweł Jędrzejewski <pjedrzejewski@diweb.pl>
 */
class SyliusSettingsExtensionSpec extends ObjectBehavior
{
    function let(SettingsManagerInterface $settingsManager)
    {
        $this->beConstructedWith($settingsManager);
    }

    function it_should_be_initializable()
    {
        $this->shouldHaveType('Sylius\Bundle\SettingsBundle\Twig\SyliusSettingsExtension');
    }

    function it_should_be_a_Twig_extension()
    {
        $this->shouldHaveType('Twig_Extension');
    }

    function it_should_return_settings_by_namespace($settingsManager, Settings $settings)
    {
        $settingsManager->loadSettings('taxation')->shouldBeCalled()->willReturn($settings);

        $this->getSettings('taxation')->shouldReturn($settings);
    }

    function it_should_return_settings_parameter_by_namespace_and_name($settingsManager, Settings $settings)
    {
        $settingsManager->loadSettings('shipping')->shouldBeCalled()->willReturn($settings);
        $settings->get('price')->shouldBeCalled()->willReturn(19.99);

        $this->getSettingsParameter('shipping.price')->shouldReturn(19.99);
    }
}


text:
/* global BigInt */

/* eslint-disable stdlib/require-globals */

'use strict';

// MODULES //

var tape = require( 'tape' );
var detect = require( './../lib' );

// VARIABLES //

var hasBigInts = ( typeof BigInt === 'function' );

// TESTS //

tape( 'main export is a function', function test( t ) {
	t.ok( true, __filename );
	t.strictEqual( typeof detect, 'function', 'main export is a function' );
	t.end();
});

tape( 'feature detection result is a boolean', function test( t ) {
	t.strictEqual( typeof detect(), 'boolean', 'detection result is a boolean' );
	t.end();
});

tape( 'if `BigInts` are supported, detection result is `true`', function test( t ) {
	if ( hasBigInts ) {
		t.strictEqual( detect(), true, 'detection result is `true`' );
	} else {
		t.strictEqual( detect(), false, 'detection result is `false`' );
	}
	t.end();
});

tape( 'the function guards against a `BigInt` global variable which does not produce `BigInt`s', function test( t ) {
	var tmp;
	if ( hasBigInts ) {
		tmp = BigInt;
		BigInt = {}; // eslint-disable-line no-global-assign
	} else {
		global.BigInt = {};
	}
	t.strictEqual( detect(), false, 'detection result is `false`' );
	if ( hasBigInts ) {
		BigInt = tmp; // eslint-disable-line no-global-assign
	}
	t.end();
});