text:
'''
Created on Jun 9, 2015

@author: Alex Ip (based on code by Matt Paget & Edward King)
'''
import netCDF4
import numpy as np
import os
import re
from collections import OrderedDict
import logging
from osgeo import gdal, gdalconst, osr
from datetime import datetime

from _gdfutils import log_multiline

# Only needed for testing
from pprint import pprint

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO) # Logging level for this module

try:
    import netcdf_builder
except ImportError:
    logger.error('Requires netcdf_builder.py (https://stash.csiro.au/projects/CMAR_RS/repos/netcdf-tools/browse/create/netcdf_builder.py)')
    raise

class GDFNetCDF(object):
    '''
    Class GDFNetCDF - Class to manage GDF netCDF storage units
    '''
    def __init__(self, storage_config, netcdf_filename=None, netcdf_mode=None, netcdf_format=None, decimal_places=None):
        '''
        Constructor for class GDFNetCDF
        Parameters:
            storage_config: nested dict containing configuration for storage type (defined in class GDF)
            netcdf_filename: Filename of netCDF file to be opened
            netcdf_mode: Mode for netCDF file open
            netcdf_format: Format for netCDF file open
        '''
        self._isopen = False
        self.storage_config = storage_config
        self.netcdf_filename = netcdf_filename
        self.netcdf_mode = netcdf_mode or 'r' # Default to 'r' for reading
        self.netcdf_format = netcdf_format or 'NETCDF4_CLASSIC'
        self.decimal_places = decimal_places if decimal_places is not None else 6 # Default to 6 decimal places if no precision specified

        if netcdf_filename is None:
            self.netcdf_object = None
        else:
            self.open(netcdf_filename)

    def __del__(self):
        '''
        Destructor for class GDFNetCDF
        '''
        self.close()

    def close(self):
        '''
        Destructor for class GDFNetCDF
        '''
        self._isopen = False
        try:
            self.netcdf_object.close()
        except:
            pass

    def open(self, netcdf_filename=None, netcdf_mode=None, netcdf_format=None):
        '''
        Constructor for class GDFNetCDF
        Parameters:
            storage_config: nested dict containing configuration for storage type (defined in class GDF)
            netcdf_filename: Filename of netCDF file to be opened
            netcdf_mode: Mode for netCDF file open
            netcdf_format: Format for netCDF file open
        '''
        self._isopen = False

        # Default to existing instance values
        self.netcdf_filename = netcdf_filename or self.netcdf_filename
        assert self.netcdf_filename, 'NetCDF filename not provided'

        self.netcdf_mode = netcdf_mode or self.netcdf_mode
        self.netcdf_format = netcdf_format or self.netcdf_format

        if netcdf_mode == 'w':
            self.netcdf_object = netCDF4.Dataset(self.netcdf_filename, mode=self.netcdf_mode, format=self.netcdf_format)
        else:
            # Format will be deduced by the netCDF modules
            self.netcdf_object = netCDF4.Dataset(self.netcdf_filename, mode=self.netcdf_mode)
            self.netcdf_format = self.netcdf_object.file_format

        self._isopen = True

    def create(self, netcdf_filename, index_tuple, dimension_index_dict={}, netcdf_format=None):
        '''
        Create new NetCDF File in 'w' mode with required dimensions
        Parameters:
            index_tuple = tuple of storage unit indices
            dimension_index_dict: dict of iterables or 1D numpy arrays keyed by dimension_tag. Required for irregular dimensions (e.g. time)
        '''
        def set_dimension(dimension, dimension_config, index, dimension_index_vector=None):
            '''
            Parameters:
                dimension: Dimension tag (e.g. X, Y, T, etc.)
                dimension_config: Nested dict containing storage configuration from GDF.storage_config['<storage_type>']
                index: index for storage unit
                dimension_index_vector: Numpy array of index values for irregular dimension (e.g. time) or None for unlimited irregular dimension
            '''

            logger.debug('dimension = %s', dimension)
            logger.debug('dimension_config = %s', dimension_config)
            logger.debug('index = %s', index)
            logger.debug('dimension_index_vector = %s', dimension_index_vector)

            if dimension_config['indexing_type'] == 'regular' and not dimension_index_vector:
                element_size = dimension_config['dimension_element_size']
                dimension_min = index * dimension_config['dimension_extent'] + dimension_config['dimension_origin'] + element_size / 2.0 # Half pixel to account for netCDF centre of pixel reference
                dimension_max = dimension_min + dimension_config['dimension_extent']

                dimension_index_vector = np.around(np.arange(dimension_min, dimension_max, element_size), self.decimal_places)

                # Cater for reversed index (e.g. positive Y index tends Southwards when image origin is in UL/NW corner)
                if dimension_config['reverse_index']:
                    dimension_index_vector = dimension_index_vector[::-1]

            #TODO: Implement fixed indexing type

            log_multiline(logger.debug, dimension_index_vector, 'dimension_index_vector for %s' % dimension, '\t')

            if dimension_index_vector is not None:
                dimension_index_shape = dimension_index_vector.shape
                assert len(dimension_index_shape) == 1, 'Invalid dimension_index_vector shape. Must be 1D'
                assert dimension_index_shape[0] <= dimension_config['dimension_elements'], 'dimension_index_vector must have %d elements or fewer' % dimension_config['dimension_elements']
                dimension_size = len(dimension_index_vector)
                #TODO: Do range checks to ensure indices are within storage unit boundaries
            else:
                dimension_size = 0 # Unlimited dimension

            dimension_name = dimension_config['dimension_name']

            # Dimensions can be renamed with the 'renameDimension' method of the file
            self.netcdf_object.createDimension(dimension_name, dimension_size)

            variable = self.netcdf_object.createVariable(dimension_name,'f8',(dimension_name,))
            for property_name, property_value in dimension_config['properties'].items():
                logger.debug('property_name = %s, property_value = %s', property_name, property_value)
                variable.__setattr__(property_name, property_value)

            variable[:] = dimension_index_vector

        def set_variable(variable_name, variable_config):
            dimensions = self.storage_config['dimensions'].keys()
            dimension_names = tuple([self.storage_config['dimensions'][dimension]['dimension_name']
                                      for dimension in dimensions])

            nc_shape_dict = {dimensions[index]: len(self.netcdf_object.dimensions[dimension_names[index]]) for index in range(len(dimensions))}

            chunksizes = tuple([min(self.storage_config['dimensions'][dimension]['dimension_cache'], nc_shape_dict[dimension])
                                      for dimension in dimensions])
            logger.debug('Creating variable %s with dimensions %s and chunk sizes %s', variable_name, dimensions, chunksizes)

            variable = self.netcdf_object.createVariable(variable_name, variable_config['netcdf_datatype_name'], dimensions=dimension_names,
                   chunksizes=chunksizes, fill_value=variable_config['nodata_value'], zlib=True)
            logger.debug('variable = %s' % variable)

            # Set variable metadata
            metadata_dict = {variable_name + ':' + 'coordinates': ' '.join(dimension_names),
                             variable_name + ':' + 'grid_mapping': 'crs',
                             variable_name + ':' + 'standard_name': variable_name,
                             variable_name + ':' + 'long_name': variable_config['measurement_type_name']
                             }

            self.set_attributes(metadata_dict)
            self.netcdf_object.sync()

        # Start of create function
        # Default to existing instance value
        self.netcdf_mode = 'w'
        self.netcdf_format = netcdf_format or self.netcdf_format

        self.open(netcdf_filename=netcdf_filename)

        for dimension, dimension_config in self.storage_config['dimensions'].items():
            set_dimension(dimension, dimension_config, index_tuple[self.storage_config['dimensions'].keys().index(dimension)], dimension_index_dict.get(dimension))

        for variable, variable_config in self.storage_config['measurement_types'].items():
            set_variable(variable, variable_config)

        logger.debug('self.netcdf_object.variables = %s' % self.netcdf_object.variables)

        creation_date = datetime.utcnow().strftime("%Y%m%d")
        self.netcdf_object.history = 'NetCDF-CF file created %s.' %(creation_date)
        self.netcdf_object.license = 'Generalised Data Framework NetCDF-CF Test File'
        self.netcdf_object.spatial_coverage = '%f %s grid' % (self.storage_config['dimensions']['X']['dimension_extent'],
                                                              self.storage_config['dimensions']['X']['reference_system_unit'])
        self.netcdf_object.featureType = 'grid'

        self.sync()

    def write_slice(self, variable_name, slice_array, indices_dict):
        '''
        Function to set a specified slice in the specified netCDF variable
        Parameters:
            variable_name: Name of variable to which slice array will be written
            slice_array: Numpy array to be written to netCDF file
            indices_dict: Dict keyed by dimension tag indicating the dimension(s) & index/indices to which the slice should be written
        '''
        if not self._isopen:
            self.open()

        dimension_config = self.storage_config['dimensions']
        dimensions = dimension_config.keys()
        index_dimensions = indices_dict.keys()
        dimension_names = [dimension_config[dimension]['dimension_name'] for dimension in dimensions]
        # Dict of dimensions and sizes read from netCDF
        nc_shape_dict = {dimensions[index]: len(self.netcdf_object.dimensions[dimension_names[index]]) for index in range(len(dimensions))}

        logger.debug('variable_name = %s', variable_name)
        logger.debug('slice_array.shape = %s', slice_array.shape)
        logger.debug('indices_dict = %s', indices_dict)
        logger.debug('nc_shape_dict = %s', nc_shape_dict)

        assert set(index_dimensions) <= set(dimensions), 'Invalid slice index dimension(s)'

        assert len(slice_array.shape) + len(indices_dict) == len(dimensions), 'Indices must be provided for all dimensions not covered by the data array'

        slice_shape = tuple(nc_shape_dict[dimension] for dimension in dimensions if dimension not in indices_dict)
        assert slice_array.shape == slice_shape, 'Shape of data array %s does not match storage unit slice shape %s' % (slice_array.shape, slice_shape)

        # Create slices for accessing netcdf array
        slicing = [slice(indices_dict[dimension], indices_dict[dimension] + 1) if dimension in index_dimensions
                  else slice(0, nc_shape_dict[dimension]) for dimension in dimensions]
        logger.debug('slicing = %s', slicing)

        logger.debug('self.netcdf_object.variables = %s' % self.netcdf_object.variables)
        variable = self.netcdf_object.variables[variable_name]
#        logger.debug('variable = %s' % variable)

        logger.debug('slice_array = %s', slice_array)

        variable[slicing] = slice_array

    def read_slice(self, variable_name, indices_dict):
        '''
        Function to read a specified slice in the specified netCDF variable
        Parameters:
            variable_name: Name of variable from which the slice array will be read
            indices_dict: Dict keyed by dimension tag indicating the dimension(s) & index/indices from which the slice should be read
        Returns:
            slice_array: Numpy array read from netCDF file
        '''
        if not self._isopen:
            self.open()

        dimension_config = self.storage_config['dimensions']
        dimensions = dimension_config.keys()
        index_dimensions = indices_dict.keys()
        dimension_names = [dimension_config[dimension]['dimension_name'] for dimension in dimensions]
        # Dict of dimensions and sizes read from netCDF
        nc_shape_dict = {dimensions[index]: len(self.netcdf_object.dimensions[dimension_names[index]]) for index in range(len(dimensions))}

        logger.debug('variable_name = %s', variable_name)
        logger.debug('indices_dict = %s', indices_dict)
        logger.debug('nc_shape_dict = %s', nc_shape_dict)

        assert set(index_dimensions) <= set(dimensions), 'Invalid slice index dimension(s)'

        # Create slices for accessing netcdf array
        slicing = [slice(indices_dict[dimension], indices_dict[dimension] + 1) if dimension in index_dimensions
                  else slice(0, nc_shape_dict[dimension]) for dimension in dimensions]
        logger.debug('slicing = %s', slicing)

        logger.debug('self.netcdf_object.variables = %s' % self.netcdf_object.variables)
        variable = self.netcdf_object.variables[variable_name]
#        logger.debug('variable = %s' % variable)

        slice_array = variable[slicing]
        logger.debug('slice_array = %s', slice_array)
        return slice_array

    def get_subset_indices(self, range_dict):
        '''
        Function to read an array subset of the specified netCDF variable
        Parameters:
            variable_name: Name of variable from which the subset array will be read
            range_dict: Dict keyed by dimension tag containing the dimension(s) & range tuples from which the subset should be read
        Returns:
            dimension_indices_dict: Dict containing array indices for each dimension
        '''
        if not self._isopen:
            self.open()

        dimension_config = self.storage_config['dimensions']
        dimensions = dimension_config.keys()
        range_dimensions = range_dict.keys()
        dimension_names = [dimension_config[dimension]['dimension_name'] for dimension in dimensions]

        # Dict of dimensions and sizes read from netCDF
        nc_shape_dict = {dimensions[index]: len(self.netcdf_object.dimensions[dimension_names[index]]) for index in range(len(dimensions))}
        logger.debug('range_dict = %s', range_dict)
        logger.debug('nc_shape_dict = %s', nc_shape_dict)

        assert set(range_dimensions) <= set(dimensions), 'Invalid range dimension(s)'

        # Create slices for accessing netcdf array
        dimension_indices_dict = {} # Dict containing all indices for each dimension
        for dimension_index in range(len(dimensions)):
            dimension = dimensions[dimension_index]
            dimension_array = self.netcdf_object.variables[dimension_names[dimension_index]][:]
            if dimension in range_dimensions:
                logger.debug('dimension_array = %s', dimension_array)
                logger.debug('range = %s', range_dict[dimension])
                mask_array = ((dimension_array > range_dict[dimension][0]) * (dimension_array <= range_dict[dimension][1]))
                index_array = np.where(mask_array)
                logger.debug('index_array = %s', index_array)
                dimension_indices_dict[dimension] = dimension_array[mask_array]

                if not index_array:
                    logger.warning('Invalid range %s for dimension %s', range_dict[dimension], dimension)
                    return None
            else: # Range not defined for this dimension - take the whole lot
                dimension_indices_dict[dimension] = dimension_array

        return dimension_indices_dict

    def read_subset(self, variable_name, range_dict, max_bytes=None):
        '''
        Function to read an array subset of the specified netCDF variable
        Parameters:
            variable_name: Name of variable from which the subset array will be read
            range_dict: Dict keyed by dimension tag containing the dimension(s) & range tuples from which the subset should be read
        Returns:
            subset_array: Numpy array read from netCDF file
            dimension_indices_dict: Dict containing array indices for each dimension
            max_bytes: integer specifying maximum number of bytes per read. None = unlimited
        '''
        if not self._isopen:
            self.open()

        dimension_config = self.storage_config['dimensions']
        dimensions = dimension_config.keys()
        range_dimensions = range_dict.keys()
        dimension_names = [dimension_config[dimension]['dimension_name'] for dimension in dimensions]
        # Dict of dimensions and sizes read from netCDF
        nc_shape_dict = {dimensions[index]: len(self.netcdf_object.dimensions[dimension_names[index]]) for index in range(len(dimensions))}

        logger.debug('variable_name = %s', variable_name)
        logger.debug('range_dict = %s', range_dict)
        logger.debug('nc_shape_dict = %s', nc_shape_dict)

        assert set(range_dimensions) <= set(dimensions), 'Invalid range dimension(s)'

        # Create slices for accessing netcdf array
        dimension_indices_dict = {} # Dict containing all indices for each dimension
        slicing = []
        for dimension_index in range(len(dimensions)):
            dimension = dimensions[dimension_index]
            dimension_array = self.netcdf_object.variables[dimension_names[dimension_index]][:]
            if dimension in range_dimensions:
                logger.debug('dimension_array = %s', dimension_array)
                logger.debug('range = %s', range_dict[dimension])
                mask_array = ((dimension_array > range_dict[dimension][0]) * (dimension_array <= range_dict[dimension][1]))
                index_array = np.where(mask_array)
                logger.debug('index_array = %s', index_array)
                dimension_indices_dict[dimension] = dimension_array[mask_array]
                try:
                    dimension_slice = slice(index_array[0][0], index_array[0][-1] + 1)
                except IndexError:
                    logger.warning('Invalid range %s for dimension %s', range_dict[dimension], dimension)
                    return None
            else: # Range not defined for this dimension
                dimension_indices_dict[dimension] = dimension_array
                dimension_slice = slice(0, nc_shape_dict[dimension])
            slicing.append(dimension_slice)

        logger.debug('slicing = %s', slicing)

        variable = self.netcdf_object.variables[variable_name]
#        logger.debug('variable = %s' % variable)

        if max_bytes == None: # Unlimited read size
            subset_array = variable[slicing]
        else: # Break read operation into separate reads each under maximum size
            #TODO: Allow for case where slice size is greater than max_bytes - i.e. partitioning in more than one dimension
            subset_shape = tuple([s.stop - s.start for s in slicing])
            logger.debug('subset_shape = %s', subset_shape)

            slice_bytes = variable[[slice(0,1) for dimension in dimension_names]].itemsize * reduce(lambda x, y: x*y, [s.stop - s.start for s in slicing[1:]])
            max_slices = (max_bytes //
                             slice_bytes //
                             self.storage_config['dimensions'][dimensions[0]]['dimension_cache'] *
                             self.storage_config['dimensions'][dimensions[0]]['dimension_cache'])

            logger.debug('max_slices = %s', max_slices)

            subset_array = np.zeros(shape=subset_shape, dtype=variable.dtype)

            for source_start_index in range(slicing[0].start, slicing[0].stop, max_slices):
                source_stop_index = min([source_start_index + max_slices, slicing[0].stop])
                source_slicing = [slice(source_start_index, source_stop_index)] + slicing[1:]
                destination_slicing = [slice(source_slicing[slice_index].start - slicing[slice_index].start, source_slicing[slice_index].stop - slicing[slice_index].start)
                                       for slice_index in range(len(source_slicing))]

                logger.debug('source_slicing = %s', source_slicing)
                logger.debug('destination_slicing = %s', destination_slicing)

                subset_array[destination_slicing] = variable[source_slicing]

        logger.debug('subset_array = %s', subset_array)
        return subset_array, dimension_indices_dict

    def get_datatype(self, variable_name, convention='numpy'):
        '''
        Returns NetCDF datatype of specified variable
        '''
        return self.storage_config['measurement_types'][variable_name].get(convention + '_datatype_name')

    def get_attributes(self, verbose=None, normalise=True):
        """
        Copy the global and variable attributes from a netCDF object to an
        OrderedDict.  This is a little like 'ncdump -h' (without the formatting).
        Global attributes are keyed in the OrderedDict by the attribute name.
        Variable attributes are keyed in the OrderedDict by the variable name and
        attribute name separated by a colon, i.e. variable:attribute.

        Normalise means that some NumPy types returned from the netCDF module are
        converted to equivalent regular types.

        Notes from the netCDF module:
          The ncattrs method of a Dataset or Variable instance can be used to
          retrieve the names of all the netCDF attributes.

          The __dict__ attribute of a Dataset or Variable instance provides all
          the netCDF attribute name/value pairs in an OrderedDict.

          self.netcdf_object.dimensions.iteritems()
          self.netcdf_object.variables
          self.netcdf_object.ncattrs()
          self.netcdf_object.__dict__
        """
        return netcdf_builder.get_attributes(self.netcdf_object, verbose, normalise)

    def set_attributes(self, ncdict, delval='DELETE'):
        """
        Copy attribute names and values from a dict (or OrderedDict) to a netCDF
        object.
        Global attributes are keyed in the OrderedDict by the attribute name.
        Variable attributes are keyed in the OrderedDict by the variable name and
        attribute name separated by a colon, i.e. variable:attribute.

        If any value is equal to delval then, if the corresponding attribute exists
        in the netCDF object, the corresponding attribute is removed from the
        netCDF object.  The default value of delval is 'DELETE'. For example,
          nc3_set_attributes(self.netcdf_object, {'temperature:missing_value':'DELETE'})
        will delete the missing_value attribute from the temperature variable.

        A ValueError exception is raised if a key refers to a variable name that
        is not defined in the netCDF object.
        """
        netcdf_builder.set_attributes(self.netcdf_object, ncdict, delval)

    def show_dimensions(self):
        """
        Print the dimension names, lengths and whether they are unlimited.
        """
        netcdf_builder.show_dimensions(self.netcdf_object)

    def set_variable(self, varname, dtype='f4', dims=None, chunksize=None, fill=None, zlib=False, **kwargs):
        """
        Define (create) a variable in a netCDF object.  No data is written to the
        variable yet.  Give the variable's dimensions as a tuple of dimension names.
        Dimensions must have been previously created with self.netcdf_object.createDimension
        (e.g. see set_timelatlon()).

        Recommended ordering of dimensions is:
          time, height or depth (Z), latitude (Y), longitude (X).
        Any other dimensions should be defined before (placed to the left of) the
        spatio-temporal coordinates.

        To create a scalar variable, use an empty tuple for the dimensions.
        Variables can be renamed with the 'renameVariable' method of the netCDF
        object.

        Specify compression with zlib=True (default = False).

        Specify the chunksize with a sequence (tuple, list) of the same length
        as dims (i.e., the number of dimensions) where each element of chunksize
        corresponds to the size of the chunk along the corresponding dimension.
        There are some tips and tricks associated with chunking - see
        http://data.auscover.org.au/node/73 for an overview.

        The default behaviour is to create a floating-point (f4) variable
        with dimensions ('time','latitude','longitude'), with no chunking and
        no compression.
        """
        netcdf_builder.set_variable(self.netcdf_object, varname, dtype=dtype, dims=dims, chunksize=chunksize, fill=fill, zlib=zlib, **kwargs)

    def add_bounds(self, dimension_tag, bounds):
        """Add a bounds array of data to the netCDF object.
        Bounds array can be a list, tuple or NumPy array.

        A bounds array gives the values of the vertices corresponding to a dimension
        variable (see the CF documentation for more information). The dimension
        variable requires an attribute called 'bounds', which references a variable
        that contains the bounds array. The bounds array has the same shape as the
        corresponding dimension with an extra size for the number of vertices.

        This function:
            - Adds a 'bounds' attribute to the dimension variable if required.
              If a bounds attribute exits then its value will be used for the bounds
              variable (bndname). Otherwise if a bndname is given then this will be
              used. Otherwise the default bndname will be '_bounds' appended to the
              dimension name.
            - If the bounds variable exists then a ValueError will be raised if its
              shape does not match the bounds array.
            - If the bounds variable does not exist then it will be created. If so
              an exra dimension is required for the number of vertices. Any existing
              dimension of the right size will be used. Otherwise a new dimension
              will be created. The new dimension's name will be 'nv' (number of
              vertices), unless this dimension name is already used in which case
              '_nv' appended to the dimension name will be used instead.
            - Lastly, the bounds array is written to the bounds variable. If the
              corresponding dimension is time (name = 'time' or dim.axis = 't') then
              the bounds array will be written as date2num data.
        """
        dimension_tag = dimension_tag.upper()
        dimension_name=self.storage_config['dimensions'][dimension_tag]['dimension_name']
        bounds_name = dimension_name + '_bounds'

        netcdf_builder.add_bounds(self.netcdf_object, dimension_name, bounds, bounds_name)

    def georeference_from_file(self, gdal_dataset_path):
        '''
        Function to set georeferencing from template GDAL dataset
        '''
        def getMinMaxExtents(samples, lines, geoTransform):
            """
            Calculates the min/max extents based on the input latitude and longitude vectors.

            :param samples:
                An integer representing the number of samples (columns) in an array.

            :param lines:
                An integer representing the number of lines (rows) in an array.

            :param geoTransform:
                A tuple containing the geotransform information returned by GDAL.

            :return:
                A tuple containing (min_lat, max_lat, min_lon, max_lat)

            :notes:
                Hasn't been tested for northern or western hemispheres.
            """
            extents = []
            x_list  = [0,samples]
            y_list  = [0,lines]

            for px in x_list:
                for py in y_list:
                    x = geoTransform[0]+(px*geoTransform[1])+(py*geoTransform[2])
                    y = geoTransform[3]+(px*geoTransform[4])+(py*geoTransform[5])
                    extents.append([x,y])

            extents = np.array(extents)
            min_lat = np.min(extents[:,1])
            max_lat = np.max(extents[:,1])
            min_lon = np.min(extents[:,0])
            max_lon = np.max(extents[:,0])

            return (min_lat, max_lat, min_lon, max_lon)

        # Start of georeference_from_file(self, gdal_dataset_path) definition
        gdal_dataset = gdal.Open(gdal_dataset_path)
        assert gdal_dataset, 'Unable to open file %s' % gdal_dataset_path

        geotransform = gdal_dataset.GetGeoTransform()
        logger.debug('geotransform = %s', geotransform)
        projection = gdal_dataset.GetProjection()
        logger.debug('projection = %s', projection)

        # Set coordinate reference system metadata variable
        spatial_reference = osr.SpatialReference()
        spatial_reference.ImportFromWkt(projection)
        crs_metadata = {'crs:name': spatial_reference.GetAttrValue('geogcs'),
                    'crs:longitude_of_prime_meridian': 0.0, #TODO: This needs to be fixed!!! An OSR object should have this, but maybe only for specific OSR references??
                    'crs:inverse_flattening': spatial_reference.GetInvFlattening(),
                    'crs:semi_major_axis': spatial_reference.GetSemiMajor(),
                    'crs:semi_minor_axis': spatial_reference.GetSemiMinor(),
                    }
        self.set_variable('crs', dims=(), dtype='i4')
        self.set_attributes(crs_metadata)
        logger.debug('crs_metadata = %s', crs_metadata)

        extents = getMinMaxExtents(gdal_dataset.RasterXSize, gdal_dataset.RasterYSize, geotransform)
        #pdb.set_trace()
        self.netcdf_object.geospatial_lat_min = extents[0]
        self.netcdf_object.geospatial_lat_max = extents[1]
        self.netcdf_object.geospatial_lat_units = 'degrees_north'
        self.netcdf_object.geospatial_lat_resolution = geotransform[5]
        self.netcdf_object.geospatial_lon_min = extents[2]
        self.netcdf_object.geospatial_lon_max = extents[3]
        self.netcdf_object.geospatial_lon_units = 'degrees_east'
        self.netcdf_object.geospatial_lon_resolution = geotransform[1]

    def sync(self):
        '''
        Function to sync file to disk
        '''
        if self._isopen:
            self.netcdf_object.sync()

    @property
    def isopen(self):
        return self._isopen


text:
layout: post
title: "[Tech보다 UX] 8회. Maker Faire Shenzhen 2015 탐방기 1편 - Hardware Accelerator를 아시나요?"
---

<img class="alignnone size-full wp-image-58" src="https://raw.githubusercontent.com/midaeng/articles/gh-pages/images/blog/techux_8th.jpg"/>

redpajama_set_name: RedPajamaGithub


text:
var db = $.couch.db(getCurrentDBName());

function getCurrentDBName() {
    return window.location.pathname.split("/")[1];
}

$(document).ready(function() {

    // init page
	clearForm();
    refreshItems();
	$('#productName').focus();

    // create new product
    $('input#createButton').click(function(e) {
       	var productName = $('#productName').val();

       	if ($('#productName').val().length == 0) {
		   	alert("Product name is required.");
			$('#productName').focus();
           	return;
       	}

       	if ($('#productDescription').val().length == 0) {
		   	alert("Product description is required.");
			$('#productDescription').focus();
           	return;
       	}

       var aTask = {
			name: $('#productName').val(),
			description: $('#productDescription').val(),
			image: $('#productImage').val()
		}

       db.saveDoc(aTask, { success: function(resp) {
	 		$('#messages').html(productName + ' created.');
           	refreshItems();
       }});

      clearForm();
    });

	// update product
    $('input#updateButton').click(function(e) {
		var productName = $('#productName').val();

       if ($('#productName').val().length == 0) {
		   alert("Product name is required.");
			$('#productName').focus();
           return;
       }

       if ($('#productDescription').val().length == 0) {
		   	alert("Product description is required.");
			$('#productDescription').focus();
           	return;
       }

       var aTask = {
           	_id: $('#productID').val(),
           	_rev: $('#productRev').val(),
           	name: $('#productName').val(),
			description: $('#productDescription').val(),
			image: $('#productImage').val()
       }

       db.saveDoc(aTask, { success: function(resp) {
			$('#messages').html(productName + ' updated.');
           	refreshItems();

			$('#productRev').val(resp.rev);
       }});
    });

	// delete product
    $('input#deleteButton').click(function(e) {
		var productName = $('#productName').val();

       if ($('#productID').val().length == 0) {
           return;
       }

       var aTask = {
           _id: $('#productID').val(),
           _rev: $('#productRev').val()
       }

       db.removeDoc(aTask, { success: function(resp) {
			$('#messages').html(productName + ' deleted.');
			refreshItems();
       }});

       clearForm();
    });

	// clear fields
	$('input#clearButton').click(function(e) {
		clearForm();
		$('#productName').focus();
	});
});

function refreshItems() {
    $("ul#productData").empty();

    db.view("iphonecouchsync/myView", {
        success: function(data){
            data.rows.map(function(row) {
            	$('ul#productData').append('
'
                	+ row.value.name
                	+ '
');

				$("ul#productData").mouseover(function() {
				    $(this).css('cursor', 'pointer');
				});

   				$('#' + row.value._id).mouseover(function() {
                	$(this).addClass('hover');
            	});

   				$('#' + row.value._id).mouseout(function() {
                	$(this).removeClass('hover');
            	});

   				$('#' + row.value._id).click(function() {
					clearForm();
                	$('#productID').val(row.value._id);
                	$('#productRev').val(row.value._rev);
                	$('#productName').val(row.value.name);
					$('#productDescription').val(row.value.description);
					$('#productImage').val(row.value.image);

                	return false;
            	});
            });
        },
        error: function(req, textStatus, errorThrown){alert('Error: '+ textStatus);}
    });
}

function clearForm() {
    $('#productID').val('');
    $('#productRev').val('');
    $('#productName').val('');
    $('#productDescription').val('');
	$('#productImage').val('');
};


text:
module Makasi
  class SearchIndex
    MAX_LITERAL_SIZE = 4095
    MAX_TEXT_SIZE = 262144

    def reindex
      sync_db_with_sitemap
      puts "End Sync, starting parse and uploading pages"
      CloudSearchDocument.desc(:reindexed_at).each do |cloudsearch_doc|

        #Makes the request pause 1.5 seconds (Workaround for rack-attack)
        sleep 1.5

        puts cloudsearch_doc.url
        html_content = load_page(cloudsearch_doc.url)
        html_doc = Nokogiri::HTML(html_content)

        if Rails.logger.debug?
          Rails.logger.debug ">>> URL: "              + cloudsearch_doc.url +
                             "\n\tTITLE: "            + title_of(html_doc) +
                             "\n\tCONTENT: "          + content_of(html_doc)[0..300] +
                             "\n\tAUTHOR: "           + meta_tag_for(html_doc, "author") +
                             "\n\tCONTENT_LANGUAGE: " + language_of(html_doc) +
                             "\n\tDESCRIPTION: "      + meta_tag_for(html_doc, "description")[0..300] +
                             "\n\tKEYWORDS: "         + meta_tag_for(html_doc, "keywords") +
                             "\n\tRESOURCE_TYPE: "    + meta_tag_for(html_doc, "resource_type") +
                             "\n\tRESOURCE_NAME: "    + resource_name_of(html_doc) +
                             "\n\tRESOURCE_ID: "      + meta_tag_for(html_doc, "resource_id") +
                             "\n"
        end
        add_item_to_cloudsearch(cloudsearch_doc, html_doc)

        cloudsearch_doc.update_attributes(reindexed_at: DateTime.now)
      end
    end

    def add_item_to_cloudsearch(cloudsearch_doc, html_doc)
      asari.add_item(cloudsearch_doc.url, {
        url:              cloudsearch_doc.url,
        title:            title_of(html_doc)[0..MAX_TEXT_SIZE],
        content:          content_of(html_doc)[0..MAX_TEXT_SIZE],
        author:           meta_tag_for(html_doc, "author")[0..MAX_TEXT_SIZE],
        content_language: language_of(html_doc)[0..MAX_LITERAL_SIZE],
        description:      meta_tag_for(html_doc, "description")[0..MAX_TEXT_SIZE],
        keywords:         meta_tag_for(html_doc, "keywords").split(",").map(&:strip),
        resource_type:    meta_tag_for(html_doc, "resource_type")[0..MAX_TEXT_SIZE],
        resource_name:    resource_name_of(html_doc)[0..MAX_TEXT_SIZE],
        resource_id:      meta_tag_for(html_doc, "resource_id")[0..MAX_TEXT_SIZE]
      })
    end

    def sync_db_with_sitemap
      CloudSearchDocument.update_all(present_in_sitemap: false)
      url_nodes = Nokogiri::XML(read_sitemap).css('url loc')

      url_nodes.each do |url_node|
        cloudsearch_doc = CloudSearchDocument.find_or_initialize_by(url: url_node.text.strip)
        cloudsearch_doc.update_attributes(present_in_sitemap: true)
      end

      if Rails.logger.debug?
        Rails.logger.debug "SEARCH_INDEX: Updated #{CloudSearchDocument.where(present_in_sitemap: true).count} documents"
        Rails.logger.debug "SEARCH_INDEX: Removed #{CloudSearchDocument.where(present_in_sitemap: false).count} documents"
      end

      CloudSearchDocument.where(present_in_sitemap: false).destroy_all
    end

    def load_page(url, limit = 10)
      if limit == 0
        Rails.logger.error "ERROR: Faild load sitemap's url #{url}"
        return ""
      end

      # we redirected all the url which end with / to url with no / at the end of the url from application.rb - so the next line will failed to load url as the response: HTTPMovedPermanently
      # url += "/" unless url.ends_with?("/")
      parsed_url = URI.parse(url)
      if url.start_with?('https') then
        http = Net::HTTP.new(parsed_url.host, parsed_url.port)
        http.use_ssl = true
        request = Net::HTTP::Get.new(url)
        response = http.start { |http| http.request(request) }
      else
        request = Net::HTTP::Get.new(url)
        response = Net::HTTP.start(parsed_url.host, parsed_url.port) { |http| http.request(request) }
      end
      case response
      when Net::HTTPSuccess
        response.body
      when Net::HTTPRedirection
        if response['location'].start_with?('http') then
          redirect_url = response['location']
        else
          redirect_url = "#{parsed_url.scheme}://#{parsed_url.host}#{response['location']}"
        end
        load_page(redirect_url, limit - 1)
      else
        Rails.logger.error "Makasi::SearchIndex ERROR: Faild load sitemap's url #{url}"
        return ""
      end
    end

    def asari
      @asari ||= Makasi::AsariClient.new
    end

    def read_sitemap
      sitemap_file = open(Makasi::Config.sitemap_url)
      Zlib::GzipReader.new(sitemap_file).read
    end

    def meta_tag_for(doc, name)
      nodes = doc.css("meta[name='#{name}']")
      nodes.present? ? HTMLEntities.new.decode(nodes[0]["content"].to_s.strip) : ""
    end

    def title_of(doc)
      nodes = doc.xpath("//title")
      nodes.present? ? HTMLEntities.new.decode(nodes[0].text) : ""
    end

    def content_of(doc)
      content_nodes = doc.css(Makasi::Config.content_selector)
      if content_nodes.present?
        extract_text(content_nodes)
      else
        extract_text([doc])
      end
    end

    def language_of(doc)
      nodes = doc.xpath("//html")
      nodes.present? ? nodes[0]["lang"].to_s : ""
    end

    def extract_text(nodes)
      content = StringIO.new
      nodes.each do |node|
        node.traverse do |child_node|
          if child_node.text?
            content << child_node.text
          elsif child_node.name == "img"
            content << child_node["alt"]
          end
          content << " "
        end
      end
      HTMLEntities.new.decode content.string.gsub(/\s+/, " ").strip
    end

    def resource_title_of(doc)
      content_nodes = doc.css(Makasi::Config.resource_title_selector)
      if content_nodes.present?
        HTMLEntities.new.decode content_nodes.map(&:text).join(" ")
      else
        title_of(doc)
      end
    end

    def resource_name_of(doc)
      content_nodes = doc.css(Makasi::Config.resource_name_selector)
      if content_nodes.present?
        HTMLEntities.new.decode content_nodes.map(&:text).join(" ")
      else
        title_of(doc)
      end
    end
  end
end


text:
layout:       post
title:        Partitions
class:       "cart"
date:         2015-11-24T17:52:48+11:00
description: "Building Components: Partitions"
categories:
image:
---

**Identify Partitions**

{% include building-components/table-layout-initial.markdown letter="B" key="building-components/partitions/00-key.markdown" value="building-components/partitions/00-value.markdown" %}
{% include building-components/table-layout-text.markdown image="building-components/partitions/00-text.markdown" %}

**Describe** construction only

{% include building-components/table-layout-block.markdown letter="A" key="building-components/partitions/01-key.markdown" value="building-components/partitions/01-value.markdown" %}
{% include building-components/table-layout-image.markdown image="building-components/partitions/01-image.svg" %}

{% include building-components/table-layout-block.markdown letter="G" key="building-components/partitions/02-key.markdown" value="building-components/partitions/02-value.markdown"  %}
{% include building-components/table-layout-image.markdown image="building-components/partitions/02-image.svg" %}

{% include building-components/table-layout-block.markdown letter="K" key="building-components/partitions/03-key.markdown" value="building-components/partitions/03-value.markdown"  %}
{% include building-components/table-layout-image.markdown image="building-components/partitions/03-image.svg" %}

{% include building-components/table-layout-block.markdown letter="Z" key="building-components/partitions/04-key.markdown" value="building-components/partitions/04-value.markdown"  %}
{% include building-components/table-layout-image.markdown image="building-components/specifications-image.svg" %}


text:
/*
 * lib/libc/malloc/malloc.c
 */

#include <xconfigs.h>
#include <assert.h>
#include <spinlock.h>
#include <string.h>
#include <stdio.h>
#include <malloc.h>
#include <xboot/kobj.h>
#include <xboot/module.h>

/*
 * Some macros.
 */
#define tlsf_cast(t, exp)		((t)(exp))
#define tlsf_min(a, b)			((a) < (b) ? (a) : (b))
#define tlsf_max(a, b)			((a) > (b) ? (a) : (b))

#define tlsf_assert				assert
#define tlsf_insist(x)			{ tlsf_assert(x); if (!(x)) { status--; } }

#if defined(__ARM64__) || defined(__X64__) || (defined(__riscv) && (__riscv_xlen == 64))
# define TLSF_64BIT
#else
# undef TLSF_64BIT
#endif

/*
 * Public constants
 */
enum tlsf_public
{
	/*
	 * log2 of number of linear subdivisions of block sizes
	 */
	SL_INDEX_COUNT_LOG2 = 5,
};

/*
 * Private constants
 */
enum tlsf_private
{
#if defined(TLSF_64BIT)
	/*
	 * All allocation sizes and addresses are aligned to 16 bytes
	 */
	ALIGN_SIZE_LOG2 = 4,
#else
	/*
	 * All allocation sizes and addresses are aligned to 8 bytes
	 */
	ALIGN_SIZE_LOG2 = 3,
#endif
	ALIGN_SIZE = (1 << ALIGN_SIZE_LOG2),

#if defined(TLSF_64BIT)
	FL_INDEX_MAX = 32,
#else
	FL_INDEX_MAX = 30,
#endif
	SL_INDEX_COUNT = (1 << SL_INDEX_COUNT_LOG2),
	FL_INDEX_SHIFT = (SL_INDEX_COUNT_LOG2 + ALIGN_SIZE_LOG2),
	FL_INDEX_COUNT = (FL_INDEX_MAX - FL_INDEX_SHIFT + 1),

	SMALL_BLOCK_SIZE = (1 << FL_INDEX_SHIFT),
};

/*
 * Block header structure
 */
typedef struct block_header_t
{
	union
	{
		/*
		 * Points to the previous physical block
		 */
		struct block_header_t * prev_phys_block;
		char align_data[ALIGN_SIZE];
	};

	/*
	 * The size of this block, excluding the block header
	 */
	size_t size;

	/*
	 * Next and previous free blocks
	 */
	struct block_header_t * next_free;
	struct block_header_t * prev_free;
} block_header_t;

/*
 * The TLSF control structure.
 */
typedef struct control_t
{
	/*
	 * Empty lists point at this block to indicate they are free.
	 */
	block_header_t block_null;

	/*
	 * Bitmaps for free lists.
	 */
	unsigned int fl_bitmap;
	unsigned int sl_bitmap[FL_INDEX_COUNT];

	/*
	 * Head of free lists.
	 */
	block_header_t * blocks[FL_INDEX_COUNT][SL_INDEX_COUNT];
} control_t;

/*
 * A type used for casting when doing pointer arithmetic.
 */
typedef ptrdiff_t	tlsfptr_t;

/*
 * Associated constants
 */
static const size_t block_header_free_bit = 1 << 0;
static const size_t block_header_prev_free_bit = 1 << 1;
static const size_t block_header_overhead = sizeof(size_t);
static const size_t block_start_offset = offsetof(block_header_t, size) + sizeof(size_t);
static const size_t block_size_min = sizeof(block_header_t) - sizeof(block_header_t *);
static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX;

#if defined(__riscv)
static int tlsf_fls_generic(unsigned int word)
{
	int bit = 32;

	if (!word) bit -= 1;
	if (!(word & 0xffff0000)) { word <<= 16; bit -= 16; }
	if (!(word & 0xff000000)) { word <<= 8; bit -= 8; }
	if (!(word & 0xf0000000)) { word <<= 4; bit -= 4; }
	if (!(word & 0xc0000000)) { word <<= 2; bit -= 2; }
	if (!(word & 0x80000000)) { word <<= 1; bit -= 1; }

	return bit;
}

static int tlsf_ffs(unsigned int word)
{
	return tlsf_fls_generic(word & (~word + 1)) - 1;
}

static int tlsf_fls(unsigned int word)
{
	return tlsf_fls_generic(word) - 1;
}
#else
static int tlsf_ffs(unsigned int word)
{
	return __builtin_ffs(word) - 1;
}

static int tlsf_fls(unsigned int word)
{
	const int bit = word ? 32 - __builtin_clz(word) : 0;
	return bit - 1;
}
#endif

#if defined(TLSF_64BIT)
static int tlsf_fls_sizet(size_t size)
{
	int high = (int)(size >> 32);
	int bits = 0;
	if(high)
	{
		bits = 32 + tlsf_fls(high);
	}
	else
	{
		bits = tlsf_fls((int)size & 0xffffffff);

	}
	return bits;
}
#else
#define tlsf_fls_sizet		tlsf_fls
#endif

static size_t block_get_size(const block_header_t * block)
{
	return block->size & ~(block_header_free_bit | block_header_prev_free_bit);
}

static void block_set_size(block_header_t * block, size_t size)
{
	const size_t oldsize = block->size;
	block->size = size | (oldsize & (block_header_free_bit | block_header_prev_free_bit));
}

static int block_is_last(const block_header_t * block)
{
	return (block_get_size(block) == 0);
}

static int block_is_free(const block_header_t * block)
{
	return tlsf_cast(int, block->size & block_header_free_bit);
}

static void block_set_free(block_header_t * block)
{
	block->size |= block_header_free_bit;
}

static void block_set_used(block_header_t * block)
{
	block->size &= ~block_header_free_bit;
}

static int block_is_prev_free(const block_header_t * block)
{
	return tlsf_cast(int, block->size & block_header_prev_free_bit);
}

static void block_set_prev_free(block_header_t * block)
{
	block->size |= block_header_prev_free_bit;
}

static void block_set_prev_used(block_header_t * block)
{
	block->size &= ~block_header_prev_free_bit;
}

static block_header_t * block_from_ptr(const void * ptr)
{
	return tlsf_cast(block_header_t *, tlsf_cast(unsigned char*, ptr) - block_start_offset);
}

static void * block_to_ptr(const block_header_t * block)
{
	return tlsf_cast(void *, tlsf_cast(unsigned char*, block) + block_start_offset);
}

static block_header_t * offset_to_block(const void * ptr, size_t size)
{
	return tlsf_cast(block_header_t *, tlsf_cast(tlsfptr_t, ptr) + size);
}

static block_header_t * block_prev(const block_header_t * block)
{
	return block->prev_phys_block;
}

static block_header_t * block_next(const block_header_t * block)
{
	block_header_t * next = offset_to_block(block_to_ptr(block), block_get_size(block) - block_header_overhead);
	tlsf_assert(!block_is_last(block));
	return next;
}

static block_header_t * block_link_next(block_header_t * block)
{
	block_header_t * next = block_next(block);
	next->prev_phys_block = block;
	return next;
}

static void block_mark_as_free(block_header_t * block)
{
	block_header_t * next = block_link_next(block);
	block_set_prev_free(next);
	block_set_free(block);
}

static void block_mark_as_used(block_header_t * block)
{
	block_header_t * next = block_next(block);
	block_set_prev_used(next);
	block_set_used(block);
}

static size_t align_up(size_t x, size_t align)
{
	tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
	return (x + (align - 1)) & ~(align - 1);
}

static size_t align_down(size_t x, size_t align)
{
	tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
	return x - (x & (align - 1));
}

static void * align_ptr(const void * ptr, size_t align)
{
	const tlsfptr_t aligned = (tlsf_cast(tlsfptr_t, ptr) + (align - 1)) & ~(align - 1);
	tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
	return tlsf_cast(void*, aligned);
}

static size_t adjust_request_size(size_t size, size_t align)
{
	size_t adjust = 0;
	if(size)
	{
		const size_t aligned = align_up(size, align);
		if(aligned < block_size_max)
			adjust = tlsf_max(aligned, block_size_min);
	}
	return adjust;
}

static void mapping_insert(size_t size, int * fli, int * sli)
{
	int fl, sl;
	if(size < SMALL_BLOCK_SIZE)
	{
		fl = 0;
		sl = tlsf_cast(int, size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT);
	}
	else
	{
		fl = tlsf_fls_sizet(size);
		sl = tlsf_cast(int, size >> (fl - SL_INDEX_COUNT_LOG2)) ^ (1 << SL_INDEX_COUNT_LOG2);
		fl -= (FL_INDEX_SHIFT - 1);
	}
	*fli = fl;
	*sli = sl;
}

static void mapping_search(size_t size, int * fli, int * sli)
{
	if(size >= (1 << SL_INDEX_COUNT_LOG2))
	{
		const size_t round = (1 << (tlsf_fls_sizet(size) - SL_INDEX_COUNT_LOG2)) - 1;
		size += round;
	}
	mapping_insert(size, fli, sli);
}

static block_header_t * search_suitable_block(control_t * control, int * fli, int * sli)
{
	int fl = *fli;
	int sl = *sli;

	unsigned int sl_map = control->sl_bitmap[fl] & (~0U << sl);
	if(!sl_map)
	{
		const unsigned int fl_map = control->fl_bitmap & (~0 << (fl + 1));
		if(!fl_map)
		{
			return 0;
		}

		fl = tlsf_ffs(fl_map);
		*fli = fl;
		sl_map = control->sl_bitmap[fl];
	}
	tlsf_assert(sl_map && "internal error - second level bitmap is null");
	sl = tlsf_ffs(sl_map);
	*sli = sl;

	return control->blocks[fl][sl];
}

static void remove_free_block(control_t * control, block_header_t * block, int fl, int sl)
{
	block_header_t * prev = block->prev_free;
	block_header_t * next = block->next_free;
	tlsf_assert(prev && "prev_free field can not be null");
	tlsf_assert(next && "next_free field can not be null");
	next->prev_free = prev;
	prev->next_free = next;

	if(control->blocks[fl][sl] == block)
	{
		control->blocks[fl][sl] = next;

		if(next == &control->block_null)
		{
			control->sl_bitmap[fl] &= ~(1U << sl);

			if(!control->sl_bitmap[fl])
			{
				control->fl_bitmap &= ~(1U << fl);
			}
		}
	}
}

static void insert_free_block(control_t * control, block_header_t * block, int fl, int sl)
{
	block_header_t * current = control->blocks[fl][sl];
	tlsf_assert(current && "free list cannot have a null entry");
	tlsf_assert(block && "cannot insert a null entry into the free list");
	block->next_free = current;
	block->prev_free = &control->block_null;
	current->prev_free = block;

	tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE) && "block not aligned properly");

	control->blocks[fl][sl] = block;
	control->fl_bitmap |= (1U << fl);
	control->sl_bitmap[fl] |= (1U << sl);
}

static void block_remove(control_t * control, block_header_t * block)
{
	int fl, sl;
	mapping_insert(block_get_size(block), &fl, &sl);
	remove_free_block(control, block, fl, sl);
}

static void block_insert(control_t * control, block_header_t * block)
{
	int fl, sl;
	mapping_insert(block_get_size(block), &fl, &sl);
	insert_free_block(control, block, fl, sl);
}

static int block_can_split(block_header_t * block, size_t size)
{
	return block_get_size(block) >= sizeof(block_header_t) + size;
}

static block_header_t * block_split(block_header_t * block, size_t size)
{
	block_header_t * remaining = offset_to_block(block_to_ptr(block), size - block_header_overhead);
	const size_t remain_size = block_get_size(block) - (size + ALIGN_SIZE);

	tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE) && "remaining block not aligned properly");

	tlsf_assert(block_get_size(block) == remain_size + size + ALIGN_SIZE);
	block_set_size(remaining, remain_size);
	tlsf_assert(block_get_size(remaining) >= block_size_min && "block split with invalid size");

	block_set_size(block, size);
	block_mark_as_free(remaining);

	return remaining;
}

static block_header_t * block_absorb(block_header_t * prev, block_header_t * block)
{
	tlsf_assert(!block_is_last(prev) && "previous block can't be last!");
	prev->size += block_get_size(block) + ALIGN_SIZE;
	block_link_next(prev);
	return prev;
}

static block_header_t * block_merge_prev(control_t * control, block_header_t * block)
{
	if(block_is_prev_free(block))
	{
		block_header_t* prev = block_prev(block);
		tlsf_assert(prev && "prev physical block can't be null");
		tlsf_assert(block_is_free(prev) && "prev block is not free though marked as such");
		block_remove(control, prev);
		block = block_absorb(prev, block);
	}

	return block;
}

static block_header_t * block_merge_next(control_t * control, block_header_t * block)
{
	block_header_t* next = block_next(block);
	tlsf_assert(next && "next physical block can't be null");

	if(block_is_free(next))
	{
		tlsf_assert(!block_is_last(block) && "previous block can't be last!");
		block_remove(control, next);
		block = block_absorb(block, next);
	}

	return block;
}

static void block_trim_free(control_t * control, block_header_t * block, size_t size)
{
	tlsf_assert(block_is_free(block) && "block must be free");
	if(block_can_split(block, size))
	{
		block_header_t* remaining_block = block_split(block, size);
		block_link_next(block);
		block_set_prev_free(remaining_block);
		block_insert(control, remaining_block);
	}
}

static void block_trim_used(control_t * control, block_header_t * block, size_t size)
{
	tlsf_assert(!block_is_free(block) && "block must be used");
	if(block_can_split(block, size))
	{
		block_header_t* remaining_block = block_split(block, size);
		block_set_prev_used(remaining_block);

		remaining_block = block_merge_next(control, remaining_block);
		block_insert(control, remaining_block);
	}
}

static block_header_t * block_trim_free_leading(control_t * control, block_header_t * block, size_t size)
{
	block_header_t * remaining_block = block;
	if(block_can_split(block, size))
	{
		remaining_block = block_split(block, size - ALIGN_SIZE);
		block_set_prev_free(remaining_block);

		block_link_next(block);
		block_insert(control, block);
	}

	return remaining_block;
}

static block_header_t * block_locate_free(control_t * control, size_t size)
{
	int fl = 0, sl = 0;
	block_header_t * block = 0;

	if(size)
	{
		mapping_search(size, &fl, &sl);
		if(fl < FL_INDEX_COUNT)
			block = search_suitable_block(control, &fl, &sl);
	}

	if(block)
	{
		tlsf_assert(block_get_size(block) >= size);
		remove_free_block(control, block, fl, sl);
	}

	return block;
}

static void * block_prepare_used(control_t * control, block_header_t * block, size_t size)
{
	void * p = 0;
	if(block)
	{
		block_trim_free(control, block, size);
		block_mark_as_used(block);
		p = block_to_ptr(block);
	}
	return p;
}

static void control_construct(control_t * control)
{
	int i, j;

	control->block_null.next_free = &control->block_null;
	control->block_null.prev_free = &control->block_null;

	control->fl_bitmap = 0;
	for(i = 0; i < FL_INDEX_COUNT; ++i)
	{
		control->sl_bitmap[i] = 0;
		for(j = 0; j < SL_INDEX_COUNT; ++j)
		{
			control->blocks[i][j] = &control->block_null;
		}
	}
}

static inline void * tlsf_add_pool(void * tlsf, void * mem, size_t bytes)
{
	block_header_t * block;
	block_header_t * next;
	const size_t pool_bytes = align_down(bytes - block_header_overhead - ALIGN_SIZE, ALIGN_SIZE);

	if(((ptrdiff_t)mem % ALIGN_SIZE) != 0)
		return 0;

	if(pool_bytes < block_size_min || pool_bytes > block_size_max)
		return 0;

	block = offset_to_block(mem, -(tlsfptr_t)block_header_overhead);
	block_set_size(block, pool_bytes);
	block_set_free(block);
	block_set_prev_used(block);
	block_insert(tlsf_cast(control_t*, tlsf), block);

	next = block_link_next(block);
	block_set_size(next, 0);
	block_set_used(next);
	block_set_prev_free(next);

	return mem;
}

static inline void tlsf_remove_pool(void * tlsf, void * mem)
{
	control_t * control = tlsf_cast(control_t *, tlsf);
	block_header_t * block = offset_to_block(mem, -(int)block_header_overhead);
	int fl = 0, sl = 0;

	tlsf_assert(block_is_free(block) && "block should be free");
	tlsf_assert(!block_is_free(block_next(block)) && "next block should not be free");
	tlsf_assert(block_get_size(block_next(block)) == 0 && "next block size should be zero");

	mapping_insert(block_get_size(block), &fl, &sl);
	remove_free_block(control, block, fl, sl);
}

static inline void * tlsf_create(void * mem)
{
	if(((tlsfptr_t)mem % ALIGN_SIZE) != 0)
		return 0;

	control_construct(tlsf_cast(control_t *, mem));
	return tlsf_cast(void *, mem);
}

static inline void * tlsf_create_with_pool(void * mem, size_t bytes)
{
	void * tlsf = tlsf_create(mem);
	tlsf_add_pool(tlsf, (char *)mem + align_up(sizeof(control_t), ALIGN_SIZE), bytes - align_up(sizeof(control_t), ALIGN_SIZE));
	return tlsf;
}

static inline void tlsf_destroy(void * mem)
{
	(void)mem;
}

static inline void * tlsf_get(void * mem)
{
	return tlsf_cast(void *, (char *)mem + align_up(sizeof(control_t), ALIGN_SIZE));
}

static inline void * tlsf_malloc(void * tlsf, size_t size)
{
	control_t * control = tlsf_cast(control_t *, tlsf);
	const size_t adjust = adjust_request_size(size, ALIGN_SIZE);
	block_header_t * block = block_locate_free(control, adjust);
	return block_prepare_used(control, block, adjust);
}

static inline void * tlsf_memalign(void * tlsf, size_t align, size_t size)
{
	control_t * control = tlsf_cast(control_t *, tlsf);
	const size_t adjust = adjust_request_size(size, ALIGN_SIZE);

	const size_t gap_minimum = sizeof(block_header_t);
	const size_t size_with_gap = adjust_request_size(adjust + align + gap_minimum, align);

	const size_t aligned_size = (adjust && align > ALIGN_SIZE) ? size_with_gap : adjust;

	block_header_t* block = block_locate_free(control, aligned_size);

	tlsf_assert(sizeof(block_header_t) == block_size_min + block_header_overhead);

	if(block)
	{
		void * ptr = block_to_ptr(block);
		void * aligned = align_ptr(ptr, align);
		size_t gap = tlsf_cast(size_t, tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr));

		if(gap && (gap < gap_minimum))
		{
			const size_t gap_remain = gap_minimum - gap;
			const size_t offset = tlsf_max(gap_remain, align);
			const void * next_aligned = tlsf_cast(void *, tlsf_cast(tlsfptr_t, aligned) + offset);

			aligned = align_ptr(next_aligned, align);
			gap = tlsf_cast(size_t, tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr));
		}

		if(gap)
		{
			tlsf_assert(gap >= gap_minimum && "gap size too small");
			block = block_trim_free_leading(control, block, gap);
		}
	}

	return block_prepare_used(control, block, adjust);
}

static inline void tlsf_free(void * tlsf, void * ptr)
{
	if(ptr)
	{
		control_t * control = tlsf_cast(control_t *, tlsf);
		block_header_t * block = block_from_ptr(ptr);
		tlsf_assert(!block_is_free(block) && "block already marked as free");
		block_mark_as_free(block);
		block = block_merge_prev(control, block);
		block = block_merge_next(control, block);
		block_insert(control, block);
	}
}

static inline void * tlsf_realloc(void * tlsf, void * ptr, size_t size)
{
	control_t * control = tlsf_cast(control_t *, tlsf);
	void * p = 0;

	if(ptr && (size == 0))
	{
		tlsf_free(tlsf, ptr);
	}
	else if(!ptr)
	{
		p = tlsf_malloc(tlsf, size);
	}
	else
	{
		block_header_t * block = block_from_ptr(ptr);
		block_header_t * next = block_next(block);

		const size_t cursize = block_get_size(block);
		const size_t combined = cursize + block_get_size(next) + block_header_overhead;
		const size_t adjust = adjust_request_size(size, ALIGN_SIZE);

		tlsf_assert(!block_is_free(block) && "block already marked as free");

		if((adjust > cursize) && (!block_is_free(next) || adjust > combined))
		{
			p = tlsf_malloc(tlsf, size);
			if(p)
			{
				const size_t minsize = tlsf_min(cursize, size);
				memcpy(p, ptr, minsize);
				tlsf_free(tlsf, ptr);
			}
		}
		else
		{
			if(adjust > cursize)
			{
				block_merge_next(control, block);
				block_mark_as_used(block);
			}

			block_trim_used(control, block, adjust);
			p = ptr;
		}
	}

	return p;
}

static inline void tlsf_info(void * tlsf, size_t * mused, size_t * mfree)
{
	block_header_t * block = offset_to_block(tlsf, -(int)block_header_overhead);

	*mused = 0;
	*mfree = 0;
	while(block && !block_is_last(block))
	{
		if(block_is_free(block))
			*mfree += block_get_size(block);
		else
			*mused += block_get_size(block);
		block = block_next(block);
	}
}

void * mm_create(void * mem, size_t bytes)
{
	return tlsf_create_with_pool(mem, bytes);
}

void mm_destroy(void * mem)
{
	tlsf_destroy(mem);
}

void * mm_get(void * mem)
{
	return tlsf_get(mem);
}

void * mm_add_pool(void * mm, void * mem, size_t bytes)
{
	return tlsf_add_pool(mm, mem, bytes);
}

void mm_remove_pool(void * mm, void * mem)
{
	tlsf_remove_pool(mm, mem);
}

void * mm_malloc(void * mm, size_t size)
{
	return tlsf_malloc(mm, size);
}

void * mm_memalign(void * mm, size_t align, size_t size)
{
	return tlsf_memalign(mm, align, size);
}

void * mm_realloc(void * mm, void * ptr, size_t size)
{
	return tlsf_realloc(mm, ptr, size);
}

void mm_free(void * mm, void * ptr)
{
	tlsf_free(mm, ptr);
}

void mm_info(void * mm, size_t * mused, size_t * mfree)
{
	if(mused && mfree)
		tlsf_info(mm, mused, mfree);
}

static void * __heap_pool = NULL;
static spinlock_t __heap_lock = SPIN_LOCK_INIT();

static void * __malloc(size_t size)
{
	void * m;

	if(__heap_pool)
	{
		spin_lock(&__heap_lock);
		m = tlsf_malloc(__heap_pool, size);
		spin_unlock(&__heap_lock);
		return m;
	}
	return NULL;
}
extern __typeof(__malloc) malloc __attribute__((weak, alias("__malloc")));

static void * __memalign(size_t align, size_t size)
{
	void * m;

	if(__heap_pool)
	{
		spin_lock(&__heap_lock);
		m = tlsf_memalign(__heap_pool, align, size);
		spin_unlock(&__heap_lock);
		return m;
	}
	return NULL;
}
extern __typeof(__memalign) memalign __attribute__((weak, alias("__memalign")));

static void * __realloc(void * ptr, size_t size)
{
	void * m;

	if(__heap_pool)
	{
		spin_lock(&__heap_lock);
		m = tlsf_realloc(__heap_pool, ptr, size);
		spin_unlock(&__heap_lock);
		return m;
	}
	return NULL;
}
extern __typeof(__realloc) realloc __attribute__((weak, alias("__realloc")));

static void * __calloc(size_t nmemb, size_t size)
{
	void * m;

	if((m = malloc(nmemb * size)))
		memset(m, 0, nmemb * size);
	return m;
}
extern __typeof(__calloc) calloc __attribute__((weak, alias("__calloc")));

static void __free(void * ptr)
{
	if(__heap_pool)
	{
		spin_lock(&__heap_lock);
		tlsf_free(__heap_pool, ptr);
		spin_unlock(&__heap_lock);
	}
}
extern __typeof(__free) free __attribute__((weak, alias("__free")));

static void __meminfo(size_t * mused, size_t * mfree)
{
	if(__heap_pool)
	{
		if(mused && mfree)
			tlsf_info(mm_get(__heap_pool), mused, mfree);
	}
}
extern __typeof(__meminfo) meminfo __attribute__((weak, alias("__meminfo")));

static struct kobj_t * search_class_memory_kobj(void)
{
	struct kobj_t * kclass = kobj_search_directory_with_create(kobj_get_root(), "class");
	return kobj_search_directory_with_create(kclass, "memory");
}

static ssize_t memory_read_meminfo(struct kobj_t * kobj, void * buf, size_t size)
{
	size_t mused = 0;
	size_t mfree = 0;
	char * p = buf;
	int len = 0;

	meminfo(&mused, &mfree);
	len += sprintf((char *)(p + len), " memory used: %ld\r\n", mused);
	len += sprintf((char *)(p + len), " memory free: %ld\r\n", mfree);
	return len;
}

void do_init_mem(void)
{
#ifndef __SANDBOX__
	extern unsigned char __heap_start[];
	extern unsigned char __heap_end[];
	spin_lock_init(&__heap_lock);
	__heap_pool = mm_create((void *)__heap_start, (size_t)(__heap_end - __heap_start));
#endif
	kobj_add_regular(search_class_memory_kobj(), "meminfo", memory_read_meminfo, NULL, NULL);
}


text:
package org.chocosolver.solver.constraints.unary;

import gnu.trove.map.hash.THashMap;
import org.chocosolver.solver.Solver;
import org.chocosolver.solver.constraints.Propagator;
import org.chocosolver.solver.constraints.PropagatorPriority;
import org.chocosolver.solver.exception.ContradictionException;
import org.chocosolver.solver.explanations.Deduction;
import org.chocosolver.solver.explanations.Explanation;
import org.chocosolver.solver.explanations.ExplanationEngine;
import org.chocosolver.solver.variables.IntVar;
import org.chocosolver.solver.variables.events.IntEventType;
import org.chocosolver.util.ESat;

/**
 * X >= C
 * <br/>
 *
 * @author Charles Prud'homme
 * @since 16/06/11
 */
public class PropGreaterOrEqualXC extends Propagator<IntVar> {

    private final int constant;

    public PropGreaterOrEqualXC(IntVar var, int cste) {
        super(new IntVar[]{var}, PropagatorPriority.UNARY, false);
        this.constant = cste;
    }

    @Override
    public int getPropagationConditions(int vIdx) {
        return IntEventType.boundAndInst();
    }

    @Override
    public void propagate(int evtmask) throws ContradictionException {
        // with views such as abs(...), the prop can be not entailed after initial propagation
        if (vars[0].updateLowerBound(constant, aCause) || vars[0].getLB() >= constant) {
            this.setPassive();
        }
    }

    @Override
    public ESat isEntailed() {
        if (vars[0].getLB() >= constant) {
            return ESat.TRUE;
        } else if (vars[0].getUB() < constant) {
            return ESat.FALSE;
        }
        return ESat.UNDEFINED;
    }

    @Override
    public String toString() {
        return vars[0].getName() + " >= " + constant;
    }

    @Override
    public void explain(ExplanationEngine xengine, Deduction d, Explanation e) {
        e.add(xengine.getPropagatorActivation(this));
    }

    @Override
    public void duplicate(Solver solver, THashMap<Object, Object> identitymap) {
        if (!identitymap.containsKey(this)) {
            identitymap.put(this, new PropGreaterOrEqualXC((IntVar) identitymap.get(vars[0]), constant));
        }
    }
}


text:
The keshlib for iOS uses code from the following libraries

* socket.io-objc by Philipp Kyeck https://github.com/pkyeck/socket.IO-objc/
* SocketRocket by Square Inc. https://github.com/square/SocketRocket/
* Reachability by Tony Million https://github.com/tonymillion/Reachability/

In addition, the keshlib-ios-demo uses code from the following library

* MF_Base64Additions by Dave Poirier https://github.com/ekscrypto/Base64/ (public domain)

## Full licenses

socket.io-objc, MIT license
```
Copyright (c) 2011-12 Philipp Kyeck <http://beta-interactive.de>

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
```

SocketRocket, Apache License V2.0
```
Copyright 2012 Square Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
```

Reachability
```
Copyright (c) 2011-2013, Tony Million. All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
```


text:
module Spree
  module PagesHelper

    def pages_list_link(params)
      page = Spree::StaticPage.published.order("position ASC")
      items = page.map do |pg|
        content_tag(:li, link_to(pg.name, page_path("#{pg.slug}")))
      end

      content_tag(:ul, raw(items.join("\n")), :class => params[:class])
    end

  end
end


text:
package org.jboss.hal.ballroom.table;

import org.jboss.hal.ballroom.table.SelectorModifier.Order;
import org.jboss.hal.ballroom.table.SelectorModifier.Page;
import org.jboss.hal.ballroom.table.SelectorModifier.Search;

/** Builder for a {@link SelectorModifier}. */
class SelectorModifierBuilder {

    private Order order;
    private Page page;
    private Search search;
    private Boolean selected;

    SelectorModifierBuilder() {
        this.order = Order.current;
        this.page = Page.all;
        this.search = Search.none;
        this.selected = null;
    }

    SelectorModifierBuilder order(Order order) {
        this.order = order;
        return this;
    }

    SelectorModifierBuilder page(Page page) {
        this.page = page;
        return this;
    }

    SelectorModifierBuilder search(Search search) {
        this.search = search;
        return this;
    }

    SelectorModifierBuilder selected() {
        this.selected = true;
        return this;
    }

    SelectorModifierBuilder unselected() {
        this.selected = false;
        return this;
    }

    SelectorModifier build() {
        SelectorModifier selectorModifier = new SelectorModifier();
        selectorModifier.order = order.name();
        selectorModifier.page = page.name();
        selectorModifier.search = search.name();
        if (selected != null) {
            selectorModifier.selected = selected;
        }
        return selectorModifier;
    }
}


text:
package jp.fieldnotes.hatunatu.dao.resultset;

import jp.fieldnotes.hatunatu.api.PropertyType;
import jp.fieldnotes.hatunatu.dao.jdbc.QueryObject;

import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

public class BeanListResultSetHandler extends AbstractBeanResultSetHandler {

    /**
     * {@link BeanListResultSetHandler}を作成します。
     *
     * @param beanClass
     *            Beanクラス
     */
    public BeanListResultSetHandler(Class beanClass) {
        super(beanClass);
    }

    @Override
    public Object handle(ResultSet rs, QueryObject queryObject) throws SQLException {
        PropertyType[] propertyTypes = createPropertyTypes(rs.getMetaData());
        List list = new ArrayList();
        while (rs.next()) {
            Object row = createRow(rs, propertyTypes);
            list.add(row);
        }
        return list;
    }
}


text:
package scriptella.util;

import scriptella.AbstractTestCase;

import java.io.ByteArrayInputStream;
import java.io.IOException;

/**
 * Tests for {@link PropertiesMap}.
 *
 * @author Fyodor Kupolov
 * @version 1.0
 */
public class PropertiesMapTest extends AbstractTestCase {
    public void testLoad() throws IOException {
        PropertiesMap pm = new PropertiesMap();
        pm.load(new ByteArrayInputStream("p1=1\np1=11\np2=2".getBytes()));
        assertEquals(2, pm.size());
        assertEquals("1", pm.get("p1"));
        assertEquals("2", pm.get("p2"));
        assertEquals("p1", pm.keySet().iterator().next());
        //Now test the stream based constructor
        pm = new PropertiesMap(new ByteArrayInputStream("p=1".getBytes()));
        assertEquals("1", pm.get("p"));
    }

    public void testPut() throws IOException {
        PropertiesMap pm = new PropertiesMap();
        pm.put("p2", "2");
        pm.put("p2", "");
        pm.put("p1", "1");
        assertEquals(2, pm.size());
        assertEquals("1", pm.get("p1"));
        assertEquals("2", pm.get("p2"));
        //Insertion order is preserved
        assertEquals("p2", pm.keySet().iterator().next());
        //Check if remove is working
        pm.remove("p1");
        pm.put("p1", "");
        assertEquals("", pm.get("p1"));

    }

}


text:
import gql from "lib/gql"
import { runAuthenticatedQuery } from "schema/v2/test/utils"
import { assign } from "lodash"

describe("Auction Context", () => {
  let context: any
  const parentArtwork = {}

  const query = gql`
    {
      artwork(id: "donn-delson-space-invader") {
        contextGrids {
          title
          ctaTitle
          ctaHref
          artworksConnection(first: 2) {
            edges {
              node {
                slug
                title
              }
            }
          }
        }
      }
    }
  `

  beforeEach(() => {
    assign(parentArtwork, {
      _id: "abc123",
      id: "parentArtwork",
      title: "the Parent artwork",
      artist: {
        id: "andy-warhol",
        name: "Andy Warhol",
        forsale_artworks_count: 123,
      },
      partner: {
        id: "cama-gallery",
        name: "CAMA Gallery",
      },
      sale_ids: ["phillips-auction"],
    })

    const artistArtworks = [
      { id: "artwork1", title: "Artwork 1" },
      { id: "artwork2", title: "Artwork 2" },
      { id: "artwork3", title: "Artwork 3" },
    ]

    const partnerArtworks = [
      { id: "partnerArtwork1", title: "Partner Artwork 1" },
      { id: "partnerArtwork2", title: "Partner Artwork 2" },
      { id: "partnerArtwork3", title: "Partner Artwork 3" },
    ]

    const saleArtworks = [
      { id: "saleArtwork1", title: "Sale Artwork 1" },
      { id: "saleArtwork2", title: "Sale Artwork 2" },
      { id: "saleArtwork3", title: "Sale Artwork 3" },
    ]

    context = {
      artworkLoader: () => Promise.resolve(parentArtwork),
      artistArtworksLoader: () => Promise.resolve(artistArtworks),
      relatedFairsLoader: () => Promise.resolve(null),
      relatedShowsLoader: () => Promise.resolve(null),
      partnerArtworksLoader: () => {
        return Promise.resolve({
          body: partnerArtworks,
          headers: { "x-total-count": "10" },
        })
      },
      relatedLayerArtworksLoader: () => Promise.resolve(null),
      relatedLayersLoader: () => Promise.resolve([]),
      saleLoader: () => {
        return Promise.resolve({
          id: "phillips-auction",
          is_auction: true,
          auction_state: "open",
          eligible_sale_artworks_count: 10,
          name: "Phillips Auction",
        })
      },
      saleArtworksLoader: () => {
        return Promise.resolve({
          body: saleArtworks,
        })
      },
    }
  })

  it("Does not return auction-related grids for a non-auction sale", async () => {
    expect.assertions(1)
    context.saleLoader = () =>
      Promise.resolve({
        id: "phillips-auction",
        is_auction: false,
        auction_state: "closed",
      })

    const data = await runAuthenticatedQuery(query, context)
    // Returns the default grid
    expect(data.artwork.contextGrids.length).toEqual(3)
  })

  it("Returns the correct values for metadata fields for an open auction", async () => {
    const data = await runAuthenticatedQuery(query, context)
    // Should have one artist grid and one related grid with 0 works
    expect(data.artwork.contextGrids.length).toEqual(1)
    const {
      title,
      ctaTitle,
      ctaHref,
      artworksConnection,
    } = data.artwork.contextGrids[0]
    expect(title).toEqual("Other works from Phillips Auction")
    expect(ctaTitle).toEqual("View all works from the auction")
    expect(ctaHref).toEqual("/auction/phillips-auction")
    expect(artworksConnection.edges.length).toEqual(2)
  })

  it("Returns the correct values for metadata fields for a closed auction", async () => {
    context.saleLoader = () =>
      Promise.resolve({
        id: "phillips-auction",
        is_auction: true,
        auction_state: "closed",
      })

    const data = await runAuthenticatedQuery(query, context)
    // Should have one partner grid and one related grid with 0 works
    expect(data.artwork.contextGrids.length).toEqual(2)
    const {
      title,
      ctaTitle,
      ctaHref,
      artworksConnection,
    } = data.artwork.contextGrids[0]
    expect(title).toEqual("Other works by Andy Warhol")
    expect(ctaTitle).toEqual("View all works by Andy Warhol")
    expect(ctaHref).toEqual("/artist/andy-warhol")
    expect(artworksConnection.edges.map(({ node }) => node.slug)).toEqual([
      "artwork1",
      "artwork2",
    ])
    // Related artworks grid should have no artworks
    expect(data.artwork.contextGrids[1].artworksConnection).toEqual(null)
  })

  it("allows related artworks to be excluded", async () => {
    context.relatedLayersLoader = () => Promise.resolve([{ id: "main" }])
    context.relatedLayerArtworksLoader = jest.fn()

    const response = await runAuthenticatedQuery(
      gql`
        {
          artwork(id: "donn-delson-space-invader") {
            contextGrids(includeRelatedArtworks: false) {
              title
              artworksConnection(first: 2) {
                edges {
                  node {
                    slug
                    title
                  }
                }
              }
            }
          }
        }
      `,
      context
    )

    const grids = response.artwork.contextGrids
    const gridTitles = grids.map((grid) => grid.title)

    expect(grids.length).toEqual(1)
    expect(gridTitles).not.toInclude("Related works")
    expect(context.relatedLayerArtworksLoader).not.toHaveBeenCalled()
  })
})


text:
import {assert} from 'chai';
import ltrim from '../src/ltrim.js';

describe('#ltrim', () => {
  it('should trim all whitespace from only the left side of a string', () => {
    assert.equal(ltrim('  Hello there'), 'Hello there');
    assert.equal(ltrim('Hello there  '), 'Hello there  ');
    assert.equal(ltrim('Hello  there'), 'Hello  there');
  });
});


text:
var Base = require('./base.js')
var Spawn = require('./spawn.js')
var Stdin = require('./stdin.js')
var Deferred = require('trivial-deferred')
var Pool = require('yapool')
var TestPoint = require('./point.js')
var parseTestArgs = require('./parse-test-args.js')
var loop = require('function-loop')

var extraFromError = require('./extra-from-error.js')
var stack = require('./stack.js')
var assert = require('assert')
var util = require('util')
util.inherits(Test, Base)
var ownOr = require('own-or')
var ownOrEnv = require('own-or-env')
var tapAsserts = require('./asserts.js')
var Promise = require('bluebird')
var bindObj = require('bind-obj-methods')

// A sigil object for implicit end() calls that should not
// trigger an error if the user then calls t.end()
var IMPLICIT = {}

// Sigil to put in the queue to signal the end of all things
var EOF = { EOF: true }

function hasOwn (obj, key) {
  return Object.prototype.hasOwnProperty.call(obj, key)
}

module.exports = Test

function Test (options) {
  options = options || {}
  if (!(this instanceof Test))
    return new Test(options)

  Base.call(this, options)
  this.pushedEnd = false
  this.jobs = ownOr(options, 'jobs', 1)
  this.subtests = []
  this.pool = new Pool()
  this.queue = ['TAP version 13\n']
  this.noparallel = false
  this.cb = this.domain.bind(options.cb)
  this.occupied = false
  this.currentAssert = null
  this.count = 0
  this.n = 0
  this.ended = false
  this.explicitEnded = false
  this.multiEndThrew = false
  this.currentAssert = null
  this.assertAt = null
  this.assertStack = null
  this.planEnd = -1
  this.onBeforeEach = []
  this.onAfterEach = []
  this.ranAfterEach = false

  // bind all methods to this object, so we can pass t.end as a callback
  // and do `var test = require('tap').test` like people do.
  var bound = Object.create(null)
  bindObj(this, this, bound)
  bindObj(this, Object.getPrototypeOf(this), bound)
  bindObj(this, Test.prototype, bound)
}

Test.prototype.current = function () {
  throw new Error('Test.current() as been removed and is no more')
}

Test.prototype.spawn = function spawn (cmd, args, options, name) {
  if (typeof args === 'string') {
    args = [ args ]
  }

  args = args || []

  if (typeof options === 'string') {
    name = options
    options = {}
  }

  options = options || {}
  options.name = ownOr(options, 'name', name)
  options.command = cmd
  options.args = args

  return this.sub(Spawn, options, spawn)
}

Test.prototype.sub = function (Class, extra, caller) {
  extra = extra || {}
  if (!extra.skip && this.grep.length) {
    var match = this.grep[0].test(extra.name)
    if (this.grepInvert) {
      match = !match
    }
    if (!match) {
      var p = 'filter' + (this.grepInvert ? ' out' : '') + ': '
      extra.skip = p + this.grep[0]
    }
  }

  if (extra.only && !this.runOnly) {
    this.comment('%j has `only` set but all tests run', extra.name)
  }

  if (this.runOnly && !extra.only) {
    extra.skip = 'filter: only'
  }

  if (extra.todo || extra.skip) {
    this.pass(extra.name, extra)
    return Promise.resolve(this)
  }

  if (!extra.grep) {
    extra.grep = this.grep.slice(1)
    extra.grepInvert = this.grepInvert
  }

  extra.indent = '    '
  if (this.jobs > 1 && process.env.TAP_BUFFER === undefined)
    extra.buffered = ownOr(extra, 'buffered', true)
  else
    extra.buffered = ownOrEnv(extra, 'buffered', 'TAP_BUFFER', true)

  extra.bail = ownOr(extra, 'bail', this.bail)
  extra.parent = this
  extra.stack = stack.captureString(80, caller)
  var t = new Class(extra)

  this.queue.push(t)
  this.subtests.push(t)

  var d = new Deferred()
  t.deferred = d
  this.process()
  return d.promise
}

Test.prototype.only = function test (name, extra, cb) {
  extra = parseTestArgs(name, extra, cb)
  extra.only = true
  return this.sub(Test, extra, test)
}

Test.prototype.test = function test (name, extra, cb) {
  extra = parseTestArgs(name, extra, cb)
  return this.sub(Test, extra, test)
}

Test.prototype.stdin = function stdin (name, extra) {
  extra = parseTestArgs(name, extra, function () {}, '/dev/stdin')
  return this.sub(Stdin, extra || {}, stdin)
}

Test.prototype.bailout = function (message) {
  if (this.parent && (this.results || this.ended))
    this.parent.bailout(message)
  else {
    this.process()
    message = message ? ' ' + ('' + message).trim() : ''
    message = message.replace(/[\r\n]/g, ' ')
    this.parser.write('Bail out!' + message + '\n')
  }
  this.end(IMPLICIT)
  this.process()
}

Test.prototype.comment = function () {
  var message = util.format.apply(util, arguments)
  message = '# ' + message.split(/\r?\n/).join('\n# ') + '\n'

  if (this.results)
    this.push(message)
  else
    this.queue.push(message)
  this.process()
}

Test.prototype.timeout = function (options) {
  options = options || {}
  options.expired = options.expired || this.name
  if (this.occupied)
    this.occupied.timeout(options)
  else
    Base.prototype.timeout.call(this, options)
  this.end(IMPLICIT)
}

Test.prototype.main = function (cb) {
  this.setTimeout(this.options.timeout)
  this.debug('MAIN pre', this)

  var self = this
  try {
    var ret = this.cb(this)
  } catch (er) {
    this.threw(er)
  }

  if (ret && ret.then) {
    this.promise = ret
    ret.tapAbortPromise = done
    ret.then(end, done)
  } else
    done()

  function end () {
    self.debug(' > implicit end for promise')
    self.end(IMPLICIT)
    done()
  }

  function done (er) {
    if (er)
      self.threw(er)

    if (self.results || self.bailedOut)
      cb()
    else
      self.ondone = cb
  }

  this.debug('MAIN post', this)
}

Test.prototype.process = function () {
  if (this.processing)
    return this.debug(' < already processing')

  this.debug('\nPROCESSING(%s)', this.name, this.queue.length)
  this.processing = true

  var p

  while (!this.occupied && (p = this.queue.shift())) {
    this.debug('PROCESS(%s)', this.name, p)
    if (p instanceof Base) {
      this.processSubtest(p)
    } else if (p === EOF) {
      this.debug(' > EOF', this.name)
      // I AM BECOME EOF, DESTROYER OF STREAMS
      this.parser.end()
    } else if (p instanceof TestPoint) {
      this.debug(' > TESTPOINT')
      this.parser.write(p.ok + (++this.n) + p.message)
    } else if (typeof p === 'string') {
      this.debug(' > STRING')
      this.parser.write(p)
    } else if (Array.isArray(p)) {
      this.debug(' > METHOD')
      var m = p.shift()
      this[m].apply(this, p)
    } else {
      throw new Error('weird thing got in the queue')
    }
  }

  while (!this.noparallel &&
         this.pool.length < this.jobs &&
         (p = this.subtests.shift())) {
    if (!p.buffered) {
      this.noparallel = true
      break
    }
    this.debug('start subtest', p)
    this.pool.add(p)
    if (this.bailedOut)
      this.onbufferedend(p)
    else
      this.runBeforeEach(p,
        p.main.bind(p,
          this.onbufferedend.bind(this, p)))
  }

  this.debug('done processing', this.queue, this.occupied)
  this.processing = false

  // just in case any tests ended, and we have sync stuff still
  // waiting around in the queue to be processed
  if (!this.occupied && this.queue.length)
    this.process()

  this.maybeAutoend()
}

Test.prototype.processSubtest = function (p) {
  this.debug(' > subtest')
  this.occupied = p
  if (!p.buffered) {
    if (this.bailedOut)
      return this.onindentedend(p)
    this.debug(' > subtest indented')
    p.pipe(this.parser, { end: false })
    this.runBeforeEach(p,
      this.writeSubComment.bind(this, p,
        p.main.bind(p,
          this.onindentedend.bind(this, p))))
  } else if (p.readyToProcess) {
    this.debug(' > subtest buffered, finished')
    // finished!  do the thing!
    this.occupied = null
    if (!p.passing() || !p.silent) {
      this.queue.unshift(['emitSubTeardown', p])
      this.printResult(p.passing(), p.name, p.options, true)
    }
  } else {
    this.occupied = p
    this.debug(' > subtest buffered, unfinished', p)
    // unfinished buffered test.
    // nothing to do yet, just leave it there.
    this.queue.unshift(p)
  }
}

Test.prototype.emitSubTeardown = function (p) {
  try {
    p.emit('teardown')
  } catch (er) {
    delete p.options.time
    p.threw(er)
  }
}

Test.prototype.writeSubComment = function (p, cb) {
  var comment = '# Subtest'
  if (p.name)
    comment += ': ' + p.name
  comment += '\n'
  this.parser.write(comment)
  cb()
}

Test.prototype.onbufferedend = function (p, er) {
  delete p.ondone
  p.results = p.results || {}
  p.readyToProcess = true
  var to = p.options.timeout
  if (to && p.passing())
    var dur = Date.now() - p.start
  if (dur && dur > to)
    p.timeout()
  else
    p.setTimeout(false)
  this.debug('%s.onbufferedend', this.name, p.name, p.results.bailout)
  this.pool.remove(p)
  p.options.tapChildBuffer = p.output || ''
  p.options.stack = ''
  if (p.time)
    p.options.time = p.time
  if (this.occupied === p)
    this.occupied = null
  if (er)
    this.threw(er)
  p.deferred.resolve(this)
  this.process()
}

Test.prototype.onindentedend = function (p, er) {
  delete p.ondone
  this.debug('onindentedend', p)
  this.noparallel = false
  var sti = this.subtests.indexOf(p)
  if (sti !== -1)
    this.subtests.splice(sti, 1)
  p.readyToProcess = true
  p.results = p.results || {}
  if (p.time)
    p.options.time = p.time
  var to = p.options.timeout
  if (to && p.passing())
    var dur = Date.now() - p.start
  if (dur && dur > to)
    p.timeout()
  else
    p.setTimeout(false)
  this.debug('onindentedend %s(%s)', this.name, p.name, er || 'ok')
  assert(this.occupied === p)
  this.occupied = null
  this.debug('OIE(%s) b>shift into queue', this.name, this.queue)
  p.options.stack = ''

  this.queue.unshift(['emitSubTeardown', p])
  this.printResult(p.passing(), p.name, p.options, true)

  this.debug('OIE(%s) shifted into queue', this.name, this.queue)
  if (er)
    this.threw(er)
  p.deferred.resolve(this)
  this.process()
}

Test.prototype.addAssert = function (name, length, fn) {
  if (!name)
    throw new TypeError('name is required for addAssert')

  if (!(typeof length === 'number' && length >= 0))
    throw new TypeError('number of args required')

  if (typeof fn !== 'function')
    throw new TypeError('function required for addAssert')

  if (Test.prototype[name] || this[name])
    throw new TypeError('attempt to re-define `' + name + '` assert')

  this[name] = function ASSERT () {
    if (!this.currentAssert) {
      this.currentAssert = ASSERT
    }
    var args = new Array(length + 2)
    for (var i = 0; i < length; i++) {
      args[i] = arguments[i]
    }
    if (typeof arguments[length] === 'object') {
      args[length] = ''
      args[length + 1] = arguments[length]
    } else {
      args[length] = arguments[length] || ''
      args[length + 1] = arguments[length + 1] || {}
    }

    return fn.apply(this, args)
  }
}

Test.prototype.fail = function fail (message, extra) {
  if (!this.currentAssert) {
    this.currentAssert = fail
  }

  if (message && typeof message === 'object') {
    extra = message
    message = ''
  } else {
    if (!message) {
      message = ''
    }
    if (!extra) {
      extra = {}
    }
  }

  this.printResult(false, message, extra)

  var ret = true
  if (!extra.todo && !extra.skip)
    ret = false

  return ret
}

Test.prototype.pass = function pass (message, extra) {
  if (!this.currentAssert) {
    this.currentAssert = pass
  }
  this.printResult(true, message || '(unnamed test)', extra)
  return true
}

Test.prototype.printResult = function pR (ok, message, extra, front) {
  var n = this.count + 1
  if (this.planEnd !== -1 && n > this.planEnd) {
    if (!this.passing())
      return

    var failMessage = this.explicitEnded
        ? 'test after end() was called'
        : 'test count exceeds plan'

    var er = new Error(failMessage)
    Error.captureStackTrace(er, this.currentAssert || pR)
    er.test = this.name
    er.plan = this.planEnd
    this.threw(er)
    return
  }

  extra = extra || {}

  if (this.assertAt) {
    extra.at = this.assertAt
    this.assertAt = null
  }

  if (this.assertStack) {
    extra.stack = this.assertStack
    this.assertStack = null
  }

  if (hasOwn(extra, 'stack') && !hasOwn(extra, 'at'))
    extra.at = stack.parseLine(extra.stack.split('\n')[0])

  var fn = this.currentAssert || pR
  this.currentAssert = null
  if (!ok && !extra.skip && !hasOwn(extra, 'at')) {
    assert.equal(typeof fn, 'function')
    extra.at = stack.at(fn)
    if (!extra.todo)
      extra.stack = stack.captureString(80, fn)
  }

  var diagnostic
  if (!ok)
    diagnostic = true

  if (extra.skip)
    diagnostic = false

  if (process.env.TAP_DIAG === '0')
    diagnostic = false

  if (typeof extra.diagnostic === 'boolean')
    diagnostic = extra.diagnostic

  if (diagnostic)
    extra.diagnostic = true

  this.count = n
  var res = { ok: ok, message: message, extra: extra }
  var output = new TestPoint(ok, message, extra)
  // when we jump the queue, skip an extra line
  if (front)
    output.message = output.message.trimRight() + '\n\n'

  if (front) {
    this.emit('result', res)
    this.parser.write(output.ok + (++this.n) + output.message)
  } else
    this.queue.push(['emit', 'result', res], output)

  if (this.planEnd === this.count)
    this.end(IMPLICIT)

  this.process()
}

Test.prototype.pragma = function (set) {
  var p = ''
  Object.keys(set).forEach(function (i) {
    p += 'pragma ' + (set[i] ? '+' : '-') + i + '\n'
  })
  this.queue.push(p)
  this.process()
}

Test.prototype.plan = function (n, comment) {
  if (this.bailedOut)
    return

  if (this.planEnd !== -1) {
    throw new Error('Cannot set plan more than once')
  }

  if (typeof n !== 'number' || n < 0) {
    throw new TypeError('plan must be a number')
  }

  // Cannot get any tests after a trailing plan, or a plan of 0
  var ending = false
  if (this.count !== 0 || n === 0) {
    ending = true
  }

  if (n === 0)
    this.skip = comment || true

  this.planEnd = n
  comment = comment ? ' # ' + comment.trim() : ''
  this.queue.push('1..' + n + comment + '\n')

  if (ending)
    this.end(IMPLICIT)
  else
    this.process()
}

Test.prototype.done = Test.prototype.end = function (implicit) {
  this.debug('END implicit=%j', implicit === IMPLICIT)
  if (this.ended && implicit === IMPLICIT)
    return

  // beyond here we have to be actually done with things, or else
  // the semantic checks on counts and such will be off.
  if (!queueEmpty(this) || this.occupied) {
    if (!this.pushedEnd)
      this.queue.push(['end', implicit])
    this.pushedEnd = true
    return this.process()
  }

  if (!this.ranAfterEach && this.parent) {
    this.ranAfterEach = true
    this.parent.runAfterEach(this, end.bind(this, implicit))
  } else
    end.call(this, implicit)
}

function end (implicit) {
  this.ended = true

  if (implicit !== IMPLICIT && !this.multiEndThrew) {
    if (this.explicitEnded) {
      this.multiEndThrew = true
      var er = new Error('test end() method called more than once')
      Error.captureStackTrace(er, this.currentAssert || end)
      er.test = this.name
      this.threw(er)
      return
    }
    this.explicitEnded = true
  }

  if (this.planEnd === -1) {
    this.debug('END(%s) implicit plan', this.name, this.count)
    this.plan(this.count)
  }

  this.queue.push(EOF)
  this.process()
}

Test.prototype.threw = function (er, extra, proxy) {
  this.debug('THREW', er.message, extra, proxy)

  // event emitters 'error' events need to re-throw so that they
  // can jump out of the flow like a normal throw.  They'll just
  // end up back here once that happens, though, unless there's a
  // try/catch somewhere in the call stack.
  if (er.domainEmitter) {
    delete er.domainEmitter
    throw er
  }

  if (this.name && !proxy)
    er.test = this.name
  if (!proxy)
    extra = extraFromError(er, extra, this.options)
  Base.prototype.threw.call(this, er, extra, proxy)

  if (!this.results) {
    this.fail(extra.message || er.message, extra)
    if (!proxy)
      this.end(IMPLICIT)
  }
  this.process()
}

Test.prototype.runBeforeEach = function (who, cb) {
  var self = this
  if (this.parent)
    this.parent.runBeforeEach(who, function () {
      loop(who, self.onBeforeEach, cb, who.threw)
    })
  else
    loop(who, self.onBeforeEach, cb, who.threw)
}

Test.prototype.runAfterEach = function (who, cb) {
  var self = this
  loop(who, self.onAfterEach, function () {
    if (self.parent)
      self.parent.runAfterEach(who, cb)
    else
      cb()
  }, who.threw)
}

Test.prototype.beforeEach = function (fn) {
  this.onBeforeEach.push(fn)
}

Test.prototype.afterEach = function (fn) {
  this.onAfterEach.push(fn)
}

Test.prototype.teardown = Test.prototype.tearDown = function (fn) {
  this.on('teardown', fn)
}

Test.prototype.shouldAutoend = function () {
  var should = (
    this.options.autoend &&
    !this.ended &&
    !this.occupied &&
    queueEmpty(this) &&
    !this.pool.length &&
    !this.subtests.length &&
    this.planEnd === -1
  )
  return should
}

Test.prototype.autoend = function () {
  this.options.autoend = true
  this.maybeAutoend()
}

Test.prototype.maybeAutoend = function () {
  if (this.autoendTimer)
    clearTimeout(this.autoendTimer)

  if (this.shouldAutoend()) {
    var self = this
    self.autoendTimer = setTimeout(function () {
      if (self.shouldAutoend()) {
        self.autoendTimer = setTimeout(function () {
          if (self.shouldAutoend()) {
            self.end(IMPLICIT)
          }
        })
      }
    })
  }
}

function endAllQueue (queue) {
  queue.forEach(function (p, i) {
    if ((p instanceof Base) && !p.readyToProcess)
      queue[i] = new TestPoint(false,
        'child test left in queue ' + p.constructor.name + ': ' +
        p.name, p.options)
  })
  queue.push(['end', IMPLICIT])
}

function queueEmpty (t) {
  return t.queue.length === 0 ||
    t.queue.length === 1 && t.queue[0] === 'TAP version 13\n'
}

Test.prototype.endAll = function (sub) {
  this.processing = true
  if (this.occupied) {
    var p = this.occupied
    if (p.endAll)
      p.endAll(true)
    else {
      p.parser.abort('test unfinished')
    }
  } else if (sub) {
    this.process()
    if (queueEmpty(this)) {
      var options = Object.keys(this.options).reduce(function (o, k) {
        o[k] = this.options[k]
        return o
      }.bind(this), {})
      this.options.at = null
      this.options.stack = ''
      options.test = this.name
      this.fail('test unfinished', options)
    }
  }
  if (this.promise && this.promise.tapAbortPromise)
    this.promise.tapAbortPromise()
  if (this.occupied) {
    this.queue.unshift(this.occupied)
    this.occupied = null
  }
  endAllQueue(this.queue)
  this.processing = false
  this.process()
  this.parser.end()
}

// Add all the asserts
tapAsserts.decorate(Test.prototype)