"""mrecords

Defines the equivalent of recarrays for maskedarray.
Masked arrays already support named fields, but masking works only by records.
By comparison, mrecarrays support masking individual fields.

:author: Pierre Gerard-Marchant
"""
#!!!: * We should make sure that no field is called '_mask','mask','_fieldmask',
#!!!:   or whatever restricted keywords.
#!!!:   An idea would be to no bother in the first place, and then rename the
#!!!:   invalid fields with a trailing underscore...
#!!!:   Maybe we could just overload the parser function ?


__author__ = "Pierre GF Gerard-Marchant"

import sys

import numpy as np
from numpy import bool_, dtype, \
    ndarray, recarray, array as narray
import numpy.core.numerictypes as ntypes
from numpy.core.records import fromarrays as recfromarrays, \
                               fromrecords as recfromrecords

_byteorderconv = np.core.records._byteorderconv
_typestr = ntypes._typestr

import numpy.ma as ma
from numpy.ma import MAError, MaskedArray, masked, nomask, masked_array,\
                     getdata, getmaskarray, filled

_check_fill_value = ma.core._check_fill_value

import warnings

__all__ = ['MaskedRecords','mrecarray',
           'fromarrays','fromrecords','fromtextfile','addfield',
           ]

reserved_fields = ['_data','_mask','_fieldmask', 'dtype']

def _getformats(data):
    "Returns the formats of each array of arraylist as a comma-separated string."
    if hasattr(data,'dtype'):
        return ",".join([desc[1] for desc in data.dtype.descr])

    formats = ''
    for obj in data:
        obj = np.asarray(obj)
        formats += _typestr[obj.dtype.type]
        if issubclass(obj.dtype.type, ntypes.flexible):
            formats += `obj.itemsize`
        formats += ','
    return formats[:-1]

def _checknames(descr, names=None):
    """Checks that the field names of the descriptor ``descr`` are not some
reserved keywords. If this is the case, a default 'f%i' is substituted.
If the argument `names` is not None, updates the field names to valid names.
    """
    ndescr = len(descr)
    default_names = ['f%i' % i for i in range(ndescr)]
    if names is None:
        new_names = default_names
    else:
        if isinstance(names, (tuple, list)):
            new_names = names
        elif isinstance(names, str):
            new_names = names.split(',')
        else:
            raise NameError("illegal input names %s" % `names`)
        nnames = len(new_names)
        if nnames < ndescr:
            new_names += default_names[nnames:]
    ndescr = []
    for (n, d, t) in zip(new_names, default_names, descr.descr):
        if n in reserved_fields:
            if t[0] in reserved_fields:
                ndescr.append((d,t[1]))
            else:
                ndescr.append(t)
        else:
            ndescr.append((n,t[1]))
    return np.dtype(ndescr)


def _get_fieldmask(self):
    mdescr = [(n,'|b1') for n in self.dtype.names]
    fdmask = np.empty(self.shape, dtype=mdescr)
    fdmask.flat = tuple([False]*len(mdescr))
    return fdmask


class MaskedRecords(MaskedArray, object):
    """

*IVariables*:
    _data : {recarray}
        Underlying data, as a record array.
    _mask : {boolean array}
        Mask of the records. A record is masked when all its fields are masked.
    _fieldmask : {boolean recarray}
        Record array of booleans, setting the mask of each individual field of each record.
    _fill_value : {record}
        Filling values for each field.
    """
    #............................................
    def __new__(cls, shape, dtype=None, buf=None, offset=0, strides=None,
                formats=None, names=None, titles=None,
                byteorder=None, aligned=False,
                mask=nomask, hard_mask=False, fill_value=None, keep_mask=True,
                copy=False,
                **options):
        #
        self = recarray.__new__(cls, shape, dtype=dtype, buf=buf, offset=offset,
                                strides=strides, formats=formats, names=names,
                                titles=titles, byteorder=byteorder,
                                aligned=aligned,)
        #
        mdtype = ma.make_mask_descr(self.dtype)
        if mask is nomask or not np.size(mask):
            if not keep_mask:
                self._mask = tuple([False]*len(mdtype))
        else:
            mask = np.array(mask, copy=copy)
            if mask.shape != self.shape:
                (nd, nm) = (self.size, mask.size)
                if nm == 1:
                    mask = np.resize(mask, self.shape)
                elif nm == nd:
                    mask = np.reshape(mask, self.shape)
                else:
                    msg = "Mask and data not compatible: data size is %i, "+\
                          "mask size is %i."
                    raise MAError(msg % (nd, nm))
                copy = True
            if not keep_mask:
                self.__setmask__(mask)
                self._sharedmask = True
            else:
                if mask.dtype == mdtype:
                    _mask = mask
                else:
                    _mask = np.array([tuple([m]*len(mdtype)) for m in mask],
                                     dtype=mdtype)
                self._mask = _mask
        return self
    #......................................................
    def __array_finalize__(self,obj):
        # Make sure we have a _fieldmask by default ..
        _mask = getattr(obj, '_mask', None)
        if _mask is None:
            objmask = getattr(obj, '_mask', nomask)
            _dtype = ndarray.__getattribute__(self,'dtype')
            if objmask is nomask:
                _mask = ma.make_mask_none(self.shape, dtype=_dtype)
            else:
                mdescr = ma.make_mask_descr(_dtype)
                _mask = narray([tuple([m]*len(mdescr)) for m in objmask],
                               dtype=mdescr).view(recarray)
        # Update some of the attributes
        _dict = self.__dict__
        _dict.update(_mask=_mask)
        self._update_from(obj)
        if _dict['_baseclass'] == ndarray:
            _dict['_baseclass'] = recarray
        return


    def _getdata(self):
        "Returns the data as a recarray."
        return ndarray.view(self,recarray)
    _data = property(fget=_getdata)

    def _getfieldmask(self):
        "Alias to mask"
        return self._mask
    _fieldmask = property(fget=_getfieldmask)

    def __len__(self):
        "Returns the length"
        # We have more than one record
        if self.ndim:
            return len(self._data)
        # We have only one record: return the nb of fields
        return len(self.dtype)

    def __getattribute__(self, attr):
        try:
            return object.__getattribute__(self, attr)
        except AttributeError: # attr must be a fieldname
            pass
        fielddict = ndarray.__getattribute__(self,'dtype').fields
        try:
            res = fielddict[attr][:2]
        except (TypeError, KeyError):