# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/rcmp/__init__.py
# Compiled at: 2013-09-24 13:27:40
"""
##############
:py:mod:`RCMP`
##############

:py:mod:`Rcmp` is a more flexible replacement for :py:mod:`filecmp`
from the standard `Python <http://python.org>`_ library.

The basic idea here is that depending on content, files don't always
have to be *entirely* bitwise identical in order to be equivalent or
"close enough" for many purposes like comparing the results of two
builds.  For example, some (broken) file formats embed a time stamp
indicating the time when a file was produced even though the file
system already tracks this information.  Build the same file twice and
the two copies will initially appear to be different due to the
embedded time stamp.  Only when the irrelevant embedded time stamp
differences are ignored do the two files show out to otherwise be the
same.

:py:mod:`Rcmp` includes a flexible extension structure to allow for precisely
these sorts of living and evolving comparisons.

Extended Path Names
===================

:py:mod:`Rcmp` is capable of recursively descending into a number
of different file types including:

* file system directories
* archival and aggregating types including:

  * `ar <http://en.wikipedia.org/wiki/Ar_%28Unix%29>`_
  * `cpio <http://en.wikipedia.org/wiki/Cpio>`_
  * `tar <http://en.wikipedia.org/wiki/Tar_%28file_format%29>`_

* compressed files including:

  * `zip <http://en.wikipedia.org/wiki/Zip_%28file_format%29>`_
  * `gzip <http://en.wikipedia.org/wiki/Gzip>`_

In order to describe file locations which may extend beyond the
traditional file system paths, :py:mod:`rcmp` introduces an extended
path naming scheme.  Traditional paths are described using the
traditional slash separated list of names, :file:`/etc/hosts`.  And
components which are included in other files, like a file located
*within* a `tar <http://en.wikipedia.org/wiki/Tar_%28file_format%29>`_
archive, are described using a sequence of brace encapsulated file
format separaters.  So, for instance, a file named :file:`foo` located
within a gzip compressed, (:file:`.gz`), tar archive named
:file:`tarchive.tar` would be described as
:file:`tarchive.tar.gz{{gzip}}tarchive.tar{{tar}}foo`.  And these can
be combined as with
:file:`/home/rich/tarchive.tar.gz{{gzip}}tarchive.tar{{tar}}foo`.

Script Usage
============

:py:mod:`Rcmp` is both a library and a command line script for driving
the library.

Class Architecture
==================

.. autoclass:: Item
   :members:

.. autoclass:: Items
   :members:

.. autoclass:: Same
   :members:

.. autoclass:: Different
   :members:

.. autoclass:: Comparator
   :members:

.. autoclass:: Box
   :members:

.. autoclass:: Comparison
   :members:

.. autoclass:: ComparisonList
   :members:

Comparators
===========

..fixme:: comparators should probably be zero instance strategies.

Listed in default order of application:

.. autoclass:: NoSuchFileComparator
.. autoclass:: InodeComparator
.. autoclass:: EmptyFileComparator
.. autoclass:: DirComparator
.. autoclass:: ArMemberMetadataComparator
.. autoclass:: BitwiseComparator
.. autoclass:: SymlinkComparator

.. autoclass:: BuriedPathComparator

.. autoclass:: ElfComparator
.. autoclass:: ArComparator
.. autoclass:: AMComparator
.. autoclass:: ConfigLogComparator
.. autoclass:: KernelConfComparator
.. autoclass:: ZipComparator
.. autoclass:: TarComparator
.. autoclass:: GzipComparator
.. autoclass:: Bz2Comparator
.. autoclass:: CpioMemberMetadataComparator
.. autoclass:: CpioComparator
.. autoclass:: DateBlotBitwiseComparator
.. autoclass:: FailComparator

Utilities
=========

.. autofunction:: date_blot
.. autofunction:: ignoring

Exceptions
==========

.. autoexception:: RcmpException
.. autoexception:: IndeterminateResult

Logging strategy:
=================

Rcmp uses the python standard logging facility.  The only non-obvious
bits are that definitive differences are logged at WARNING level.
Definitive Sames are logged at WARNING - 1.  And indefinite results
are logged at WARNING - 2.  This allows for linearly increasing
volumes of logging info starting with the information that is usually
more important first.

.. Note:: I keep thinking that it would be better to create an
   IgnoringComparator that simply returned Same.  It would make much
   of the code much simpler.  However, it would mean that we'd build
   entire trees in some cases and compare them all just to produce
   constants.  This way we clip the tree.
"""
from __future__ import print_function, unicode_literals
__docformat__ = b'restructuredtext en'
__all__ = [
 b'Item',
 b'Items',
 b'Same',
 b'Different',
 b'Comparator',
 b'Box',
 b'Comparison',
 b'ComparisonList',
 b'rootItem',
 b'ignoring',
 b'date_blotNoSuchFileComparator',
 b'InodeComparator',
 b'EmptyFileComparator',
 b'DirComparator',
 b'ArMemberMetadataComparator',
 b'BitwiseComparator',
 b'SymlinkComparator',
 b'BuriedPathComparator',
 b'ElfComparator',
 b'ArComparator',
 b'AMComparator',
 b'ConfigLogComparator',
 b'KernelConfComparator',
 b'ZipComparator',
 b'TarComparator',
 b'GzipComparator',
 b'Bz2Comparator',
 b'CpioMemberMetadataComparator',
 b'CpioComparator',
 b'DateBlotBitwiseComparator',
 b'FailComparator']
lzma = False
import abc
if lzma:
    import backports.lzma as lzma
import bz2file as bz2, contextlib, difflib, errno, fnmatch, gzip, io, logging, mmap, operator, os, re, stat, subprocess, sys, tarfile, tempfile, zipfile, elffile, arpy, cpiofile
DIFFERENCES = logging.WARNING
SAMES = logging.WARNING - 1
INDETERMINATES = logging.WARNING - 2
logging.addLevelName(DIFFERENCES, b'differences')
logging.addLevelName(SAMES, b'sames')
logging.addLevelName(INDETERMINATES, b'indeterminates')
logger = logging.getLogger(__name__)
import pprint
pp = pprint.PrettyPrinter()

def _loggable(cls):
    cls.logger = logging.getLogger((b'{}.{}').format(__name__, cls.__name__))
    return cls

_read_count = 3

@_loggable
class Item(object):
    """
    Things which can be compared are represented internally by
    instances of class :py:class:`Item`.  These can be items in the
    file system, like a file or directory, or in an archive, like an
    archive member.

    This is used for caching the results from calls like stat and for
    holding content.

    :param name: file system name
    :type name: string
    """

    def __init__(self, name, parent, box=None):
        assert parent
        self._name = name
        self._statbuf = False
        self._fd = False
        self._content = False
        self._link = False
        self._size = None
        self._read_count = 0
        self._native = False
        self.parent = parent
        self._box = box if box else DirComparator
        self.logger.log(logging.DEBUG, b'Item(name = %s, parent = %s, box = %s)', name, parent.name if hasattr(parent, b'name') else b'None', self._box.__name__)
        return

    @property
    def box(self):
        return self._box

    @box.setter
    def box(self, value):
        """setter"""
        assert value
        self._box = value

    @property
    def name(self):
        """
        name in the extended file system name space of this :py:class:`Item`.

        :rtype: string
        """
        return self._name

    @property
    def shortname(self):
        return self.box.member_shortname(self)

    @property
    def content(self):
        """
        The contents of the entire file, in memory.

        :rtype: bytearray.
        """
        global _read_count
        try:
            if self._content is False:
                self._content = self.parent.box.member_content(self)
                self._read_count += 1
                if self._read_count > _read_count:
                    _read_count = self._read_count
        except TypeError:
            self.logger.log(logging.ERROR, b'self = %s, %s', self, self.name)
            self.logger.log(logging.ERROR, b'self.parent = %s, %s', self.parent, self.parent.name)
            self.logger.log(logging.ERROR, b'self.box = %s', self.box)
            self.logger.log(logging.ERROR, b'self.box.member_content = %s', self.box.member_content)
            self.logger.log(logging.ERROR, b'self.parent.box = %s, %s', self.parent.box, self.parent.box.__name__)
            self.logger.log(logging.ERROR, b'self.parent.box.member_content = %s', self.parent.box.member_content)
            raise

        return self._content

    def reset(self):
        self.logger.log(logging.DEBUG, b'resetting %s', self.name)
        self._content = False

    @property
    def stat(self):
        """
        If we have a statbuf, return it.

        If not, then look one up, cache it, and return it.

        :rtype: statbuf
        """
        if not self._statbuf:
            self._statbuf = self.parent.box.member_stat(self)
        return self._statbuf

    @property
    def exists(self):
        """
        Check for existence.

        :rtype: boolean
        """
        try:
            return self.parent.box.member_exists(self)
        except:
            self.logger.log(logging.DEBUG, b'self = %s, self.box = %s', self, self.box)
            raise

    @property
    def inode(self):
        """
        Return the inode number from stat.

        :rtype: string
        """
        return self.box.member_inode(self)

    @property
    def device(self):
        """
        Return device number from stat.

        :rtype: string
        """
        return self.box.member_device(self)

    @property
    def size(self):
        """
        Return our size.  Look it up in stat, (and cache the result), if
        we don't already know what it is.

        :rtype: int
        """
        if self._size is None:
            self._size = self.parent.box.member_size(self)
        return self._size

    @property
    def isdir(self):
        """
        Return True if and only if we are represent a file system
        directory.

        :rtype: boolean
        """
        try:
            return self.parent.box.member_isdir(self)
        except:
            self.logger.log(logging.DEBUG, b'isdir self = %s, self.box = %s', self.name, self.box)
            raise

    @property
    def isreg(self):
        """
        Return True if and only if we represent a regular file.

        :rtype: boolean
        """
        return self.parent.box.member_isreg(self)

    @property
    def islnk(self):
        """
        Return True if and only if we represent a symbolic link.

        :rtype: boolean
        """
        return self.parent.box.member_islnk(self)

    @property
    def link(self):
        """
        Return a string representing the path to which the symbolic link
        points.  This presumes that we are a symbolic link.

        :rtype: string
        """
        if not self._link:
            self._link = self.parent.box.member_link(self)
        return self._link

class Items(object):
    """
    There is a global set of all instances of class :py:class:`Item`
    stored in the singular class :py:class:`Items`.

    This exists primarily to prevent us from creating a duplicate
    :py:class:`Item` for the same path name.

    .. note:: The class is used directly here as a global aggregator,
       a singleton.  It is never instantiated but instead the class
       itself is used as a singleton.
    """
    _content = {}

    @classmethod
    def find_or_create(cls, name, parent, box=None):
        """
        Look up an :py:class:`Item` with *name*.  If necessary, create it.

        :param name: the name of the :py:class`Item` to look up
        :type name: string
        :rtype: :py:class:`Item`
        """
        if not box:
            box = DirComparator
        if name in cls._content:
            return cls._content[name]
        else:
            x = Item(name, parent, box)
            cls._content[name] = x
            return x

    @classmethod
    def delete(cls, name):
        """
        Delete an :py:class:`Item` from the set.

        :param name: name of the :py:class:`Item` to be deleted.
        :type name: string
        """
        del cls._content[name]

    @classmethod
    def reset(cls):
        cls._content = {}

class Same(object):
    """
    Returned to indicate an authoritative claim of sufficient
    identicality.  No further comparators need be tried.

    .. note:: The class itself is used as a constant.  It is never
       instantiated.
    """
    pass

class Different(object):
    """
    Returned to indicate an authoritative claim of difference.  No
    further comparators need be tried.

    .. note:: The class itself is used as a constant.  It is never
       instantiated.
    """
    pass

@_loggable
class Comparator(object):
    """
    Represents a single comparison heuristic.  This is an abstract
    class.  It is intended solely to act as a base class for
    subclasses.  It is never instantiated. (lie - fixme).

    Subclasses based on :py:class:`Comparator` implement individual
    heuristics for comparing items when applied to a
    :py:class:`Comparison`.  There are many :py:class:`Comparator`
    subclasses included.

    ..note:: :py:class:`Comparator`s are strategies.  That is, there are no
             instantiation variables nor properties.
    """
    __metaclass__ = abc.ABCMeta

    @staticmethod
    @abc.abstractmethod
    def _applies(thing):
        return False

    @classmethod
    def applies(cls, comparison):
        """
        Return True if and only if we apply to the given comparison.

        :type comparison: :py:class:`Comparison`
        :rtype: boolean
        """
        return reduce(operator.iand, [ cls._applies(i) for i in comparison.pair ])

    @classmethod
    @abc.abstractmethod
    def cmp(cls, comparison):
        """
        Apply ourselves to the given :py:class:`Comparison`.

        If can make an authoritative determination about whether the
        :py:class:`Items` are alike then return either
        :py:class:`Same` or :py:class:`Different`.  If we can make no
        such determination, then return a non-True value.

        :type comparison: :py:class:`Comparison`
        :rtype: :py:class:`Same`, :py:class:`Different`, or a non-True value
        """
        cls.logger.error(b"%s.cmp() isn't overridden.", cls.__name__)
        raise NotImplementedError
        return False

    @classmethod
    def _log_item(cls, item):
        if item.exists and item.islnk:
            return (item.name, item.link)
        else:
            return item.name

    @classmethod
    def _log_string(cls, s, comparison):
        return (b'{0} {1} {2}').format(s, cls.__name__, comparison.pair[0].name.partition(os.sep)[2])

    @classmethod
    def _log_unidiffs(cls, content, names):
        try:
            cls.logger.log(DIFFERENCES, (b'\n').join(difflib.unified_diff(content[0].split(b'\n'), content[1].split(b'\n'), names[0], names[1], b'', b'', 3, b'')))
        except UnicodeError:
            pass

    @classmethod
    def _log_unidiffs_comparison(cls, comparison):
        cls._log_unidiffs([ i.content for i in comparison.pair ], [ i.name for i in comparison.pair ])

    @classmethod
    def _log_different(cls, comparison):
        cls.logger.log(DIFFERENCES, cls._log_string(b'Different', comparison))

    @classmethod
    def _log_same(cls, comparison):
        cls.logger.log(SAMES, cls._log_string(b'Same', comparison))

    @classmethod
    def _log_indeterminate(cls, comparison):
        cls.logger.log(INDETERMINATES, cls._log_string(b'Indeterminate', comparison))

class DatePattern(object):

    def __init__(self, pattern, replacement):
        self.pattern = pattern
        self.replacement = replacement
        self.compiled = re.compile(pattern)

dow = b'(Sun|Mon|Tue|Wed|Thu|Fri|Sat)'
moy = b'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)'
lmoy = b'(January|February|March|April|May|June|July|August|September|October|November|December)'
date_patterns = [
 DatePattern(dow + b' ' + moy + b' *[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2} (PST|PDT) [0-9]{4}', b'Day Mon 00 00:00:00 LOC 2011'),
 DatePattern(dow + b' ' + moy + b' *[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4}', b'Day Mon 00 00:00:00 2011'),
 DatePattern(b'(?i) *[0-9]{1,2} (JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC) [0-9]{4} [0-9]{2}:[0-9]{2}', b'00 MON 2011 00:00'),
 DatePattern(lmoy + b' *[0-9]{1,2}\\\\?, [0-9]{4}', b'Month 00, 2011'),
 DatePattern(dow + b' ' + moy + b' *[0-9]{1,2} *[0-9]{4}', b'Day Mon 00 2011'),
 DatePattern(dow + b' *[0-9]{1,2} *' + moy + b' *[0-9]{4}', b'Day 00 Mon 2011'),
 DatePattern(dow + b' *[0-9]{1,2} *' + lmoy + b' *[0-9]{4}', b'Day 00 Month 2011'),
 DatePattern(b'20*[0-9]{2}-*[0-9]{2}-*[0-9]{2}', b'2011-00-00'),
 DatePattern(moy + b' [0-9]{4}', b'Mon 2011'),
 DatePattern(b'[0-9]{2}:[0-9]{2}:[0-9]{2}', b'00:00:00'),
 DatePattern(b'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}Z', b'00000000T000000Z')]

def date_blot(input_string):
    """Convert dates embedded in a string into innocuous constants of uniform length.

    :param input_string: input string
    :rtype: string
    """
    retval = input_string
    for pat in date_patterns:
        try:
            retval = pat.compiled.sub(pat.replacement, retval)
        except UnicodeError:
            pass

    return retval

def fntore(names):
    """
    Convert a list of wildcard style patterns into a list of compiled regexps.
    """
    return [ re.compile(fnmatch.translate(n)) for n in names ]

def fntoreconcat(names):
    """
    Convert a list of wildcard style patterns into a list of compiled regexps.
    """
    return [
     re.compile((b'|').join([ fnmatch.translate(n) for n in names ]))]

def ignoring(ignores, fname):
    """
    Given a list of file names to be ignored and a specific file name
    to check, return the first ignore pattern from the list that
    matches the file name.

    :param ignores: ignore patterns
    :type ignores: list of strings
    :param fname: file name to check
    :type fname: string
    :rtype: string or False (Can be used as a predicate.)
    """
    for ignore in ignores:
        try:
            if ignore.match(fname):
                return ignore
        except AttributeError:
            logger.log(logging.ERROR, b'ignore = %s', ignore)
            raise

    return False

@_loggable
class InodeComparator(Comparator):
    """
    Objects with the same inode and device are identical.
    """

    @classmethod
    def _applies(cls, item):
        return item.box is DirComparator

    @classmethod
    def cmp(cls, comparison):
        if reduce(operator.eq, [ i.inode for i in comparison.pair ]) and reduce(operator.eq, [ i.device for i in comparison.pair ]):
            cls._log_same(comparison)
            return Same
        else:
            cls._log_indeterminate(comparison)
            return False

@_loggable
class EmptyFileComparator(Comparator):
    """
    Two files which are each empty are equal.  In particular, we don't
    need to open them or read them to make this determination.
    """

    @classmethod
    def _applies(cls, item):
        return item.isreg

    @classmethod
    def cmp(cls, comparison):
        if comparison.pair[0].size == 0 and comparison.pair[1].size == 0:
            cls._log_same(comparison)
            return Same
        else:
            cls._log_indeterminate(comparison)
            return False

class RcmpException(Exception):
    """Base class for all :py:mod:`rcmp` exceptions"""
    pass

class IndeterminateResult(RcmpException):
    """
    Raised when we can't make any authoritative determination.  At the
    top level, this is an error condition as this case indicates that
    we've failed to accomplish our job.  Note that this is
    significantly different from the non-True value returned by
    :py:class:`Comparator` subclasses to indicate that they have no
    authoritative result.
    """
    pass

class BadZipfile(RcmpException):
    """Raised when we fail to open a zip archive"""
    pass

class _Packer(object):
    """
    just for aggregation, not intended for instantiation.
    """

    def __init__(self, joiner=b'/'):
        self.joiner = joiner

    def join(self, left, right):
        return (b'{0}{1}{2}').format(left, self.joiner, right)

    def split(self, path):
        return path.split(self.joiner)

class Box(Comparator):
    """
    This is an abstract base class intended for comparators on things
    which are composed of other things.  So, for instance, a
    directory, or a file archive.

    ..note:: subclasses are strategies - they have no properties.
    """
    __metaclass__ = abc.ABCMeta
    _packer = None

    @classmethod
    def member_shortname(cls, member):
        return cls._packer.split(member.name)[(-1)]

    @staticmethod
    @abc.abstractmethod
    def _applies(thing):
        raise NotImplementedError

    @classmethod
    @abc.abstractmethod
    def box_keys(cls, item):
        raise NotImplementedError

    @classmethod
    def _no_mate(cls, name, logger):
        cls.logger.log(DIFFERENCES, b'Different %s No mate: %s', cls.__name__, name)

    @classmethod
    def _expand(cls, ignoring, item):
        for shortname in cls.box_keys(item):
            fullname = cls._packer.join(item.name, shortname)
            ignore = ignoring(fullname)
            if ignore:
                cls.logger.log(SAMES, b'Ignoring %s cause %s', fullname, ignore)
                continue
            newitem = Items.find_or_create(fullname, item, cls)
            cls.logger.log(logging.DEBUG, b'%s expands %s -> %s', cls.__name__, item.name, shortname)
            yield (shortname, newitem)

    @staticmethod
    def _mates(item, container):
        return item.shortname in container.box.box_keys(container)

    @classmethod
    def _outer_join(cls, comparison, invert=False, spool=True):
        result = False
        if invert:
            rparent, lparent = [ p for p in comparison.pair ]
        else:
            lparent, rparent = [ p for p in comparison.pair ]
        for shortname, litem in cls._expand(comparison.ignoring, lparent):
            rname = cls._packer.join(rparent.name, shortname)
            ignore = comparison.ignoring(litem.name)
            if ignore:
                cls.logger.log(SAMES, b'Ignoring %s cause %s', lname, ignore)
                continue
            ritem = Items.find_or_create(rname, rparent, cls)
            if cls._mates(litem, rparent):
                if spool:
                    cls.logger.log(logging.DEBUG, b'spooling %s', litem.name)
                    comparison.children.append(Comparison(litem=litem, ritem=ritem, comparators=comparison.comparators, ignores=comparison.ignores, exit_asap=comparison.exit_asap))
            else:
                cls._no_mate(litem.name, logger)
                result = Different

        return result

    @classmethod
    def _left_outer_join(cls, comparison):
        return cls._outer_join(comparison)

    @classmethod
    def _right_outer_join(cls, comparison):
        return cls._outer_join(comparison, invert=True, spool=False)

    @classmethod
    def _inner_join(cls, comparison):
        retval = Same
        for c in comparison.children:
            r = c.cmp()
            if not r:
                cls._log_indeterminate(comparison)
                raise IndeterminateResult
            if r == Different:
                cls._log_different(comparison)
                retval = Different
                if comparison.exit_asap:
                    return retval

        return retval

    @classmethod
    def cmp(cls, comparison):
        """
        Compare our lists and return the result.
        """
        cls.logger.log(logging.DEBUG, b'Box.cmp(%s, ...', cls.__name__)
        retval = Same
        comparison.pair[0].box = comparison.pair[1].box = cls
        if cls._left_outer_join(comparison) == Different or cls._right_outer_join(comparison) == Different:
            retval = Different
            if comparison.exit_asap:
                comparison.reset()
                return retval
        if cls._inner_join(comparison) == Different:
            retval = Different
            if comparison.exit_asap:
                comparison.reset()
                return retval
        if retval == Same:
            cls._log_same(comparison)
            comparison.reset()
        return retval

    @staticmethod
    @abc.abstractmethod
    def member_content(member):
        raise NotImplementedError

    @staticmethod
    def member_stat(member):
        """
        If member has a statbuf, return it.

        If not, then look one up, cache it, and return it.

        :rtype: statbuf
        """
        raise NotImplementedError

    @staticmethod
    def member_exists(member):
        """
        Check for existence.

        :rtype: boolean
        """
        Box.logger.log(logging.DEBUG, b'member_exists: member = %s, parent.box(%s) -> %s', member.name, member.parent.name, member.parent.box.box_keys(member.parent))
        return member.shortname in member.parent.box.box_keys(member.parent)

    @staticmethod
    def member_inode(member):
        """
        Return the inode number from stat.

        :rtype: string
        """
        Box.logger.log(logging.ERROR, b'member_inode not implemented for %s', member.name)
        raise NotImplementedError

    @staticmethod
    def member_device(member):
        """
        Return device number from stat.

        :rtype: string
        """
        Box.logger.log(logging.ERROR, b'member_device not implemented for %s', member.name)
        raise NotImplementedError

    @staticmethod
    def member_size(member):
        """
        Return our size.

        :rtype: int
        """
        Box.logger.log(logging.ERROR, b'member_size not implemented for %s', member.name)
        raise NotImplementedError

    @staticmethod
    def member_isdir(member):
        """
        Return True if and only if we are represent a directory.

        So far, none of the archive formats recur.  That is, they're
        all flat collections of files rather than being collections of
        collections necessarily.  Although such can be created, they
        aren't an inherent part of the file format.

        :rtype: boolean
        """
        Box.logger.log(logging.ERROR, b'member_isdir not implemented for %s', member.name)
        raise NotImplementedError

    @staticmethod
    def member_isreg(member):
        """
        Return True if and only if member represents a regular file.

        :rtype: boolean
        """
        Box.logger.log(logging.ERROR, b'member_isreg not implemented for %s', member.name)
        raise NotImplementedError

    @staticmethod
    def member_islnk(member):
        """
        Return True if and only if we represent a symbolic link.

        :rtype: boolean
        """
        Box.logger.log(logging.ERROR, b'member_islnk not implemented for %s', member.name)
        raise NotImplementedError

    @staticmethod
    def member_link(member):
        """
        Return a string representing the path to which the symbolic link
        points.  This presumes that we are a symbolic link.

        :rtype: string
        """
        Box.logger.log(logging.ERROR, b'member_link not implemented for %s', member.name)
        raise NotImplementedError

class ContentOnlyBox(Box):
    """
    Some containers like zip and gzip only have members with actual
    content.  That is, no symlinks, no devices, etc.
    """

    @staticmethod
    def member_isreg(member):
        return True

    @staticmethod
    def member_isdir(member):
        return False

    @staticmethod
    def member_islnk(member):
        return False

class UnixBox(Box):
    """
    Archivers like tar and cpio are capable of tracking hard and soft
    links as well as devices, directories and ownerships, etc.
    """

    @staticmethod
    def member_isdir(member):
        return False

@_loggable
class DirComparator(Box):
    """
    Objects which are directories are special.  They match if their
    contents match.

    .. fixme: this could be a box too.
    """
    _packer = _Packer(b'/')

    @staticmethod
    def _applies(item):
        return item.isdir

    @classmethod
    def box_keys(cls, item):
        if not hasattr(item, b'dirs'):
            item.dirs = os.listdir(item.name)
        return item.dirs

    @staticmethod
    @contextlib.contextmanager
    def member_mmap(member):
        with open(member.name, b'rb') as (fd):
            yield mmap.mmap(fd.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)
            fd.close()

    @staticmethod
    def member_content(member):
        with open(member.name, b'rb') as (fd):
            return fd.read()

    @staticmethod
    def member_exists(member):
        """
        Check for existence.

        :rtype: boolean
        """
        return os.path.exists(member.name)

    @staticmethod
    def member_stat(member):
        """
        :rtype: statbuf
        """
        return os.lstat(member.name)

    @staticmethod
    def member_inode(member):
        """
        Return the inode number from stat.

        :rtype: string
        """
        return member.stat.st_ino

    @staticmethod
    def member_device(member):
        """
        Return device number from stat.

        :rtype: string
        """
        return member.stat.st_dev

    @staticmethod
    def member_size(member):
        """
        Return our size.

        :rtype: int
        """
        return member.stat.st_size

    @staticmethod
    def member_isdir(member):
        """
        Return True if and only if we are represent a file system
        directory.

        :rtype: boolean
        """
        return stat.S_ISDIR(member.stat.st_mode)

    @staticmethod
    def member_isreg(member):
        """
        Return True if and only if we represent a regular file.

        :rtype: boolean
        """
        return stat.S_ISREG(member.stat.st_mode)

    @staticmethod
    def member_islnk(member):
        """
        Return True if and only if we represent a symbolic link.

        :rtype: boolean
        """
        return stat.S_ISLNK(member.stat.st_mode)

    @staticmethod
    def member_link(member):
        """
        Return a string representing the path to which the symbolic link
        points.  This presumes that we are a symbolic link.

        :rtype: string
        """
        return os.readlink(member.name)

@_loggable
class BitwiseComparator(Comparator):
    """
    Objects which are bitwise identical are close enough.
    """

    @staticmethod
    def _applies(item):
        BitwiseComparator.logger.log(logging.DEBUG, b'testing whether BitwiseComparator applies to %s', item.name)
        return item.isreg

    @classmethod
    def cmp(cls, comparison):
        if not reduce(operator.eq, [ i.size for i in comparison.pair ]):
            cls._log_indeterminate(comparison)
            return False
        if reduce(operator.eq, [ bool(i._content) for i in comparison.pair ] + [True]) or comparison.pair[0].parent.box != DirComparator:
            if comparison.pair[0].content == comparison.pair[1].content:
                comparison.reset()
                cls._log_same(comparison)
                return Same
            else:
                cls._log_indeterminate(comparison)
                return False

        with contextlib.nested(DirComparator.member_mmap(comparison.pair[0]), DirComparator.member_mmap(comparison.pair[1])) as (m1, m2):
            if m1 == m2:
                cls._log_same(comparison)
                return Same
            comparison.pair[0]._content = m1[:]
            comparison.pair[1]._content = m2[:]
            cls._log_indeterminate(comparison)
            retval = False
        return retval

@_loggable
class DateBlotBitwiseComparator(Comparator):
    """
    Objects which are bitwise identical after date blotting are close
    enough.  But this should only be tried late.
    """

    @staticmethod
    def _applies(item):
        return item.isreg

    @classmethod
    def cmp(cls, comparison):
        if reduce(operator.eq, [ date_blot(i.content) for i in comparison.pair ]):
            cls._log_same(comparison)
            retval = Same
        else:
            cls._log_indeterminate(comparison)
            retval = False
        return retval

@_loggable
class NoSuchFileComparator(Comparator):
    """
    Objects are different if either one is missing.
    """

    @staticmethod
    def _applies(item):
        return True

    @classmethod
    def cmp(cls, comparison):
        e = [ i.exists for i in comparison.pair ]
        if reduce(operator.ne, e):
            cls._log_different(comparison)
            return Different
        if e[0] is False:
            cls._log_same(comparison)
            return Same
        cls._log_indeterminate(comparison)
        return False

@_loggable
class ElfComparator(Comparator):
    """
    Elf files are different if any of the important sections are
    different.
    """
    _magic = b'\x7fELF'

    @staticmethod
    def _applies(item):
        return item.content.find(ElfComparator._magic, 0, len(ElfComparator._magic)) == 0

    @classmethod
    def cmp(cls, comparison):
        e = [ i.content.find(cls._magic, 0, len(cls._magic)) == 0 for i in comparison.pair ]
        if not reduce(operator.iand, e):
            cls._log_different(comparison)
            return Different
        else:
            e = [ elffile.open(name=i.name, block=i.content) for i in comparison.pair ]
            if e[0].close_enough(e[1]):
                cls._log_same(comparison)
                return Same
            cls._log_different(comparison)
            with tempfile.NamedTemporaryFile(delete=False) as (left):
                leftname = left.name
                left.write(comparison.pair[0].content)
            lcontent = subprocess.check_output(str((b'objdump -sfh {}').format(leftname)).split())
            os.remove(leftname)
            with tempfile.NamedTemporaryFile(delete=False) as (right):
                rightname = right.name
                right.write(comparison.pair[1].content)
            rcontent = subprocess.check_output(str((b'objdump -sfh {}').format(rightname)).split())
            os.remove(rightname)
            cls._log_unidiffs([lcontent, rcontent], [ i.name for i in comparison.pair ])
            return Different

@_loggable
class ArMemberMetadataComparator(Comparator):
    """
    Verify the metadata of each member of an ar archive.
    """

    @staticmethod
    def _applies(item):
        return item.parent is not None and item.parent.box is ArComparator

    @classmethod
    def cmp(cls, comparison):
        cls.logger.log(logging.DEBUG, b'cmp: pair[0] = %s', comparison.pair[0].name)
        cls.logger.log(logging.DEBUG, b'cmp: parent = %s', comparison.pair[0].parent.name)
        left, right = [ i.parent.ar.archived_files[i.shortname].header for i in comparison.pair ]
        if left.uid == right.uid and left.gid == right.gid and left.mode == right.mode:
            return False
        else:
            cls._log_different(comparison)
            return Different

@contextlib.contextmanager
def openar(filename, fileobj):
    """
    """
    ar = arpy.Archive(filename=filename, fileobj=fileobj)
    ar.read_all_headers()
    yield ar
    ar.close()

@_loggable
class ArComparator(ContentOnlyBox):
    """
    Ar archive files are different if any of the important members are
    different.

    .. note:: This is a strategy - there are no instance
       properties. Rather, the content is stored in the comparison
       pairs.
    """
    _magic = b'!<arch>\n'
    _packer = _Packer(b'{ar}')

    @staticmethod
    def _applies(item):
        return item.content.find(ArComparator._magic, 0, len(ArComparator._magic)) == 0

    @classmethod
    def box_keys(cls, item):
        cls.logger.log(logging.DEBUG, b'%s.box_keys(%s) -> %s', cls.__name__, item.name, item.ar.archived_files.keys())
        return item.ar.archived_files.keys()

    @staticmethod
    def member_size(member):
        return member.parent.ar.archived_files[member.shortname].header.size

    @staticmethod
    def member_content(member):
        return member.parent.ar.archived_files[member.shortname].read()

    @classmethod
    def cmp(cls, comparison):
        with contextlib.nested(openar(comparison.pair[0].name, io.BytesIO(comparison.pair[0].content)), openar(comparison.pair[1].name, io.BytesIO(comparison.pair[1].content))) as (comparison.pair[0].ar, comparison.pair[1].ar):
            return super(cls, cls).cmp(comparison)

@_loggable
class CpioMemberMetadataComparator(Comparator):
    """
    Verify the metadata of each member of a cpio archive.
    """

    @staticmethod
    def _applies(item):
        return item.parent.box is CpioComparator

    @classmethod
    def cmp(cls, comparison):
        left, right = [ i.parent.cpio.get_member(i.shortname) for i in comparison.pair ]
        if left.mode == right.mode and left.uid == right.uid and left.gid == right.gid and left.rdevmajor == right.rdevmajor and left.rdevminor == right.rdevminor and left.filesize == right.filesize:
            if left.filesize == 0:
                return Same
            else:
                return False

        else:
            cls._log_different(comparison)
            return Different

@contextlib.contextmanager
def opencpio(filename, guts):
    """
    """
    cpio = cpiofile.CpioFile().open(name=filename, block=guts)
    yield cpio
    cpio.close()

@_loggable
class CpioComparator(UnixBox):
    """
    Cpio archive files are different if any of the important members
    are different.

    .. note:: This is a strategy - there are no instance
       properties. Rather, the content is stored in the comparison
       pairs.
    """
    _packer = _Packer(b'{cpio}')

    @staticmethod
    def _applies(item):
        return bool(cpiofile.valid_magic(item.content))

    @classmethod
    def box_keys(cls, item):
        return item.cpio.names

    @staticmethod
    def member_size(member):
        return member.parent.cpio.get_member(member.shortname).filesize

    @staticmethod
    def member_content(member):
        return member.parent.cpio.get_member(member.shortname).content

    @staticmethod
    def member_isreg(member):
        return stat.S_ISREG(member.parent.cpio.get_member(member.shortname).mode)

    @staticmethod
    def member_islnk(member):
        return stat.S_ISLNK(member.parent.cpio.get_member(member.shortname).mode)

    @staticmethod
    def member_link(member):
        return member.content

    @classmethod
    def cmp(cls, comparison):
        with contextlib.nested(opencpio(comparison.pair[0].name, comparison.pair[0].content), opencpio(comparison.pair[1].name, comparison.pair[0].content)) as (comparison.pair[0].cpio, comparison.pair[1].cpio):
            return super(cls, cls).cmp(comparison)

@_loggable
class TarMemberMetadataComparator(Comparator):
    """
    Verify the metadata of each member of an ar archive.
    """

    @staticmethod
    def _applies(item):
        return item.parent.box is TarComparator

    @classmethod
    def cmp(cls, comparison):
        left, right = [ i.parent.box.getmember(i) for i in comparison.pair ]
        if left.mode == right.mode and left.type == right.type and left.linkname == right.linkname and left.uid == right.uid and left.gid == right.gid and left.uname == right.uname and left.gname == right.gname:
            if left.size == 0:
                return Same
            else:
                return False

        else:
            cls._log_different(comparison)
            return Different

import contextlib

@contextlib.contextmanager
def opentar(filename, mode, fileobj):
    """
    .. todo:: remove opentar once we move to python-2.7
    """
    tar = tarfile.open(name=filename, mode=mode, fileobj=fileobj)
    yield tar
    tar.close()

@_loggable
class TarComparator(UnixBox):
    """
    Tar archive files are different if any of the important members
    are different.

    .. note:: must be called *after* GzipComparator in order to duck
       the Python tarfile module's pathological performace with compressed
       archives.

    .. note:: This is a strategy - there are no instance
       properties. Rather, the content is stored in the comparison
       pairs.
    """
    _packer = _Packer(b'{tar}')

    @staticmethod
    def _applies(item):
        try:
            tarfile.open(fileobj=io.BytesIO(item.content)).close()
        except:
            return False

        return True

    @staticmethod
    def getmember(item):
        if not hasattr(item, b'member'):
            item.member = item.parent.tar.getmember(item.shortname)
        return item.member

    @classmethod
    def box_keys(cls, item):
        if not hasattr(item, b'names'):
            item.names = item.tar.getnames()
        return item.names

    @staticmethod
    def member_size(member):
        return member.parent.box.getmember(member).size

    @staticmethod
    def member_content(member):
        info = member.parent.box.getmember(member)
        if not info:
            raise AssertionError
            if info.isdir() or info.isdev():
                return b''
            fileobj = member.parent.tar.extractfile(member.shortname)
            fileobj or TarComparator.logger.log(logging.ERROR, b'member_content could not find %s, (%s), in %s', member.shortname, member.name, member.parent.name)
            raise NotImplementedError
        return fileobj.read()

    @staticmethod
    def member_isreg(member):
        return member.parent.box.getmember(member).isreg()

    @staticmethod
    def member_islnk(member):
        return member.parent.box.getmember(member).issym()

    @staticmethod
    def member_link(member):
        return member.parent.box.getmember(member).linkname

    @classmethod
    def cmp(cls, comparison):
        with contextlib.nested(opentar(comparison.pair[0].name, b'r', io.BytesIO(comparison.pair[0].content)), opentar(comparison.pair[1].name, b'r', io.BytesIO(comparison.pair[1].content))) as (comparison.pair[0].tar, comparison.pair[1].tar):
            return super(cls, cls).cmp(comparison)

import contextlib

@contextlib.contextmanager
def openzip(file, mode):
    """
    .. todo:: remove openzip once we move to python-2.7
    """
    zip = zipfile.ZipFile(file, mode)
    if zip.testzip():
        raise BadZipfile
    yield zip
    zip.close()

@_loggable
class ZipComparator(ContentOnlyBox):
    """
    Zip archive files are different if any of the members are different.

    .. note:: This is a strategy - there are no instance
       properties. Rather, the content is stored in the comparison
       pairs.
    """
    _myname = b'zip'
    _packer = _Packer((b'{{}}').format(_myname))

    @staticmethod
    def _applies(item):
        """
        """
        try:
            zipfile.ZipFile(io.BytesIO(item.content), b'r').close()
        except:
            return False

        return True

    @classmethod
    def box_keys(cls, item):
        return item.zip.namelist()

    @staticmethod
    def member_size(member):
        return member.parent.zip.getinfo(member.shortname).file_size

    @staticmethod
    def member_content(member):
        return member.parent.zip.read(member.shortname)

    @classmethod
    def cmp(cls, comparison):
        with contextlib.nested(openzip(io.BytesIO(comparison.pair[0].content), b'r'), openzip(io.BytesIO(comparison.pair[1].content), b'r')) as (comparison.pair[0].zip, comparison.pair[1].zip):
            if comparison.pair[0].zip.comment != comparison.pair[1].zip.comment:
                cls._log_different(comparison)
                return Different
            else:
                return super(cls, cls).cmp(comparison)

@_loggable
class AMComparator(Comparator):
    """
    Automake generated Makefiles have some nondeterminisms.  They're
    the same if they're the same aside from that.  (May also need to
    make some allowance for different tool sets later.)
    """

    @staticmethod
    def _applies(item):
        if not item.name.endswith(b'Makefile'):
            return False
        p = -1
        for i in range(5):
            p = item.content.find(b'\n', p + 1, p + 132)
            if p is -1:
                return False

        return item.content.find(b'generated by automake', 0, p) > -1

    @classmethod
    def cmp(cls, comparison):
        left, right = [ i.content.decode(b'utf8') for i in comparison.pair ]
        left, right = [ date_blot(i) for i in [left, right] ]
        left, right = [ re.sub(b'(?m)^MODVERSION = .*$', b'MODVERSION = ...', i, 0) for i in [left, right] ]
        left, right = [ re.sub(b'(?m)^BUILDINFO = .*$', b'BUILDINFO = ...', i, 0) for i in [left, right] ]
        if left == right:
            cls._log_same(comparison)
            return Same
        else:
            cls._log_different(comparison)
            cls._log_unidiffs([left, right], [ i.name for i in comparison.pair ])
            return Different

@_loggable
class ConfigLogComparator(Comparator):
    """
    When autoconf tests fail, there's a line written to the config.log
    which exposes the name of the underlying temporary file.  Since
    the name of this temporary file changes from build to build, it
    introduces a nondeterminism.

    .. note:: I'd ignore config.log files, (and started to do exactly
       that), but it occurs to me that differences in autoconf
       configuration are quite likely to cause build differences.  So
       I've been more surgical.
    """

    @staticmethod
    def _applies(item):
        if item.name.endswith(b'config.log'):
            trigger = b'generated by GNU Autoconf'
        else:
            if item.name.endswith(b'config.status'):
                trigger = b'Generated by configure.'
            elif item.name.endswith(b'config.h'):
                trigger = b'Generated from config.h.in by configure.'
            else:
                return False
            p = -1
            for i in range(8):
                p = item.content.find(b'\n', p + 1)
                if p is -1:
                    return False

        return item.content.find(trigger, 0, p) > -1

    @classmethod
    def cmp(cls, comparison):
        left, right = [ i.content for i in comparison.pair ]
        left, right = [ re.sub(b'(?m)/cc.{6}\\.([os])', b'/cc------.\x01', i, 0) for i in [
         left, right]
                      ]
        left, right = [ re.sub(b'(?m)MODVERSION.*$', b'MODVERSION...', i, 0) for i in [
         left, right]
                      ]
        left, right = [ date_blot(i) for i in [left, right] ]
        if left == right:
            cls._log_same(comparison)
            return Same
        else:
            cls._log_different(comparison)
            cls._log_unidiffs([left, right], [ i.name for i in comparison.pair ])
            return Different

@_loggable
class KernelConfComparator(Comparator):
    """
    When "make config" is run in the kernel, it generates an auto.conf
    file which includes a time stamp.  I think these files are
    important enough to merit more surgical checking.  This comparator
    blots out the 4th line.
    """

    @staticmethod
    def _applies(item):
        if item.name.endswith(b'auto.conf'):
            trigger = b"Automatically generated make config: don't edit"
        else:
            if item.name.endswith(b'autoconf.h'):
                trigger = b"Automatically generated C config: don't edit"
            else:
                return False
            p = -1
            for i in range(8):
                p = item.content.find(b'\n', p + 1)
                if p is -1:
                    return False

        return item.content.find(trigger, 0, p) > -1

    @classmethod
    def cmp(cls, comparison):
        left, right = [ i.content.split(b'\n') for i in comparison.pair ]
        del left[3]
        del right[3]
        if left == right:
            cls._log_same(comparison)
            return Same
        else:
            cls._log_different(comparison)
            cls._log_unidiffs([left, right], [ i.name for i in comparison.pair ])
            return Different

@_loggable
class ZipMemberMetadataComparator(Comparator):
    """
    Verify the metadata of each member of a zipfile.
    """

    @staticmethod
    def _applies(item):
        return item.box is ZipComparator

    @classmethod
    def cmp(cls, comparison):
        left, right = [ i.parent.zip.getinfo(i.shortname) for i in comparison.pair ]
        if left.compress_type == right.compress_type and left.comment == right.comment and left.create_system == right.create_system and left.create_version == right.create_version and left.extract_version == right.extract_version and left.reserved == right.reserved and left.flag_bits == right.flag_bits and left.volume == right.volume and left.internal_attr == right.internal_attr and left.external_attr == right.external_attr:
            return False
        else:
            cls._log_different(comparison)
            return Different

class Encoder(ContentOnlyBox):
    """
    Most UN*X compression programs compress a single stream of data.
    Similarly, many encryption programs do the same.
    """
    __metaclass__ = abc.ABCMeta
    _content_name = None

    @staticmethod
    @abc.abstractmethod
    def open(filename, mode, fileobj):
        raise NotImplementedError

    @classmethod
    def box_keys(cls, item):
        return [cls._content_name]

    @staticmethod
    def member_size(member):
        return len(member.content)

    @classmethod
    def cmp(cls, comparison):
        for p in comparison.pair:
            p.box = cls

        return Comparison(litem=Item(cls._packer.join(comparison.pair[0].name, cls._content_name), comparison.pair[0], box=cls), ritem=Item(cls._packer.join(comparison.pair[1].name, cls._content_name), comparison.pair[1], box=cls), comparators=comparison.comparators, ignores=comparison.ignores, exit_asap=comparison.exit_asap).cmp()

@_loggable
class GzipComparator(Encoder):
    """
    Gzip archives only have one member but the archive itself sadly
    includes a timestamp.  You can see the timestamp using "gzip -l -v".
    """
    _myname = b'gzip'
    _packer = _Packer((b'{{{}}}').format(_myname))
    _content_name = (b'{{{}content}}').format(_myname)

    @staticmethod
    @contextlib.contextmanager
    def open(filename, mode, fileobj):
        gz = gzip.GzipFile(filename, mode, 9, fileobj)
        yield gz
        gz.close()

    @staticmethod
    def _applies(item):
        return bytes(item.content[0:2]) == b'\x1f\x8b'

    @staticmethod
    def member_content(member):
        with GzipComparator.open(member.parent.name, b'rb', io.BytesIO(member.parent.content)) as (gzipobj):
            return gzipobj.read()

@_loggable
class BZ2Comparator(Encoder):
    """
    BZ2 archives only have one member.
    """
    _myname = b'bz2'
    _packer = _Packer((b'{{{}}}').format(_myname))
    _content_name = (b'{{{}content}}').format(_myname)

    @staticmethod
    @contextlib.contextmanager
    def open(filename, mode, fileobj):
        """
        .. todo:: remove openzip once we drop python-2.6
        """
        bobj = bz2.BZ2File(fileobj if fileobj else filename, mode, None, 9)
        yield bobj
        bobj.close()
        return

    @staticmethod
    def _applies(item):
        return bytes(item.content[0:2]) == b'BZ'

    @staticmethod
    def member_content(member):
        with BZ2Comparator.open(member.parent.name, b'rb', io.BytesIO(member.parent.content)) as (bz2obj):
            return bz2obj.read()

@_loggable
class XZComparator(Encoder):
    """
    XZ archives only have one member.
    """
    _myname = b'xz'
    _packer = _Packer((b'{{{}}}').format(_myname))
    _content_name = (b'{{{}content}}').format(_myname)

    @staticmethod
    @contextlib.contextmanager
    def open(filename, mode, fileobj):
        xzobj = lzma.LZMAFile(fileobj if fileobj else filename, mode)
        yield xzobj
        xzobj.close()

    @staticmethod
    def _applies(item):
        """
        ..note:: lzma format files have no magic number.  So while the lzma
                 library can open them, we don't really have a way to recognize
                 them easily other than just attempting to open and living with
                 failures.  But that seems pretty expensive and besides, who
                 uses lzma?
        """
        return bytes(item.content[0:6]) == b'\xfd7zXZ\x00'

    @staticmethod
    def member_content(member):
        with XZComparator.open(member.parent.name, b'rb', io.BytesIO(member.parent.content)) as (xzobj):
            return xzobj.read()

@_loggable
class FailComparator(Comparator):
    """
    Used as a catchall - just return Difference
    """

    @staticmethod
    def _applies(item):
        return True

    @classmethod
    def cmp(cls, comparison):
        cls._log_different(comparison)
        cls.logger.log(DIFFERENCES, b'\n')
        cls._log_unidiffs_comparison(comparison)
        return Different

def _findCommonSuffix(this, that):
    """
    find common trailing subpath.  return a 3-tuple consisting of the
    unique part of the arguments followed by the common part.
    """
    if not this or not that:
        return (this, that, b'')
    else:
        this_head, this_tail = os.path.split(this)
        that_head, that_tail = os.path.split(that)
        if this_tail == that_tail:
            x, y, z = _findCommonSuffix(this_head, that_head)
            return (
             x, y, os.path.join(z, this_tail))
        return (this, that, b'')

@_loggable
class BuriedPathComparator(Comparator):
    """
    Files which differ only in that they have their paths buried in them aren't really different.

    (currently unused).
    """

    @staticmethod
    def _applies(item):
        return item.isreg

    @classmethod
    def cmp(cls, comparison):
        this, that = comparison.pair
        this.head, that.head, tail = _findCommonSuffix(this.name, that.name)
        if this.content.find(bytes(this.head)) >= 0:
            this_content, that_content = [ bytearray(t.content).replace(bytes(t.head), b'@placeholder@') for t in comparison.pair ]
            if this_content == that_content:
                cls._log_same(comparison)
                return Same
        cls._log_indeterminate(comparison)
        return False

@_loggable
class SymlinkComparator(Comparator):
    """
    Symlinks are equal if they point to the same place.
    """

    @staticmethod
    def _applies(item):
        return item.islnk

    @classmethod
    def cmp(cls, comparison):
        this, that = [ p.link for p in comparison.pair ]
        if this == that:
            cls._log_same(comparison)
            return Same
        else:
            cls._log_different(comparison)
            return Different

@_loggable
class MapComparator(Comparator):
    """
    Linker map files include a reference to the output file which is
    typically a generated temp file name.
    """

    @staticmethod
    def _applies(item):
        try:
            retval = item.content.startswith(b'Archive member included')
        except UnicodeDecodeError:
            retval = False

        return retval

    _pattern = re.compile(b'tmp-\\d*')

    @classmethod
    def cmp(cls, comparison):
        munged = [ cls._pattern.sub(b'tmp-0', i.content) for i in comparison.pair ]
        if reduce(operator.eq, munged):
            cls._log_same(comparison)
            return Same
        else:
            cls._log_indeterminate(comparison)
            return False

@_loggable
class _ComparisonCommon(object):
    """
    This is a base class that holds utilities common to both
    :py:class:`Comparison` and :py:class:`ComparisonList`.  It is not
    intended to be instantiated.

    :param comparators: comparators to be applied
    :type comparators: list of :py:class:`Comparator`
    :param ignores: fnmatch style wild card patterns
    :type ignores: list of strings
    :param exit_asap: exit as soon as possible (Indeterminate is always raised asap)
    :type exit_asap: boolean
    """
    default_comparators = [
     NoSuchFileComparator,
     InodeComparator,
     EmptyFileComparator,
     DirComparator,
     ArMemberMetadataComparator,
     BitwiseComparator,
     SymlinkComparator,
     ElfComparator,
     ArComparator,
     AMComparator,
     ConfigLogComparator,
     KernelConfComparator,
     BZ2Comparator,
     GzipComparator,
     ZipComparator,
     TarMemberMetadataComparator,
     TarComparator,
     CpioMemberMetadataComparator,
     CpioComparator,
     MapComparator,
     DateBlotBitwiseComparator,
     FailComparator]

    def __init__(self, comparators=False, ignores=[], exit_asap=False):
        self.comparators = comparators if comparators is not False else self.default_comparators
        self.ignores = ignores
        self.exit_asap = exit_asap

    def ignoring(self, fname):
        return ignoring(self.ignores, fname)

    def cmp(self):
        self.logger.log(logging.FATAL, b'%s not implemented', self.__class__.__name__)

@_loggable
class Comparison(_ComparisonCommon):
    """
    Represents a pair of objects to be compared.

    An instance of :py:class:`Comparison` comprises a pair of
    :py:class:`Item`, a list of :py:class:`Comparator`, and a method
    for applying the list of :py:class:`Comparator` to the pair of
    :py:class:`Item` and returning an answer.

    If exit_asap is true, the first difference will end the
    comparison.  If it is not true, the comparison will continue
    despite knowing that our aggregate result is that we are
    :py:class:`Different`.  This is useful for getting a complete list
    of all differences.

    exit_asap=False is like "make -k" in the sense that it reports on
    all differences rather than stopping after the first.

    .. todo:: exit_asap is not currently functional.

    :param lname: path name of the first thing, (the leftmost one)
    :type lname: string
    :param rname: path name of the second thing, (the rightmost one)
    :type rname: string
    :param comparators: list of comparators to be applied
    :type comparators: list of :py:class:`Comparator`
    :param ignores: wild card patterns of path names to be ignored
    :type ignores: list of strings
    :param exit_asap: exit as soon as possible
    :type exit_asap: boolean
    """

    @property
    def pair(self):
        """
        A 2 item list of the items to be compared

        .. todo:: this should be a tuple.
        """
        return self._pair

    @pair.setter
    def pair(self, value):
        """setter"""
        self._pair = value

    def reset(self):
        self.logger.log(logging.DEBUG, b'resetting %s', self.pair[0].name)
        for item in self.pair:
            item.reset()

    def __init__(self, lname=b'', rname=b'', litem=False, ritem=False, comparators=False, ignores=[], exit_asap=False):
        _ComparisonCommon.__init__(self, comparators=comparators, ignores=ignores, exit_asap=exit_asap)
        if rname and not ritem:
            ritem = Items.find_or_create(rname, root, DirComparator)
        if lname and not litem:
            litem = Items.find_or_create(lname, root, DirComparator)
        self.pair = (litem, ritem)
        self.children = []
        for item in self.pair:
            i = self.ignoring(item.name)
            if i:
                self.logger.log(logging.ERROR, b'Creating comparison using ignored item %s cause %s', item.name, i)
                raise sys.exit(1)

    def cmp(self):
        """
        Compare our pair of :py:class:`Item`.

        Run through our list of :py:class:`Comparator` calling each
        one in turn with our pair of :py:class:`Item`. Each comparator
        is expected to return either:

        any non True value, (null, False, etc)
           indicating an indeterminate result, that is, that this particular
           comparator could make no authoritative determinations and that the
           next comparator in the list should be tried

        :py:class:`Same`
           an authoritative declaration that the items are
           sufficiently alike and thus no further comparators need be
           tried

        :py:class:`Different`
           an authoritative declaration that the items are
           insufficiently alike and thus no further comparators need
           be tried.

        If no :py:class:`Comparator` returns non-null, then
        :py:exc:`IndeterminateResult` will be raised.

        .. todo:: exit_asap is not currently functional.
        """
        for comparator in self.comparators:
            if not comparator.applies(self):
                self.logger.log(logging.DEBUG, b'does not apply - %s %s', comparator, self._pair[0].name)
                continue
            self.logger.log(logging.DEBUG, b'applies - %s %s', comparator, self._pair[0].name)
            result = comparator.cmp(self)
            if result:
                self.logger.log(logging.DEBUG, b'%s %s', result.__name__, self.__class__.__name__)
                self.reset()
                return result

        self.logger.log(INDETERMINATES, b'indeterminate result for %s', [ p.name for p in self._pair ])
        raise IndeterminateResult

@_loggable
class ComparisonList(_ComparisonCommon):
    """
    Represents a pair of lists of path names to be compared - one from
    column a, one from column b, etc.

    An instance of :py:class:`ComparisonList` is very similar to a
    :py:class:`Comparison` except that instead of a pair of Items, it
    comprises a pair of lists of path names

    :param stuff: path names to be compared
    :type stuff: a (2-element) list of lists of string

    In all other ways, this class resembles :py:class:`Comparison`.
    """

    def __init__(self, stuff, comparators=False, ignores=[], exit_asap=False):
        _ComparisonCommon.__init__(self, comparators=comparators, ignores=ignores, exit_asap=exit_asap)
        self.stuff = []
        for lst in stuff:
            new_lst = []
            for fname in lst:
                cause = self.ignoring(fname)
                if cause:
                    self.logger.log(SAMES, b"ignoring '%s' cause '%s' in %s", fname, cause, self.__class__.__name__)
                else:
                    new_lst.append(fname)

            self.stuff.append(new_lst)

    def cmp(self):
        Comparison.__doc__
        length = [ len(i) for i in self.stuff ]
        result = Same
        if not reduce(operator.eq, length):
            self.logger.log(DIFFERENCES, b'Different %s lists are of different sizes: %s', self.__class__.__name__, length)
            retval = Different
            if self.exit_asap:
                return retval
        for i in range(0, max(length)):
            comparison = Comparison(litem=Items.find_or_create(self.stuff[0][i], root), ritem=Items.find_or_create(self.stuff[1][i], root), comparators=self.comparators, ignores=self.ignores, exit_asap=self.exit_asap)
            c = comparison.cmp()
            if not c:
                self.logger.log(INDETERMINATES, b'Indeterminate %s', self.__class__.__name__)
                raise IndeterminateResult
            else:
                comparison.reset()
            if c is Different:
                self.logger.log(logging.DEBUG, b'Different %s', self.__class__.__name__)
                result = Different
                if self.exit_asap:
                    return result

        if result is Same:
            self.logger.log(SAMES, b'Same %s', self.__class__.__name__)
        return result

root = Item(b'{root}', True)


#!/usr/bin/env python
"""Compute DDT PC1 projections."""

from __future__ import division, print_function

__all__ = ['DDTManager']
__author__ = 'Hao Zhang'
__copyright__ = 'Copyright @2017 LAMDA'
__date__ = '2017-09-29'
__email__ = 'zhangh0214@gmail.com'
__license__ = 'CC BY-SA 3.0'
__status__ = 'Development'
__updated__ = '2017-10-03'
__version__ = '1.20'

import argparse
import itertools
import os
import sys
if sys.version[0] == '2':
    filter = itertools.ifilter
    input = raw_input
    map = itertools.imap
    range = xrange
    zip = itertools.izip

import numpy as np
import sklearn.decomposition

class DDTManager(object):
    """Manager class for split Oxford images, perform PCA on each cluster,
    and compute PC1 projection on Oxford features.

    Attributes:
        _paths, dict of (str, str): Data and project paths.
        _pca_model: PCA models for DDT.
    """
    def __init__(self, paths):
        self._paths = paths
        self._pca_model = None

    def fit(self):
        """Load via pool5 features and fit a PCA model."""
        print('Load via pool5 features.')
        pool5_list = sorted([
            f for f in os.listdir(self._paths['via_all_pool5'])
            if os.path.isfile(os.path.join(self._paths['via_all_pool5'], f))])
        m = len(pool5_list)
        all_pool5 = []
        for i, f in enumerate(pool5_list):
            if i % 100 == 0:
                print('procesing %d/%d.' % (i, m))
            pool5_i = np.load(os.path.join(self._paths['via_all_pool5'], f))
            D, H, W = pool5_i.shape
            assert D == 512
            pool5_i = np.reshape(pool5_i, (D, H * W)).T
            all_pool5.append(pool5_i)
        all_pool5 = np.vstack(all_pool5)
        assert all_pool5.shape[1] == 512

        print('Fit PCA.')
        self._pca_model = sklearn.decomposition.PCA(n_components=1)
        self._pca_model.fit(all_pool5)

    def apply(self, pool5_path, pc1_path):
        """Project test pool5 features according to the PCA model, for all
        images/cropped query images, respectively.

        Args:
            pool5_path, str: Pool5 feature path.
            pc1_path, str: PC1 projection path.
        """
        print('Compute PC1 projection for %s.' % pool5_path)
        pool5_names = sorted([
            f for f in os.listdir(self._paths[pool5_path])
            if os.path.isfile(os.path.join(self._paths[pool5_path], f))])
        m = len(pool5_names)
        for i, f in enumerate(pool5_names):
            if i % 100 == 0:
                print('Processing %d/%d' % (i, m))
            pool5_i = np.load(os.path.join(self._paths[pool5_path], f))
            D, H, W = pool5_i.shape
            assert D == 512
            pool5_i = np.reshape(pool5_i, (D, H * W)).T
            pc1_i = self._pca_model.transform(pool5_i)
            np.save(os.path.join(self._paths[pc1_path],
                                 os.path.splitext(os.path.basename(f))[0]),
                    pc1_i)

def main():
    """Main function of the program."""
    parser = argparse.ArgumentParser()
    parser.add_argument('--test', dest='test', type=str, required=True,
                        help='Dataset to evaluate.')
    parser.add_argument('--via', dest='via', type=str, required=True,
                        help='Dataset to assistant PCA whitening.')
    args = parser.parse_args()
    if args.test not in ['oxford', 'paris']:
        raise AttributeError('--test parameter must be oxford/paris.')
    if args.via not in ['oxford', 'paris']:
        raise AttributeError('--via parameter must be oxford/paris.')

    test_image_root = os.path.join('/data/zhangh/data/', args.test)
    test_data_root = os.path.join('/data/zhangh/project/ddt/data/', args.test)
    via_data_root = os.path.join('/data/zhangh/project/ddt/data/', args.via)
    paths = {
        'test_all_pool5': os.path.join(test_image_root, 'pool5/all/'),
        'test_crop_pool5': os.path.join(test_image_root, 'pool5/crop/'),
        'test_full_pool5': os.path.join(test_image_root, 'pool5/full/'),
        'via_all_pool5': os.path.join(via_data_root, 'pool5/'),
        'test_all_pc1': os.path.join(test_data_root, 'pc1/all/'),
        'test_crop_pc1': os.path.join(test_data_root, 'pc1/crop/'),
        'test_full_pc1': os.path.join(test_data_root, 'pc1/full/'),
    }
    for k in paths:
        assert os.path.isdir(paths[k])

    ddt_manager = DDTManager(paths)
    ddt_manager.fit()
    ddt_manager.apply('test_all_pool5', 'test_all_pc1')
    ddt_manager.apply('test_crop_pool5', 'test_crop_pc1')
    ddt_manager.apply('test_full_pool5', 'test_full_pc1')

if __name__ == '__main__':
    main()


import logging
import time
from multiprocessing import Pool
import numpy as np

log = logging.getLogger(__name__)

__all__ = ["Recommender", "Evaluation"]

class Recommender(object):
    def __init__(self, matrix):
        super(Recommender, self).__init__()
        self.matrix = matrix
        self.num_users = matrix.shape[0]
        self.num_items = matrix.shape[1]

    def train(self, before=None, after=None):
        raise NotImplementedError

    def predict(self, user, item):
        raise NotImplementedError

    def recommend(self, user, num=5, ruleout=True):
        scores = []
        for poi in xrange(self.num_items):
            scores.append((poi, self.predict(user, poi)))
        scores.sort(key=lambda x: x[1], reverse=True)

        if self.matrix is not None and ruleout:
            ruleouts = set(np.nonzero(self.matrix[user])[1])
        else:
            ruleouts = set()

        result = []
        for poi, score in scores:
            if poi in ruleouts:
                continue
            result.append(poi)
            if len(result) >= num:
                break
        return result

def _proxy_test(args):
    evaluation, user, full = args
    bingos = evaluation.hits(user)
    n = len(bingos)
    if full and n > 0:
        log.debug("user %i hit %s" % (user, bingos))
    return (user, n)

class Evaluation(object):
    def __init__(self, matrix, model, N=5, users=None, _pool_num=6):
        """
        Evaluate a model.Report precision and recall.
        matrix: test checkin matrix, `sparse matrix`
        model: model for test, must has `recommend` methid
        N    : recommend N pois
        users: users for test, should be iterated
        _pool_num: thread number to test, most cases default is ok.
                    if 0, then turn off multiple threads.
        usage:
            >>> from scipy import sparse
            >>> import numpy as np
            >>> matrix = sparse.csr_matrix(np.matrix([[0, 1], [1, 1]]))
            >>> class M(object):
            ...     def recommend(self, u, N):
            ...         if u == 0:
            ...             return [1, -1, -1, -1, -1]
            ...         return [-1, -1, -1, -1, -1]
            >>> ev = Evaluation(matrix, model=M(), users=[0, 1], _pool_num=0)
            >>> ev.test()
            (0.5, 0.1)

        """
        self.matrix = matrix
        self.N = N
        self.model = model
        self._pool_num = _pool_num
        self.num_users = matrix.shape[0]
        self.num_items = matrix.shape[1]
        if users is None:
            self.users = xrange(self.num_users)
        else:
            self.users = users

    def hits(self, user):
        pois = set(np.nonzero(self.matrix[user])[1])
        if len(pois) <= 0:
            return []
        result = self.model.recommend(user, self.N)
        return list(set(pois) & set(result))

    def test(self, full=False):
        t0 = time.time()
        def prepare():
            for user in self.users:
                yield (self, user, full)

        if self._pool_num > 0:
            pool = Pool(self._pool_num)
            matchs = pool.map(_proxy_test, prepare())
            pool.close()
            pool.join()
        else:
            matchs = []
            for arg in prepare():
                matchs.append(_proxy_test(arg))

        nhits = sum([n for u, n in matchs])
        _recall = 0.0
        valid_num = 0
        for user, n in matchs:
            pois = np.nonzero(self.matrix[user])[1]
            if len(pois) > 0:
                valid_num += 1
                _recall += float(n) / len(pois)

        if valid_num == 0:
            raise ValueError("Checkin matrix should not be empty.")
        prec = float(nhits) / (valid_num * self.N)
        _recall = float(_recall) / valid_num
        t1 = time.time()
        log.info("recall   : %.4f" % _recall)
        log.info("precision: %.4f" % prec)
        log.debug('time %.4f seconds' % (t1 - t0))
        return (_recall, prec)


import numpy as np

import matplotlib.pyplot as plt
from matplotlib import colors
import datetime
from qutip import *
import qutip.control.pulseoptim as pulseoptim

from .miscellaneous import toLatex

def plotExpectation(time,observables,result):
    """
    observables = [{'Hcavity':Hcavity}]
    """
    fig, axs = plt.subplots(len(observables), 1)
    if len(observables)==1: axs = [axs]
    for i, row in enumerate(observables):
        for name, observable in row.items():
            expectation = expect(observable, result.states)
            axs[i].plot(time, expectation, label=name)
        axs[i].legend()
    fig.tight_layout(); plt.show()

def plotWigners(
    rhoList,radius,
    nCol=6,titleList=None, ptraceIndex=None,
):
    if(len(rhoList)<nCol): nCol = len(rhoList)
    x = np.linspace(-radius, radius, 200)
    nRow = int(np.ceil(len(rhoList) / nCol))
    fig, axs = plt.subplots(nRow, nCol,
        figsize=(3*nCol,3*nRow)
    )
    for i, rho in enumerate(rhoList):
        if ptraceIndex is not None:
            rho = ptrace(rho, ptraceIndex)
        W = wigner(rho, x, x)
        row = i // nCol
        col = i %  nCol
        ax = axs[row][col] if nRow > 1 else axs[col]
        ax.contourf(
            x, x, W, 100, cmap=plt.get_cmap('RdBu'),
            norm=colors.Normalize(-.25,.25),
        )
        if titleList: ax.set_title(titleList[i])
    plt.tight_layout(); plt.show()

def makeTitle(t):
        return "t = {:.2f}".format(t)

def plotWignersIntermediateStates(
    time,result,number,
    radius, nCol=6,titleList=None, ptraceIndex=None,
):
    N = len( result.states )
    if number > 1: number = number-1
    step = N // number
    indices = np.arange(0,N,step)
    rhoList = [ket2dm(result.states[i]) for i in indices]
    rhoList.append(result.states[-1])
    titleList = [makeTitle(time[i]) for i in indices]
    titleList.append(makeTitle(time[-1]))
    plotWigners(
        rhoList,radius,
        nCol,titleList, ptraceIndex,
    )

def plotOccupations(
    rhoList,
    nCol=6,titleList=None, ptraceIndex=None,
):
    if(len(rhoList)<nCol): nCol = len(rhoList)
    if ptraceIndex is not None:
        rho0 = ptrace(rhoList[0], ptraceIndex)
    else:
        rho0 = rhoList[0]
    x = np.arange(0,rho0.diag().shape[0])
    nRow = int(np.ceil(len(rhoList) / nCol))
    fig, axs = plt.subplots(nRow, nCol, figsize=(3*nCol,3*nRow))
    for i, rho in enumerate(rhoList):
        if ptraceIndex is not None:
            rho = ptrace(rho, ptraceIndex)
        row = i // nCol
        col = i %  nCol
        ax = axs[row][col] if nRow > 1 else axs[col]
        ax.set_ylim(-0.2, 1.2)
        ax.bar(x, np.abs(rho.diag()))
        if titleList: ax.set_title(titleList[i])
    plt.tight_layout(); plt.show()

def plotOccupationsIntermediateStates(
    time,result,number,
    nCol=6,titleList=None, ptraceIndex=None,
):
    N = len( result.states )
    if number > 1: number = number-1
    step = N // number
    indices = np.arange(0,N,step)
    rhoList = [ket2dm(result.states[i]) for i in indices]
    rhoList.append(result.states[-1])
    titleList = [makeTitle(time[i]) for i in indices]
    titleList.append(makeTitle(time[-1]))
    plotOccupations(
        rhoList,
        nCol,titleList, ptraceIndex,
    )

def plotOptimalControl(result,controlsName,title='title'):
    fig, axs = plt.subplots(2,1)
    initial = result.initial_amps
    final = result.final_amps
    def stack(x,j): return np.hstack((x[:, j], x[-1, j]))
    pulseNames = ['initial','final']
    for i,pulse in enumerate([initial,final]):
        for j in range(pulse.shape[1]):
            axs[i].step(result.time, stack(pulse,j),
                        label=toLatex(controlsName[j]))
        axs[i].set_title(pulseNames[i]+" control")
        axs[i].legend()
    fig.tight_layout(); plt.show()
    print('*'*16 +' '+ title + " summary " + '*'*16)
    print("Final fidelity error {}".format(result.fid_err))
    print("Final gradient normal {}".format(result.grad_norm_final))
    print("Terminated due to {}".format(result.termination_reason))
    print("Number of iterations {}".format(result.num_iter))
    print("Completed in {} HH:MM:SS.US".format(
        datetime.timedelta(seconds=result.wall_time)))


# fmt: off
import sys  # isort: skip
from pathlib import Path  # isort: skip
ROOT = Path(__file__).resolve().parent.parent
sys.path.append(str(ROOT))
# fmt: on

from pathlib import Path

from rmt.updated_features import FEATURE_OUTFILES, Levelvars
from rmt.updated_predict import FeatureSlice, summarize_all_updated_predictions

if __name__ == "__main__":
    DEGREES = [3, 5, 7, 9]
    for feature, fname in FEATURE_OUTFILES.items():
        df = summarize_all_updated_predictions(
            feature_cls=feature,
            degrees=DEGREES,
            feature_slices=[*FeatureSlice],
            debug=False,
        )
        outfile = ROOT.parent / fname
        df.to_json(outfile)
        print(f"Saved predictions to {outfile}")


import mogp_emulator
import numpy as np

# simple MICE examples with 2 inputs

# simulator function -- needs to take a single input and output a single number

def f(x):
    return np.exp(-np.sum((x-2.)**2, axis = -1)/2.)

# Base design -- requires a list of parameter bounds if you would like to use
# uniform distributions. If you want to use different distributions, you
# can use any of the standard distributions available in scipy to create
# the appropriate ppf function (the inverse of the cumulative distribution).
# Internally, the code creates the design on the unit hypercube and then uses
# the distribution to map from [0,1] to the real parameter space.

ed = mogp_emulator.LatinHypercubeDesign([(0., 5.), (0., 5.)])

###################################################################################

# first example -- run entire design internally within the MICE class.

# first argument is base design (required), second is simulator function (optional,
# but required if you want the code to run the simualtions internally)

# Other optional arguments include:
# n_samples (number of sequential design steps, optional, default is not specified
# meaning that you will specify when running the sequential design)
# n_init (size of initial design, default 10)
# n_cand (number of candidate points, default is 50)
# nugget (nugget parameter for design GP, default is to set adaptively)
# nugget_s (nugget parameter for candidate GP, default is 1.)

md = mogp_emulator.MICEDesign(ed, f, n_samples = 20, n_init = 5, n_cand = 100)

md.run_sequential_design()

# get design and outputs

inputs = md.get_inputs()
targets = md.get_targets()

print("Example 1:")
print("Design inputs:\n", inputs)
print("Design targets:\n", targets)
print()

###################################################################################

# second example: run design manually

md2 = mogp_emulator.MICEDesign(ed, n_init = 5, n_cand = 100)

init_design = md2.generate_initial_design()

print("Example 2:")
print("Initial design:\n", init_design)

# run initial points manually

init_targets = np.array([f(s) for s in init_design])

# set initial targets

md2.set_initial_targets(init_targets)

# run 20 sequential design steps

for d in range(20):
    next_point = md2.get_next_point()
    next_target = f(next_point)
    md2.set_next_target(next_target)

# look at design and outputs

inputs = md2.get_inputs()
targets = md2.get_targets()

print("Final inputs:\n", inputs)
print("Final targets:\n", targets)

gp = mogp_emulator.GaussianProcess(inputs, targets)
gp.learn_hyperparameters()

testing = ed.sample(200000)

mean, unc, _ = gp.predict(testing)

print(mean)


import sys
import argparse
import numpy as np
import pandas as pd
from dtsr.config import Config
from dtsr.signif import bootstrap
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt

if __name__ == '__main__':

    argparser = argparse.ArgumentParser('''
        Computes pairwise significance of error differences between DTSR models and competitors.
        Assumes models are named using the template <MODELNAME>_<TASKNAME>, where <TASKNAME> is
        shared between models that should be compared. For example, if the config file contains
        4 models --- DTSR_TASK1, DTSR_TASK2, COMPETITOR_TASK1, and COMPETITOR_TASK2 --- the script
        will perform 2 comparisons: DTSR_TASK1 vs COMPETITOR_TASK1 and DTSR_TASK2 vs. COMPETITOR_TASK2.
    ''')
    argparser.add_argument('config_path', help='Path to configuration (*.ini) file')
    argparser.add_argument('-m', '--models', nargs='*', default=[], help='Path to configuration (*.ini) file')
    argparser.add_argument('-p', '--partition', type=str, default='dev', help='Name of partition to use (one of "train", "dev", "test")')
    argparser.add_argument('-M', '--metric', type=str, default='loss', help='Metric to use for comparison (either "loss" or "loglik")')
    argparser.add_argument('-t', '--tails', type=int, default=2, help='Number of tails (1 or 2)')
    args, unknown = argparser.parse_known_args()

    p = Config(args.config_path)
    if len(args.models) > 0:
        models = args.models
    else:
        models = p.model_list[:]

    run_baseline = False
    run_dtsr = False
    for m in models:
        if not run_baseline and m.startswith('LM') or m.startswith('GAM'):
            run_baseline = True
        elif not run_dtsr and m.startswith('DTSR'):
            run_dtsr = True

    sys.stderr.write('\n')
    dtsr_models = [x for x in models if x.startswith('DTSR')]

    if args.metric == 'loss':
        file_name = '/%s_losses_%s.txt'%(p.loss, args.partition)
    else:
        file_name = '/loglik_%s.txt'%args.partition

    for i in range(len(dtsr_models)):
        m1 = dtsr_models[i]
        for j in range(i+1, len(dtsr_models)):
            m2 = dtsr_models[j]
            name = '%s_v_%s' %(m1, m2)
            a = pd.read_csv(p.outdir + '/' + m1 + file_name, sep=' ', header=None, skipinitialspace=True)
            b = pd.read_csv(p.outdir + '/' + m2 + file_name, sep=' ', header=None, skipinitialspace=True)
            select = np.logical_and(np.isfinite(np.array(a)), np.isfinite(np.array(b)))
            diff = float(len(a) - select.sum())
            p_value, base_diff, diffs = bootstrap(a[select], b[select], n_iter=10000, n_tails=args.tails, mode=args.metric)
            sys.stderr.write('\n')
            with open(p.outdir + '/' + name + '_' + args.partition + '.txt', 'w') as f:
                f.write('='*50 + '\n')
                f.write('Model comparison: %s vs %s\n' %(m1, m2))
                if diff > 0:
                    f.write('%d NaN rows filtered out (out of %d)\n' %(diff, len(a)))
                f.write('Partition: %s\n' %args.partition)
                f.write('Loss difference: %.4f\n' %base_diff)
                f.write('p: %.4e\n' %p_value)
                f.write('='*50 + '\n')
            plt.hist(diffs, bins=1000)
            plt.savefig(p.outdir + '/' + name + '_' + args.partition + '.png')
            plt.close('all')


from PyQt5 import QtCore, QtWidgets
from PyQt5.Qt import *
from PyQt5.QtCore import *

class imageBox(QtWidgets.QLabel):
    signal_refresh_list = QtCore.pyqtSignal(int)

    def __init__(self, parent=None):
        super(imageBox, self).__init__(parent)
        self.points = []
        self.wires = []
        self.currentPoint = QPoint(0, 0)
        self.anchorPoint = QPoint(-1, -1)
        self.setCursor(Qt.CrossCursor)
        self.setMouseTracking(True)
        self.setContextMenuPolicy(Qt.CustomContextMenu)
        self.contextMenu = QMenu(self)
        self.customContextMenuRequested.connect(self.showMenu)
        self.reselectLast = self.contextMenu.addAction('Reselect last')
        self.abandon = self.contextMenu.addAction('Abandon')
        self.apply = self.contextMenu.addAction('Apply')
        self.reselectLast.triggered.connect(self.reselectLastMenu_triggered)
        self.abandon.triggered.connect(self.abandonMenu_triggered)
        self.apply.triggered.connect(self.applyMenu_triggered)

    def showMenu(self):
        self.contextMenu.exec_(QCursor.pos())

    def reselectLastMenu_triggered(self):
        if len(self.points) > 1:
            self.points.pop(-1)
            self.anchorPoint = self.points[len(self.points) - 1]
            self.update()
        elif len(self.points) == 1:
            self.points = []
            self.anchorPoint = QPoint(-1, -1)
            self.update()

    def abandonMenu_triggered(self):
        self.points = []
        self.anchorPoint = QPoint(-1, -1)
        self.update()

    def applyMenu_triggered(self):
        if len(self.points) >= 2:
            self.wires.append(self.points)
            self.points = []
            self.anchorPoint = QPoint(-1, -1)
            self.update()
            self.signal_refresh_list.emit(len(self.wires))

    def mouseMoveEvent(self, QMouseEvent):
        self.currentPoint = QMouseEvent.pos()
        self.update()

    def mousePressEvent(self, QMouseEvent):
        if QMouseEvent.buttons() == QtCore.Qt.LeftButton:
            self.anchorPoint = QMouseEvent.pos()
            self.points.append(self.anchorPoint)
            self.update()

    def paintEvent(self, QPaintEvent):
        p = QPainter(self)
        p.setPen(QPen(Qt.red, 2, Qt.SolidLine))
        font = QFont('宋体', 16, QFont.Bold, True)
        p.setFont(font)
        for idx, w in enumerate(self.wires):
            p.drawPolyline(self.convert2polygon(w))
            p.drawText(w[0].x(), w[0].y(), str(idx))
        if len(self.points) > 1:
            p.drawPolyline(self.convert2polygon(self.points))
        if self.anchorPoint.x() >= 0 and self.anchorPoint.y() >= 0:
            p.drawLine(self.anchorPoint.x(), self.anchorPoint.y(), self.currentPoint.x(), self.currentPoint.y())

    def convert2polygon(self, points):
        return QPolygon(points)

    def deleteAt(self, index):
        self.wires.pop(index)
        self.update()

    def clearAll(self):
        self.points = []
        self.wires = []
        self.currentPoint = QPoint(0, 0)
        self.anchorPoint = QPoint(-1, -1)
        self.update()


import pandas as pd
import os.path
import xlwt
import xlrd

excel_names = []
for i in range(1,5):
	fileName = "results/tagsFrame week-"+str(i)+".xlsx"
	if os.path.exists(fileName):
		excel_names.append(fileName)

excels = [pd.ExcelFile(name) for name in excel_names]
frames = [x.parse(x.sheet_names[0], header=None,index_col=None) for x in excels]
frames[1:] = [df[1:] for df in frames]
combined = pd.concat(frames)
combined.to_excel("results/tagsFrame.xlsx", header=False, index=False)