'''
This package provides a function and a base class:
    unify and AbstractSegmentation.

unify(data_sets, directory, size=None, segmentation_size=None, crop=False,
        min_frequency=None, min_coverage=None, synonyms=None,
        test_limit=None, single_process=False, verbose=False):

unify creates a stadnard format multichannel segementation
out of heterogenous labeled image sources.

Input:

    data_sets is a list of instances of subclasses of
    AbstractSegmentation (or similar objects).  Five methods
    must be provided on each object.

    names(cat, j) - a list of names that describe label j
        as it is used within category cat in this dataset.

    size() - number of images in the dataset.

    filename(i) - the jpg filename for the i'th image in the dataset.

    metadata(i) - a small pickleable record from which the
         segmentation of the ith item can be derived, withou reference
         to a specific corpus instance.

    @classmethod
    resolve_segmentation(m) - resolves to a dict and a shape of a segmentation.
         The dict contains string key names and numpy-array vals describing
         the segmentation of the image drived from metadata m.  The keys
         of this dict are arbitrary short strings, but for example could
         look like:
       {
         'scene': 1293, # 0-d: applies one label to the whole image
         'object': (1024x1280 int16), # 2-d: one label per pixel
         'part': (3x1024x1280 int16), # 3-d: layers of labels, dominant first
         'texture': [14,22] # 1-d: multiple whole-image labels, dominant first
       }

    The reason for the separation between metadata(i) and
    resolve_segmentation(m) so that we can run metadata(i) quickly on the
    main process, and then shard resolve_segmentation(m) out to worker
    processes.

Output is the following directory structure:
'''
README_TEXT = '''
This directory contains the following data and metadata files:

    images/[datasource]/...
         images drawn from a specific datasource are reformatted
         and scaled and saved as jpeg and png files in subdirectories
         of the images directory.

    index.csv
        contains a list of all the images in the dataset, together with
        available labeled data, in the form:

        image,split,ih,iw,sh,sw,[color,object,material,part,scene,texture]

        for examplle:
        dtd/g_0106.jpg,train,346,368,346,368,dtd/g_0106_color.png,,,,,314;194

        The first column is always the original image filename relative to
        the images/ subdirectory; then image height and width and segmentation
        heigh and width dimensions are followed by six columns for label
        data in the six categories.  Label data can be per-pixel or per-image
        (assumed to apply to every pixel in the image), and 0 or more labels
        can be specified per category.  If there are no labels, the field is ''.
        If there are multiple labels, they are separated by semicolons,
        and it is assumed that dominant interpretations are listed first.
        Per-image labels are represented by a decimal number; and per-image
        labels are represented by a filename for an image which encodes
        the per-pixel labels in the (red + 256 * green) channels.

    category.csv
        name,first,last,count,frequency

        for example:
        object,12,1138,529,208688

        In the generic case there may not be six categories; this directory
        may contain any set of categories of segmentations.  This file
        lists all segmentation categories found in the data sources,
        along with statistics on now many class labels apply to
        each category, and in what range; as well as how many
        images mention a label of that category

    label.csv
        number,name,category,frequency,coverage,syns

        for example:
        10,red-c,color(289),289,9.140027,
        21,fabric,material(36);object(3),39,4.225474,cloth

        This lists all label numbers and corresponding names, along
        with some statistics including the number of images for
        which the label appears in each category; the total number
        of images which have the label; and the pixel portions
        of images that have the label.

    c_[category].csv (for example, map_color.csv)
        code,number,name,frequency,coverage

        for example:
        4,31,glass,27,0.910724454131

        Although labels are store under a unified code, this file
        lists a standard dense coding that can be used for a
        specific subcategory of labels.
'''

import codecs
from functools import partial
import itertools
from multiprocessing import Pool, cpu_count
import numpy
import operator
import os
import re
import subprocess
import time
from PIL import Image
from collections import OrderedDict
from scipy.misc import imread, imresize, imsave
from scipy.ndimage.interpolation import zoom
import signal
import shutil
import sys
from unicsv import DictUnicodeWriter


def unify(data_sets, directory, size=None, segmentation_size=None, crop=False,
          splits=None, min_frequency=None, min_coverage=None,
          synonyms=None, test_limit=None, single_process=False, verbose=False):
    # Make sure we have a directory to work in
    directory = os.path.expanduser(directory)
    ensure_dir(directory)

    # Step 0: write a README file with generated information.
    write_readme_file([
        ('data_sets', data_sets), ('size', size),
        ('segmentation_size', segmentation_size), ('crop', crop),
        ('splits', splits),
        ('min_frequency', min_frequency), ('min_coverage', min_coverage),
        ('synonyms', synonyms), ('test_limit', test_limit),
        ('single_process', single_process)],
        directory=directory, verbose=verbose)

    # Clear old images data
    try:
        if verbose:
            print 'Removing old images.'
        shutil.rmtree(os.path.join(directory, 'images'), ignore_errors=True)
    except:
        pass
    ensure_dir(os.path.join(directory, 'images'))
    # Phase 1: Count label statistics
    # frequency = number of images touched by each label
    # coverage  = total portion of images covered by each label
    frequency, coverage = gather_label_statistics(
        data_sets, test_limit, single_process,
        segmentation_size, crop, verbose)
    # Phase 2: Sort, collapse, and filter labels
    labnames, syns = normalize_labels(data_sets, frequency, coverage, synonyms)
    report_aliases(directory, labnames, data_sets, frequency, verbose)
    # Phase 3: Filter by frequncy, and assign numbers
    names, assignments = assign_labels(
        labnames, frequency, coverage, min_frequency, min_coverage, verbose)
    # Phase 4: Collate and output label stats
    cats = write_label_files(
        directory, names, assignments, frequency, coverage, syns, verbose)
    # Phase 5: Create normalized segmentation files
    create_segmentations(
        directory, data_sets, splits, assignments, size, segmentation_size,
        crop, cats, test_limit, single_process, verbose)


def gather_label_statistics(data_sets, test_limit, single_process,
                            segmentation_size, crop, verbose):
    '''
    Phase 1 of unification.  Counts label statistics.
    '''
    # Count frequency and coverage for each individual image
    stats = map_in_pool(partial(count_label_statistics,
                                segmentation_size=segmentation_size,
                                crop=crop,
                                verbose=verbose),
                        all_dataset_segmentations(data_sets, test_limit),
                        single_process=single_process,
                        verbose=verbose)
    # Add them up
    frequency, coverage = (sum_histogram(d) for d in zip(*stats))
    # TODO: also, blacklist images that have insufficient labled pixels
    return frequency, coverage


def normalize_labels(data_sets, frequency, coverage, synonyms):
    '''
    Phase 2 of unification.
    Assigns unique label names and resolves duplicates by name.
    '''
    # Sort by frequency, descending, and assign a name to each label
    top_syns = {}
    labname = {}
    freq_items = zip(*sorted((-f, lab) for lab, f in frequency.items()))[1]
    for lab in freq_items:
        dataset, category, label = lab
        names = [n.lower() for n in data_sets[dataset].all_names(
            category, label) if len(n) and n != '-']
        if synonyms:
            names = synonyms(names)
        # Claim the synonyms that have not already been taken
        for name in names:
            if name not in top_syns:
                top_syns[name] = lab
        # best_labname may decide to collapse two labels because they
        # have the same names and seem to mean the same thing
        labname[lab], unique = best_labname(
            lab, names, labname, top_syns, coverage, frequency)
    return labname, top_syns


def report_aliases(directory, labnames, data_sets, frequency, verbose):
    '''
    Phase 2.5
    Report the aliases.  These are printed into 'syns.txt'
    '''
    show_all = True  # Print all details to help debugging
    name_to_lab = invert_dict(labnames)
    with codecs.open(os.path.join(directory, 'syns.txt'), 'w', 'utf-8') as f:
        def report(txt):
            f.write('%s\n' % txt)
            if verbose:
                print txt
        for name in sorted(name_to_lab.keys(),
                           key=lambda n: (-len(name_to_lab[n]), n)):
            keys = name_to_lab[name]
            if not show_all and len(keys) <= 1:
                break
            # Don't bother reporting aliases if all use the same index;
            # that is probably due to an underlying shared code.
            if not show_all and len(set(i for d, c, i in keys)) <= 1:
                continue
            report('name: %s' % name)
            for ds, cat, i in keys:
                names = ';'.join(data_sets[ds].all_names(cat, i))
                freq = frequency[(ds, cat, i)]
                report('%s/%s#%d: %d, (%s)' % (ds, cat, i, freq, names))
            report('')


def assign_labels(
        labnames, frequency, coverage, min_frequency, min_coverage, verbose):
    '''
    Phase 3 of unification.
    Filter names that are too infrequent, then assign numbers.
    '''
    # Collect by-name frequency and coverage
    name_frequency = join_histogram(frequency, labnames)
    name_coverage = join_histogram(coverage, labnames)
    names = name_frequency.keys()
    if min_frequency is not None:
        names = [n for n in names if name_frequency[n] >= min_frequency]
    if min_coverage is not None:
        names = [n for n in names if name_coverage[n] >= min_coverage]
    # Put '-' at zero
    names = [n for n in names if n != '-']
    names = ['-'] + sorted(names,
                           key=lambda x: (-name_frequency[x], -name_coverage[x]))
    nums = dict((n, i) for i, n in enumerate(names))
    assignments = dict((k, nums.get(v, 0)) for k, v in labnames.items())
    return names, assignments


def write_label_files(
        directory, names, assignments, frequency, coverage, syns, verbose):
    '''
    Phase 4 of unification.
    Collate some stats and then write then to two metadata files.
    '''
    # Make lists of synonyms claimed by each label
    synmap = invert_dict(
        dict((w, assignments[lab]) for w, lab in syns.items()))
    # We need an (index, category) count
    ic_freq = join_histogram_fn(frequency, lambda x: (assignments[x], x[1]))
    ic_cov = join_histogram_fn(coverage, lambda x: (assignments[x], x[1]))
    for z in [(j, cat) for j, cat in ic_freq if j == 0]:
        del ic_freq[z]
        del ic_cov[z]