import sys

import os
import zipfile
import tempfile
import sched, time
import numpy as np

from b3w import B3W
from glob import glob
from osgeo import gdal
from shutil import rmtree
from subprocess import Popen, PIPE, STDOUT
from typing import Any, List, Tuple, Set

from sentinel import Config
from sentinel import DataHub
from sentinel import Polygons
from utils import get_environment, print_snapshots, remove

# Main loop period (seconds)
INTERVAL: int = 10#0

def check_in_aws(s3: B3W, prefix: str, depth: int = 1) -> Set[str]:
    objects: Set[str] = set() #List[str] = []
    try:
        for s3o in s3.ls(prefix):
            if prefix:
                s3o = s3o.replace(prefix.strip('/'), '').strip('/')
            if not s3o:
                continue
            objects.add('/'.join(s3o.split('/')[:depth]))
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return objects

def get_from_aws(s3: B3W, prefix: str,
        path: str = '/dev/shm/gps/input'
    ) -> List[str]:
    files: List[str] = []
    try:
        #objects = s3.ls(prefix)
        #print(f"DEBUG: objects = {objects}")
        extensions = ('.geojson', '.shp', '.shx', '.dbf', '.prj', '.cpg')
        for s3o in s3.ls(prefix):
            #print(f"DEBUG: get_from_aws s3o = {s3o}")
            if prefix:
                filename = s3o.replace(prefix.strip('/'), '').strip('/')
            else:
                filename = s3o
            if not filename.endswith(extensions):
                continue
            #filename = os.path.join(path, *filename.lstrip('/').split('/'))
            filename = os.path.join(path, *filename.split('/'))
            #print(f"DEBUG: {s3o} -> {filename}")
            os.makedirs(os.path.dirname(filename), exist_ok=True)
            s3.get(s3o, filename)
            files.append(filename)
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return files

def put_to_aws(s3: B3W, prefix: str,
        path: str = '/dev/shm/gps/output'
    ) -> List[str]:
    objects: List[str] = []
    try:
        path = os.path.normpath(path)
        for filename in glob(os.path.join(path, '**'),  recursive=True):
            if not os.path.isfile(filename):
                continue
            s3o = os.path.relpath(filename, path)
            if prefix:
                s3o = '/'.join([prefix.strip('/'), *s3o.split(os.path.sep)])
            else:
                s3o = '/'.join([*s3o.split(os.path.sep)])
            #print(f"DEBUG: '{filename}' -> '{s3o}'")
            s3.put(filename, s3o, force=True)
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return objects

def remove_from_aws(s3: B3W, prefix: str) -> List[str]:
    # TODO: remove this function (DEBUG only)
    objects: List[str] = []
    try:
        #objects = s3.ls(prefix)
        #print(f"DEBUG: objects to remove = {objects}")
        for s3o in s3.ls(prefix):
            s3._B3W__s3r.Object(s3._B3W__bucket_name, s3o).delete()
            print(f"DEBUG: removed S3 object '{s3o}'")
            objects.append(s3o)
    except Exception as e:
        print(f"Error removing S3 object:\n{e}")

    return objects

def sync_with_aws(
        s3: B3W, prefix: str, data_hub: DataHub,
        snapshot: Any, path: str = '/dev/shm/gps/data'
    ) -> str:
    filename: str = None
    try:
        data_hub.config.output = os.path.join(path, snapshot.uuid)
        os.makedirs(data_hub.config.output, exist_ok=True)
        objects = s3.ls('/'.join([prefix, snapshot.uuid]))
        print(f"DEBUG: S3 objects = {objects}")
        #if len(objects) > 0:
        for s3o in objects:
            #s3o = s3o.replace(prefix, '').lstrip('/')
            if s3o.strip('/').endswith(snapshot.uuid):
                continue
            filename = os.path.join(path, s3o.replace(prefix, '').lstrip('/'))
            print(f"DEBUG: syncing '{s3o}' -> '{filename}'")
            s3.get(s3o, filename)
            break
        if not filename:
            data_hub.download(snapshot)
            #print(f"DEBUG: data_hub.config.output = {data_hub.config.output}")
            for filename in glob(os.path.join(data_hub.config.output, '*')):
            #print(f"DEBUG: filename = {filename}")
            #if len(filename) < 1:
            #    raise FileNotFoundError(f"failed to download {snapshot.uuid}!")
            #else:
            #    filename = filename[0]
                s3o = filename.replace(path, '').lstrip(os.path.sep)
                # TODO: check filename against snapshot.uuid
                s3o = '/'.join([prefix, s3o])
                print(f"DEBUG: syncing '{filename}' -> '{s3o}'")
                s3.put(filename, s3o)
                break
            if not filename:
                print(f"failed to download {snapshot.uuid}!")
        data_hub.config.output = None # TODO: devise something smart
    except FileNotFoundError as e:
        print(f"Failure: {e}")
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return filename

def set_debug_aws() -> None:
    s3_id, s3_key, s3_bucket, s3_input, s3_output, s3_sync = get_environment()
    path_input, path_output = ('/dev/shm/gps/input', '/dev/shm/gps/output')
    path_data = '/dev/shm/gps/data'
    #print(f"DEBUG:\n\t{'S3_ID'.ljust(16)}{s3_id}"
    #      f"\n\t{'S3_KEY'.ljust(16)}{s3_key}"
    #      f"\n\t{'S3_BUCKET'.ljust(16)}{s3_bucket}"
    #      f"\n\t{'S3_INPUT'.ljust(16)}{s3_input}"
    #      f"\n\t{'S3_OUTPUT'.ljust(16)}{s3_output}"
    #      f"\n\t{'S3_SYNC'.ljust(16)}{s3_sync}")
    s3 = B3W(s3_bucket, s3_id, s3_key)
    # DEBUG: put test data to S3
    remove_from_aws(s3, s3_input + '/test')
    remove_from_aws(s3, s3_output + '/test')
    for filename in glob(os.path.join('data', '*')):
        #s3.put('data/geotiff.xml', '/'.join([s3_input, 'test', 'geotiff.xml']))
        s3o = '/'.join([s3_input, 'test', os.path.basename(filename)])
        s3.put(filename, s3o)
        print(f"DEBUG: '{filename}' -> '{s3o}'")
    #print(s3.ls(s3_input))
    #sys.exit(0)

    return None

def process_sentinel1(filename, path_output, area, shapes):
    title = os.path.splitext(os.path.basename(filename))[0]
    options_warp = {
            'format': 'GTiff',
            'dstSRS': 'EPSG:32640',
            'creationOptions': ['COMPRESS=DEFLATE'],
            'xRes': 40,
            'yRes': 40
    }
    with tempfile.TemporaryDirectory() as path_temp:
        with zipfile.ZipFile(filename, 'r') as archive:
            archive.extractall(path_temp)
            path_safe = glob(os.path.join(path_temp, f"*.SAFE"))[0]
            dataset = gdal.Open(path_safe, gdal.GA_ReadOnly)
            subsets = dataset.GetSubDatasets()
            datasets = {}
            for i, p in enumerate(['HH', 'HV', 'RGB']):
                print(f"Reading {subsets[i][1]}...")
                datasets[p] = gdal.Open(subsets[i][0], gdal.GA_ReadOnly)
            filenames = []
            name_area = os.path.splitext(os.path.basename(area))[0]
            print(f"Warping polarizations...")
            for name, source in datasets.items():
                #
                # Prepare filenames and paths
                #
                for shape in shapes:
                    if shape:
                        name_shape = os.path.basename(shape)
                        name_shape = os.path.splitext(name_shape)[0]
                        data_prefix = f"{name_area}_{name_shape}"
                        options_cutline = {'cutlineDSName': shape,
                                           'cropToCutline': True}
                    else:
                        data_prefix = f"{name_area}"
                        options_cutline = {}
                    data_output = os.path.join(path_output, data_prefix)
                    if not name in ['RGB', 'INV']:
                        data_output = os.path.join(data_output, name.lower())
                        os.makedirs(data_output, exist_ok=True)
                        print(f"{data_output.replace(path_output, '')}")
                        destination = f"{os.path.join(data_output, title)}.tiff"
                        filenames.append(destination)
                        gdal.Warp(destination, source, **options_warp,
                                  **options_cutline)
                    else:
                        # Basic RGB processing
                        driver_mem = gdal.GetDriverByName('MEM')
                        memoset = driver_mem.CreateCopy('', source, 0)
                        band_hh = memoset.GetRasterBand(1)
                        image_hh = band_hh.ReadAsArray()
                        mask_hh = image_hh == 0
                        image_hh = np.ma.array(image_hh, mask=mask_hh,
                                               dtype=np.float32)
                        del mask_hh
                        band_hv = memoset.GetRasterBand(2)
                        image_hv = band_hv.ReadAsArray()
                        mask_hv = image_hv == 0
                        image_hv = np.ma.array(image_hv, mask=mask_hv,
                                               dtype=np.float32)
                        del mask_hv
                        stats_hh = (image_hh.mean().astype(np.float32),
                                    image_hh.std().astype(np.float32))
                        stats_hv = (image_hv.mean().astype(np.float32),
                                    image_hv.std().astype(np.float32))
                        image_hh = np.ma.tanh(image_hh / (stats_hh[0] + 2 *
                                                          stats_hh[1]))
                        image_hv = np.ma.tanh(image_hv / (stats_hv[0] + 2 *
                                                          stats_hv[1]))
                        image_ratio = image_hh / image_hv
                        stats_ratio = (image_ratio.mean().astype(np.float32),
                                       image_ratio.std().astype(np.float32))
                        image_ratio = image_ratio / image_ratio.max()
                        image_negative = (np.float32(1) -
                                          np.ma.tanh(image_hh / image_hv))
                        # Convert to byte type
                        image_hh = (image_hh * 254 + 1).astype(np.uint8)
                        image_hv = (image_hv * 254 + 1).astype(np.uint8)
                        image_ratio = (image_ratio * 254 + 1).astype(np.uint8)
                        image_negative = (image_negative * 254 + 1)\
                                         .astype(np.uint8)
                        # Write channels to the MEM dataset
                        memoset.AddBand()
                        band_ex = memoset.GetRasterBand(3)
                        band_ex.SetColorInterpretation(gdal.GCI_BlueBand)
                        band_hh.WriteArray(image_hh)
                        band_hh.SetColorInterpretation(gdal.GCI_RedBand)
                        band_hv.WriteArray(image_hv)
                        band_hv.SetColorInterpretation(gdal.GCI_GreenBand)
                        # Create ratio band (HH, HV, HH/HV)
                        band_ex.WriteArray(image_ratio)
                        band_ex.SetMetadata({'POLARISATION': 'HH/HV',
                                             'SWATH': 'EW'})
                        path_ratio = os.path.join(data_output, 'ratio')
                        os.makedirs(path_ratio, exist_ok=True)
                        print(f"{path_ratio.replace(path_output, '')}")
                        destination = f"{os.path.join(path_ratio, title)}.tiff"
                        filenames.append(destination)
                        gdal.Warp(destination, memoset, **options_warp,
                                  outputType=gdal.GDT_Byte, **options_cutline)
                        # Create negative band (HH, HV, 1 - HH/HV)
                        band_ex.WriteArray(image_negative)
                        band_ex.SetMetadata({'POLARISATION': '1 - HH/HV',
                                             'SWATH': 'EW'})
                        path_negative = os.path.join(data_output, 'negative')
                        os.makedirs(path_negative, exist_ok=True)
                        print(f"{path_negative.replace(path_output, '')}")
                        destination = f"{os.path.join(path_negative, title)}.tiff"
                        filenames.append(destination)
                        gdal.Warp(destination, memoset, **options_warp,
                                  outputType=gdal.GDT_Byte, **options_cutline)
    print(f"Done!")
    return filenames

def process_sentinel2(filename, path_output, area, shapes):
    title = os.path.splitext(os.path.basename(filename))[0]
    dataset = gdal.Open(filename, gdal.GA_ReadOnly)
    subsets = dataset.GetSubDatasets()
    assert len(subsets) > 0, f"no sub datasets found!"
    dataset = gdal.Open(subsets[0][0], gdal.GA_ReadOnly)
    #print(f"{snapshot.title} -->")
    print(f"Reading {subsets[0][1][:1].lower()}",
          f"{subsets[0][1][1:]}", sep='')
    image = dataset.ReadAsArray()
    if image.ndim < 3:
        image = image[None, ...]
    image = np.moveaxis(image[:3, ...], 0, -1) # CHW -> HWC
    image = image.astype(np.float32)
    print(f"Calculating optimal histogram...")
    clip = image.mean().astype(np.float32) * np.float32(2)
    image = image / clip
    image = np.tanh(image)
    # Apply gamma correction here (TODO)
    image = (image * 254 + 1).round().astype(np.uint8)
    # Apply nodata mask here (TODO)
    image = np.moveaxis(image, -1, 0) # HWC -> CHW
    print(f"Applying histogram...")
    tempset = gdal.GetDriverByName('MEM')\
              .CreateCopy('', dataset, 0)
    for i in range(image.shape[0]):
        band = tempset.GetRasterBand(i + 1)
        band.WriteArray(image[i].astype(np.uint16))
        del band
    print(f"Writing to temporary file...")
    with tempfile.TemporaryDirectory() as path_temp:
        temp = os.path.join(path_temp, 'temp.tiff')
        gdal.Translate(temp, tempset,
                       creationOptions=['COMPRESS=DEFLATE'],
                       format='GTiff', bandList=[1, 2, 3],
                       outputType=gdal.GDT_Byte)
        del tempset
        #
        # Prepare filenames and paths
        #
        filenames = []
        name_area = os.path.splitext(os.path.basename(area))[0]
        for shape in shapes:
            if shape:
                name_shape = os.path.basename(shape)
                name_shape = os.path.splitext(name_shape)[0]
                data_prefix = f"{name_area}_{name_shape}"
                options = {'cutlineDSName': shape,
                           'cropToCutline': True}
            else:
                data_prefix = f"{name_area}"
                options = {}
            data_output = os.path.join(path_output,
                                       #data_name,
                                       data_prefix)
            os.makedirs(data_output, exist_ok=True)
            destination = f"{os.path.join(data_output, title)}.tiff"
            filenames.append(destination)
            gdal.Warp(destination, temp, **options)
    print(f"Done!")
    return filenames

def main(periodic: sched.scheduler) -> None:
    # Set working variables
    s3_id, s3_key, s3_bucket, s3_input, s3_output, s3_sync = get_environment()
    path_input, path_output = ('/dev/shm/gps/input', '/dev/shm/gps/output')
    path_data = '/dev/shm/gps/data'

    #print(f"\n=== Started input processing cycle ===\n")
    s3 = B3W(s3_bucket, s3_id, s3_key)

    # Get input files from S3
    files_input = get_from_aws(s3, s3_input, path_input)
    #print("DEBUG: input files -->")
    #print("\n".join([f"DEBUG: {filename}" for filename in files_input]))
    objects_output = check_in_aws(s3, s3_output, depth=1)
    #print("DEBUG: output sets -->")
    #print("\n".join([f"DEBUG: {name}" for name in objects_output]))
    # DEBUG: list sync objects in S3, remove output test set
    #objects_sync = check_in_aws(s3, s3_sync) # don't uncomment - dangerous!
    #print("DEBUG: sync objects -->")
    #print("\n".join([f"DEBUG: {name}" for name in objects_sync]))

    # Initialize Copernicus Open Data Access Hub search object
    config = Config.load('config.yaml')
    data_hub = DataHub(config, limit=1000)

    # Cycle through all the data input sets: a set may contain multiple
    # input areas and shapes to process. Result will be a snapshot that is
    # cut with each shape (if any)
    for data_input in glob(os.path.join(path_input, '*')):
        if not os.path.isdir(data_input):
            #print(f"DEBUG: '{data_input}' is not a valid data input!")
            #print("TODO: unzip archived input sets...")
            continue
        data_name = os.path.basename(data_input)
        #print(f"DEBUG: 'data_input' basename = {data_name}")
        if data_name in objects_output:
            #print(f"Output set for '{data_input}' already exists. Skipping...")
            continue
        #print(f"DEBUG: input directory --->\n{os.listdir(data_input)}\n")
        areas = glob(os.path.join(data_input, '*.geojson'))
        shapes = glob(os.path.join(data_input, '*.shp'))
        #print(f"DEBUG: shapes = {shapes}")
        if not shapes:
            shapes.append(None)
        for area in areas:
            try:
                print(f"\n=== Processing '{area}' ===\n")
                polygon, properties = Polygons.read_geojson(area)
            except Exception as e:
                print(f"Failed to read '{area}'!\n{str(e)}")
                continue
            #print(f"DEBUG:\n{polygon}")

            # Set config key (search area)
            #print(f"DEBUG: config.search -->\n{config.search}")
            search = config.search.copy()
            search.update(properties)
            #config.search["footprint"] = f"\"Intersects({polygon})\""
            #print(f"DEBUG: config.search -->\n{config.search}")
            #print(f"Config 'search' section:\n{config.search}")

            snapshots = data_hub.search(search, area=polygon)
            snapshots = sorted(snapshots,
                               key=lambda item: item.begin_position)

            print(f"\n=== {len(snapshots)} snapshots found ===\n")
            # print_snapshots(snapshots) # DEBUG
            # break # DEBUG

            print(f"\n=== Processing snapshots and shapes ===\n")
            for index, snapshot in enumerate(snapshots):
                filename = sync_with_aws(s3, s3_sync, data_hub, snapshot,
                                         path_data)
                if not filename:
                    print(f"'\n{snapshot.uuid}' not synced. Skipping...")
                    continue
                else:
                    print(f"\n{index:8d}: {snapshot.title}")
                try:
                    # Process each superposition of an area and a shape
                    #
                    # Process a snapshot
                    #
                    #print(f"DEBUG: search keys = {search.keys()}")
                    path_target = os.path.join(path_output, data_name)
                    #print(f"DEBUG: path_data = '{path_data}'")
                    if search['platformName'] == 'Sentinel-2':
                        filenames = process_sentinel2(filename, path_target,
                                                      area, shapes)
                    elif search['platformName'] == 'Sentinel-1':
                        filenames = process_sentinel1(filename, path_target,
                                                      area, shapes)
                    else:
                        filenames = []
                        print(f"NOT IMPLEMENTED: {snapshot.title}",
                              f"{config.search['platformName']}")
                    #print(f"DEBUG: exporting '{data_prefix}' to S3 -->")
                    # Put processing result (for each output set) to S3
                    result = put_to_aws(s3, s3_output, path_output) # result...
                    for outfile in filenames:
                        remove(outfile) # all files (TODO: file or directory)
                except Exception as e:
                    print(f"FAILED: {e}")
                    raise e
                remove(filename) # remove snapshot
                #break # DEBUG: the first snapshot only
            print(f"\n=== Done snapshots for '{area}' ===\n")
        # Clean up output set (there should remain only logs)
        try:
            rmtree(os.path.join(path_output, data_name)) # data output - prefix
        except FileNotFoundError as e:
            pass
    # Clean up
    for path in (path_data, path_input, path_output):
        try:
            #print(f"DEBUG: removing {path}")
            rmtree(path)
        except FileNotFoundError as e:
            pass

    #print(f"\n=== Completed input processing cycle ===\n")
    periodic.enter(INTERVAL, 1, main, (periodic,))

    return None

if __name__ == '__main__':
    if os.getenv('GPS_DEBUG', '').lower() in ['1', 'y', 'on', 'yes', 'true']:
        set_debug_aws()
    periodic = sched.scheduler(time.time, time.sleep)
    periodic.enter(INTERVAL, 1, main, (periodic,))
    periodic.run()


class Stack(object):
    def __init__(self, initial_size=10):
        self.arr = [0 for _ in range(initial_size)]
        self.next_index = 0
        self.num_elements = 0

    def push(self, value):
        if self.next_index == len(self.arr):
            print('Out of space! Increasing array capacity...')
            self._handle_stack_capacity_full()

        self.arr[self.next_index] = value
        self.next_index += 1
        self.num_elements += 1

    def _handle_stack_capacity_full(self):
        old_arr = self.arr

        self.arr = [0 for _ in range(2*len(old_arr))]
        for index, element in enumerate(old_arr):
            self.arr[index] = element

    def size(self):
        return self.num_elements

    def is_empty(self):
        if self.num_elements == 0:
            return True
        else:
            return False

    def pop(self):
        if self.is_empty():
            self.next_index = 0
            return None

        self.arr[self.num_elements - 1] = 0
        self.num_elements -= 1
        self.next_index -= 1
        return self.arr[self.next_index]

foo = Stack(5)
foo.push(1)
foo.push(2)
foo.push(3)
foo.push(4)
foo.push(5)
foo.push(6)
print(foo.arr)
foo.pop()
foo.push(-4)
print(foo.arr)
# print(foo.size())
# print(foo.is_empty())


from __future__ import absolute_import, division, print_function
# LIBTBX_SET_DISPATCHER_NAME phenix.map_box

import mmtbx.utils
import mmtbx.model
from mmtbx.refinement import print_statistics
import iotbx.pdb
import libtbx.phil
from libtbx.utils import Sorry
import os, sys
from iotbx import reflection_file_utils
from iotbx.file_reader import any_file
from cctbx import maptbx
from scitbx.matrix import col
from six.moves import zip

master_phil = libtbx.phil.parse("""
  include scope libtbx.phil.interface.tracking_params
  pdb_file = None
    .type = path
    .help = Optional model file used to define region to be cut out
    .short_caption = Model file (optional)
    .style = file_type:pdb bold input_file
  map_coefficients_file = None
    .type = path
    .help = Input map coefficients file (alternative to ccp4 map file)
    .short_caption = Map coefficients
    .style = file_type:hkl bold input_file process_hkl child:map_labels:label
  label = None
    .type = str
    .short_caption = Map labels
    .help = Labels for map coefficients file
    .style = renderer:draw_map_arrays_widget parent:file_name:map_coefficients_file
  ccp4_map_file = None
    .help = Input map file (CCP4/mrc format).
    .short_caption = Input map file
    .type = str
  target_ncs_au_file = None
    .help = File with model indicating which au to choose in extract_unique
    .short_caption = Input target ncs au file
    .type = str
  half_map_list = None
    .type = strings
    .help = Half maps (extract_unique only). Supply file names \
              separated by spaces
    .short_caption = Half maps (extract_unique only)
  selection = all
    .type = str
    .help = Atom selection to be applied to input PDB file
    .short_caption = Atom selection (optional)
    .input_size = 400
  selection_radius = 3.0
    .type = float
    .help = Atoms within selection_radius of a selected atom model will be\
             kept as part of the selection.
    .short_caption = Selection radius
  box_cushion = 3.0
    .type = float
    .help = If mask_atoms is False, a box of density will be cut out around\
            the input model (after selections are applied to the model). \
            The size of the box of density will be box_cushion bigger than \
            the model.  Box cushion also applied if density_select is set.
    .short_caption = Box cushion

  mask_atoms=False
    .type=bool
    .help = Set map values to 0 outside molecular mask
    .short_caption = Mask atoms

  mask_atoms_atom_radius = 3.
    .type=float
     .help = Radius for masking around atoms
     .short_caption = Mask atoms atom radius

  value_outside_atoms = None
    .type = str
    .help = Set to 'mean' to make average value same inside and outside mask.
    .short_caption = Value outside atoms
  resolution_factor = 1./4
    .type = float
    .help = Resolution factor for calculation of map coefficients
    .short_caption = Resolution factor
  map_scale_factor = None
    .type = float
    .help = Scale factor to apply to map
    .short_caption = Map scale factor
  scale_max = 99999
    .type = float
    .help = Maximum value of amplitudes for output mtz file. If None, apply\
             volume scaling
    .short_caption = Scale max
  resolution = None
    .type = float
    .help = Resolution for calculation of output map coefficients. Default is \
            based on the gridding of the map (and may be higher-resolution than\
            you want).
    .short_caption = Resolution
  output_format = xplor *mtz *ccp4
    .type=choice(multi=True)
    .help = Output format(s) for boxed map. Note that mtz format is only\
            available if keep_origin=False or keep_map_size=True. (These \
            are the cases where the map is cut down to size and placed \
            at the origin or there is a full unit cell of data.)
    .short_caption = Output format

  output_file_name_prefix=None
    .type = str
    .help = Prefix for output file names. Default is name of the pdb file \
            without the ".pdb" suffix.
    .short_caption = Output file name prefix

  mask_select = False
    .type = bool
    .help = Select boundaries (min,max in x,y,z) based on auto-mask
    .short_caption = Mask select

  density_select = False
    .type = bool
    .help = Select boundaries based on where density is located.
    .short_caption = Density select

  density_select_threshold = 0.05
    .type = float
    .help = Choose region where density is this fraction of maximum or greater
    .short_caption = density_select threshold

  get_half_height_width = True
    .type = bool
    .help = Use 4 times half-width at half-height as estimate of max size \
              in density_select
    .short_caption = Use half-height width

  symmetry = None
    .type = str
    .help = Optional symmetry (e.g., D7, I, C2) to be used if extract_unique\
            is set.  Alternative to symmetry_file.  To find symmetry \
            automatically specify symmetry=ALL.
    .short_caption = Symmetry
  symmetry_file = None
    .type = path
    .help = Symmetry file.\
            Symmetry or symmetry_file required if extract_unique=True.  \
            May be a \
            Phenix .ncs_spec file or BIOMTR records or a resolve ncs file.
    .short_caption = Symmetry file

  sequence_file = None
    .type = path
    .help = Sequence file (any standard format). Can be unique part or \
            all copies.  Used in identification of unique part of map \
            and in masking with mask_select
    .short_caption = Sequence file (optional)

  molecular_mass = None
    .type = float
    .help = Molecular mass of object in map in Da (i.e., 33000 for 33 Kd).\
              Used in identification \
            of unique part of map and in masking by mask_select.
    .short_caption = Molecular mass (optional)

  solvent_content = None
    .type = float
    .help = Optional fraction of volume of map that is empty.  \
            Used in identification \
            of unique part of map and in masking by mask_select
    .short_caption = Solvent content

  extract_unique = False
    .type = bool
    .help = Extract unique part of map. Requires either symmetry_file or \
            symmetry and\
            either sequence file or molecular mass to be supplied. If chain \
            type is not protein it should be set as well.
    .short_caption = Extract unique

  box_buffer = 5
    .type = int
    .help = Padding around unique region in extract_unique
    .short_caption = Padding around unique region

  soft_mask_extract_unique = True
    .type = bool
    .help = Create soft mask at edges of extract_unique box (feather map into \
               edge of box). Uses resolution as mask_radius
      .short_caption = Soft mask in extract unique

  mask_expand_ratio = 1
      .type = int
      .help = Mask expansion relative to resolution for extract_unique
      .short_caption = Mask expand ratio

  regions_to_keep = None
    .type = int
    .short_caption = Regions to keep
    .help = You can specify a limit to the number of regions to keep\
             when generating the asymmetric unit of density.

  keep_low_density = True
    .type = bool
    .help = Get remainder (weak density) with extract_unique.
    .short_caption = Get remainder

  chain_type = None *PROTEIN DNA RNA
    .type = choice
    .help = Chain type. Only used if extract_unique is set. Has minor effect \
            in setting thresholds for identification of molecular region.\
            Use None if there is a mixture.
    .short_caption = Chain type

  soft_mask = False
    .type=bool
    .help = Use Gaussian mask in mask_atoms and on outside surface of box
    .short_caption = Soft mask

  soft_mask_radius=3
    .type=float
    .help = Gaussian mask smoothing radius
    .short_caption = Soft mask radius

  lower_bounds = None
    .type = ints
    .help = Lower bounds for cut out box. You can specify them directly.\
            NOTE: lower and upper bounds refer to grid points after shifting \
            the map to place the origin at (0,0,0). To refer to absolute \
            values specify bounds_are_absolute=True.
    .short_caption = Lower bounds

  upper_bounds = None
    .type = ints
    .help = Upper bounds for cut out box.  You can specify them directly.\
            NOTE: lower and upper bounds refer to grid points after shifting \
            the map to place the origin at (0,0,0). To refer to absolute \
            values specify bounds_are_absolute=True.
    .short_caption = Upper bounds

  bounds_are_absolute = False
    .type = bool
    .help = Define lower and upper bounds as absolute. \
            NOTE: lower and upper bounds refer to grid points after shifting \
            the map to place the origin at (0,0,0). To refer to absolute \
            values specify bounds_are_absolute=True.
    .short_caption = Bounds are absolute

  zero_outside_original_map = False
    .type = bool
    .help = If bounds for new map are outside original map, zero all points\
             outside of original map
    .short_caption = Zero outside original map
  keep_map_size = False
    .type=bool
    .help = Keep original map gridding (do not cut anything out). \
            Use to apply soft_mask and/or mask_atoms keeping same map size.
    .short_caption = Keep map size

  keep_origin = True
    .type=bool
    .help = Write out map, map_coefficients, and model \
            with origin in original location.  \
            If False, shift the origin to (0,0,0).  \
            NOTE: to cut out a part of a map, shift the origin to (0,0,0),\
               and make a new small map use keep_origin=False\
               keep_input_unit_cell_and_grid=False
    .short_caption = Keep origin

  keep_input_unit_cell_and_grid = True
     .type = bool
     .help = Keep the input unit_cell dimensions and unit_cell_grid. \
             If False, use the dimensions and grid of the cut out box as the \
              unit cell map_box dimensions and grid.\
            NOTE: to cut out a part of a map, shift the origin to (0,0,0),\
               and make a new small map set keep_origin=False and \
               keep_input_unit_cell_and_grid=False
     .short_caption = Keep input unit cell and grid

  output_unit_cell = None
     .type = floats
     .help = You can specify the unit cell for your map with 3 numbers. \
              This should normally\
             not be necessary. It can be used to fix a map that has the \
             wrong unit cell.
     .short_caption = Output unit cell
     .expert_level = 3

  output_unit_cell_grid = None
    .type = ints
    .help = You can specify the grid (3 integers) corresponding to the \
              output unit cell. \
              This can be used to specify the full grid for the unit cell. \
              if output_unit_cell is not specified, new unit cell parameters\
              will be generated to maintain the grid spacing.
    .short_caption = Output unit cell grid
    .expert_level = 3

  output_origin_grid_units = None
    .type = ints
    .help = You can specify the origin of your output map.  Normally you \
           should use keep_origin=True or False to specify your origin \
           but if you want to move it to a specific grid point you can do that.\
    .short_caption = Output origin
    .expert_level = 3

  output_origin_match_this_file = None
    .type = path
    .help = As output_origin_grid_units, but use origin from this file
    .short_caption = File with origin info

  bounds_match_this_file = None
    .type = path
    .help = Take the lower and upper bounds from this map file and apply them \
             to the input map file.
    .short_caption = File with bounds to match

  output_external_origin = None
    .type = floats
    .help = Write ORIGIN record to map file (this is an external origin \
            used to specify relationship to external files such as model files.\
            Three floating point numbers (A).
    .short_caption = output external origin

  restrict_map_size = False
    .type=bool
    .help = Do not go outside original map boundaries
    .short_caption = Restrict map size

  ignore_symmetry_conflicts = False
    .type=bool
    .help = Ignore unit cell from model if it conflicts with the map.
    .short_caption = Ignore symmetry conflicts

  output_ccp4_map_mean = None
    .type = float
    .help = Choose mean and SD of output CCP4 map
    .short_caption = Mean of output CCP4 map

  output_ccp4_map_sd = None
    .type = float
    .help = Choose mean and SD of output CCP4 map
    .short_caption = SD of output CCP4 map

  output_map_labels = None
    .type = str
    .multiple = True
    .help = Add this label to output map
    .short_caption = Add label

  gui
    .help = "GUI-specific parameter required for output directory"
  {
    output_dir = None
    .type = path
    .style = output_dir
  }
""", process_includes=True)

master_params = master_phil

def remove_element(text_list,element=None):
    new_text_list=[]
    for x in text_list:
      if x != element:
        new_text_list.append(x)
    return new_text_list

def run(args, crystal_symmetry=None,
     ncs_object=None,
     pdb_hierarchy=None,
     map_data=None,
     mask_data=None,
     half_map_data_list=None,
     half_map_labels_list=None,
     lower_bounds=None,
     upper_bounds=None,
     write_output_files=True,
     log=None):
  h = "phenix.map_box: extract box with model and map around selected atoms"
  if(log is None): log = sys.stdout
  print_statistics.make_header(h, out=log)
  default_message="""\

%s.

Usage:
  phenix.map_box model.pdb map_coefficients.mtz selection="chain A and resseq 1:10"

or

  phenix.map_box map.ccp4 density_select=True

Parameters:"""%h
  if(len(args) == 0 and not pdb_hierarchy):
    print(default_message)
    master_phil.show(prefix="  ")
    return

  # Process inputs ignoring symmetry conflicts just to get the value of
  #   ignore_symmetry_conflicts...

  inputs = mmtbx.utils.process_command_line_args(args = args,
      cmd_cs=crystal_symmetry,
      master_params = master_phil,
      suppress_symmetry_related_errors=True)
  params = inputs.params.extract()

  # Now process inputs for real and write a nice error message if necessary.
  try:
    inputs = mmtbx.utils.process_command_line_args(args = args,
      cmd_cs=crystal_symmetry,
      master_params = master_phil,
      suppress_symmetry_related_errors=params.ignore_symmetry_conflicts)
  except Exception as e:
    if str(e).find("symmetry mismatch ")>1:
      raise Sorry(str(e)+"\nTry 'ignore_symmetry_conflicts=True'")
    else:
      raise e

  params = inputs.params.extract()
  master_phil.format(python_object=params).show(out=log)

  # Overwrite params with parameters in call if available
  if lower_bounds:
     params.lower_bounds=lower_bounds
  if upper_bounds:
     params.upper_bounds=upper_bounds

  # PDB file
  if params.pdb_file and not inputs.pdb_file_names and not pdb_hierarchy:
    inputs.pdb_file_names=[params.pdb_file]
  if(len(inputs.pdb_file_names)!=1 and not params.density_select and not
    params.mask_select and not
    pdb_hierarchy and not params.keep_map_size and not params.upper_bounds
     and not params.extract_unique and not params.bounds_match_this_file):
    raise Sorry("PDB file is needed unless extract_unique, "+
      "density_select, mask_select, keep_map_size \nor bounds are set .")
  if (len(inputs.pdb_file_names)!=1 and not pdb_hierarchy and \
       (params.mask_atoms )):
    raise Sorry("PDB file is needed for mask_atoms")
  if params.soft_mask and (not params.resolution) and \
        (len(inputs.pdb_file_names)!=1 and not pdb_hierarchy):
    raise Sorry("Need resolution for soft_mask without PDB file")
  if ((params.density_select or params.mask_select) and params.keep_map_size):
    raise Sorry("Cannot set both density_select/mask_select and keep_map_size")
  if ((params.density_select or params.mask_select) and params.upper_bounds):
    raise Sorry("Cannot set both density_select/mask_select and bounds")
  if (params.keep_map_size and params.upper_bounds):
    raise Sorry("Cannot set both keep_map_size and bounds")
  if (params.upper_bounds and not params.lower_bounds):
    raise Sorry("Please set lower_bounds if you set upper_bounds")
  if (params.extract_unique):
    if (not params.resolution):
      raise Sorry("Please set resolution for extract_unique")
    if (not params.symmetry) and (not params.symmetry_file) and \
        (not ncs_object):
      raise Sorry(
        "Please supply a symmetry file or symmetry for extract_unique (you "+
       "\ncan try symmetry=ALL if you do not know your symmetry or "+
        "symmetry=C1 if \nthere is none)")
      from mmtbx.ncs.ncs import ncs
      ncs_object=ncs()
      ncs_object.set_unit_ncs()

  if params.keep_input_unit_cell_and_grid and (
      (params.output_unit_cell_grid is not None ) or
      (params.output_unit_cell is not None ) ):
    raise Sorry("If you set keep_input_unit_cell_and_grid then you cannot "+\
       "set \noutput_unit_cell_grid or output_unit_cell")

  if (write_output_files) and ("mtz" in params.output_format) and (
       (params.keep_origin) and (not params.keep_map_size)):
    print("\nNOTE: Skipping write of mtz file as keep_origin=True and \n"+\
       "keep_map_size is False\n")
    params.output_format=remove_element(params.output_format,element='mtz')

  if (write_output_files) and ("mtz" in params.output_format) and (
       (params.extract_unique)):
    print("\nNOTE: Skipping write of mtz file as extract_unique=True\n")
    params.output_format=remove_element(params.output_format,element='mtz')

  if params.output_origin_match_this_file or params.bounds_match_this_file:
    if params.output_origin_match_this_file:
      fn=params.output_origin_match_this_file
      if params.bounds_match_this_file:
        raise Sorry("Cannot match origin and bounds at same time")
    else:
      fn=params.bounds_match_this_file
    if not params.ccp4_map_file:
      raise Sorry(
       "Need to specify your input file with ccp4_map_file=xxx if you use "+
        "output_origin_match_this_file=xxxx or bounds_match_this_file=xxxx")

    af = any_file(fn)
    if (af.file_type == 'ccp4_map'):
      origin=af.file_content.data.origin()
      if params.output_origin_match_this_file:
        params.output_origin_grid_units=origin
        print("Origin of (%s,%s,%s) taken from %s" %(
           origin[0],origin[1],origin[2],fn))
      else:
        all=af.file_content.data.all()
        params.lower_bounds=origin
        print("Lower bounds of (%s,%s,%s) taken from %s" %(
           params.lower_bounds[0],params.lower_bounds[1],
             params.lower_bounds[2],fn))
        params.upper_bounds=list(col(origin)+col(all)-col((1,1,1)))
        print("upper bounds of (%s,%s,%s) taken from %s" %(
           params.upper_bounds[0],params.upper_bounds[1],
            params.upper_bounds[2],fn))
        params.bounds_are_absolute=True
    else:
      raise Sorry("Unable to interpret %s as map file" %(fn))

  if params.output_origin_grid_units is not None and params.keep_origin:
    params.keep_origin=False
    print("Setting keep_origin=False as output_origin_grid_units is set")
  print_statistics.make_sub_header("pdb model", out=log)
  if len(inputs.pdb_file_names)>0:
    pdb_inp = iotbx.pdb.input(file_name=inputs.pdb_file_names[0])
    pdb_hierarchy = pdb_inp.construct_hierarchy()
  if pdb_hierarchy:
    pdb_atoms = pdb_hierarchy.atoms()
    pdb_atoms.reset_i_seq()
  else:
    pdb_hierarchy=None
  # Map or map coefficients
  map_coeff = None
  input_unit_cell_grid=None
  input_unit_cell=None
  input_map_labels=None
  if (not map_data):
    # read first mtz file
    if ( (len(inputs.reflection_file_names) > 0) or
         (params.map_coefficients_file is not None) ):
      # file in phil takes precedent
      if (params.map_coefficients_file is not None):
        if (len(inputs.reflection_file_names) == 0):
          inputs.reflection_file_names.append(params.map_coefficients_file)
        else:
          inputs.reflection_file_names[0] = params.map_coefficients_file
      map_coeff = reflection_file_utils.extract_miller_array_from_file(
        file_name = inputs.reflection_file_names[0],
        label     = params.label,
        type      = "complex",
        log       = log)
      if not crystal_symmetry: crystal_symmetry=map_coeff.crystal_symmetry()
      fft_map = map_coeff.fft_map(resolution_factor=params.resolution_factor)
      fft_map.apply_sigma_scaling()
      map_data = fft_map.real_map_unpadded()
      map_or_map_coeffs_prefix=os.path.basename(
         inputs.reflection_file_names[0][:-4])
    # or read CCP4 map
    elif ( (inputs.ccp4_map is not None) or
           (params.ccp4_map_file is not None) ):
      if (params.ccp4_map_file is not None):
        af = any_file(params.ccp4_map_file)
        if (af.file_type == 'ccp4_map'):
          inputs.ccp4_map = af.file_content
          inputs.ccp4_map_file_name = params.ccp4_map_file
      print_statistics.make_sub_header("CCP4 map", out=log)
      ccp4_map = inputs.ccp4_map
      ccp4_map.show_summary(prefix="  ",out=log)
      if not crystal_symmetry: crystal_symmetry=ccp4_map.crystal_symmetry()
      map_data = ccp4_map.data #map_data()
      input_unit_cell_grid=ccp4_map.unit_cell_grid
      input_unit_cell=ccp4_map.unit_cell_parameters
      input_map_labels=ccp4_map.get_labels()

      if inputs.ccp4_map_file_name.endswith(".ccp4"):
        map_or_map_coeffs_prefix=os.path.basename(
          inputs.ccp4_map_file_name[:-5])
      else:
        map_or_map_coeffs_prefix=os.path.basename(
          inputs.ccp4_map_file_name[:-4])
  else: # have map_data
    map_or_map_coeffs_prefix=None

  if params.half_map_list and (not half_map_data_list):
    if not params.extract_unique:
      raise Sorry("Can only use half_map_with extract_unique")
    print ("Reading half-maps",params.half_map_list)
    half_map_data_list=[]
    half_map_labels_list=[]
    for fn in params.half_map_list:
      print("Reading half map from %s" %(fn),file=log)
      af = any_file(fn)
      print_statistics.make_sub_header("CCP4 map", out=log)
      h_ccp4_map = af.file_content
      h_ccp4_map.show_summary(prefix="  ",out=log)
      h_map_data = h_ccp4_map.data
      half_map_data_list.append(h_map_data)
      half_map_labels_list.append(h_ccp4_map.get_labels())

  if params.map_scale_factor:
    print("Applying scale factor of %s to map data on read-in" %(
       params.map_scale_factor))
    map_data=map_data*params.map_scale_factor

  if params.output_origin_grid_units is not None:
    origin_to_match=tuple(params.output_origin_grid_units)
  else:
    origin_to_match=None

  if origin_to_match:
    sc=[]
    for x,o,a in zip(crystal_symmetry.unit_cell().parameters()[:3],
        origin_to_match,
        map_data.all()):
      sc.append(-x*o/a)
    shift_cart_for_origin_to_match=tuple(sc)
  else:
    origin_to_match=None
    shift_cart_for_origin_to_match=None

  if crystal_symmetry and not inputs.crystal_symmetry:
    inputs.crystal_symmetry=crystal_symmetry

  # final check that map_data exists
  if(map_data is None):
    raise Sorry("Map or map coefficients file is needed.")

  if len(inputs.pdb_file_names)>0:
    output_prefix=os.path.basename(inputs.pdb_file_names[0])[:-4]
  else:
    output_prefix=map_or_map_coeffs_prefix

  if not pdb_hierarchy: # get an empty hierarchy
    from cctbx.array_family import flex
    pdb_hierarchy=iotbx.pdb.input(
      source_info='',lines=flex.split_lines('')).construct_hierarchy()
  xray_structure = pdb_hierarchy.extract_xray_structure(
    crystal_symmetry=inputs.crystal_symmetry)
  xray_structure.show_summary(f=log)
  #
  if not params.selection: params.selection="all"
  selection = pdb_hierarchy.atom_selection_cache().selection(
    string = params.selection)
  if selection.size():
    print_statistics.make_sub_header("atom selection", out=log)
    print("Selection string: selection='%s'"%params.selection, file=log)
    print("  selects %d atoms from total %d atoms."%(selection.count(True),
        selection.size()), file=log)
  sites_cart_all = xray_structure.sites_cart()
  sites_cart = sites_cart_all.select(selection)
  selection = xray_structure.selection_within(
    radius    = params.selection_radius,
    selection = selection)

  if not ncs_object:
    from mmtbx.ncs.ncs import ncs
    ncs_object=ncs()
    if params.symmetry_file:
      ncs_object.read_ncs(params.symmetry_file,log=log)
      print("Total of %s operators read" %(ncs_object.max_operators()), file=log)
  if not ncs_object or ncs_object.max_operators()<1:
      print("No symmetry available", file=log)
  if ncs_object:
    n_ops=max(1,ncs_object.max_operators())
  else:
    n_ops=1

  # Get sequence if extract_unique is set
  sequence=None
  if params.extract_unique or params.mask_select:
    if params.sequence_file:
      if n_ops > 1: # get unique part of sequence
        remove_duplicates=True
      else:
        remove_duplicates=False
      from iotbx.bioinformatics import get_sequences
      sequence=(" ".join(get_sequences(file_name=params.sequence_file,
        remove_duplicates=remove_duplicates)))

    if params.chain_type in ['None',None]: params.chain_type=None
    if sequence and not params.molecular_mass:
      # get molecular mass from sequence
      from iotbx.bioinformatics import text_from_chains_matching_chain_type
      if params.chain_type in [None,'PROTEIN']:
        n_protein=len(text_from_chains_matching_chain_type(
          text=sequence,chain_type='PROTEIN'))
      else:
        n_protein=0
      if params.chain_type in [None,'RNA']:
        n_rna=len(text_from_chains_matching_chain_type(
          text=sequence,chain_type='RNA'))
      else:
        n_rna=0
      if params.chain_type in [None,'DNA']:
        n_dna=len(text_from_chains_matching_chain_type(
         text=sequence,chain_type='DNA'))
      else:
        n_dna=0
      params.molecular_mass=n_ops*(n_protein*110+(n_rna+n_dna)*330)
      print("\nEstimate of molecular mass is %.0f " %(params.molecular_mass), file=log)
  if params.density_select or params.mask_select:
    print_statistics.make_sub_header(
    "Extracting box around selected density and writing output files", out=log)
  else:
   print_statistics.make_sub_header(
    "Extracting box around selected atoms and writing output files", out=log)
  #
  if params.value_outside_atoms=='mean':
    print("\nValue outside atoms mask will be set to mean inside mask", file=log)
  if params.get_half_height_width and params.density_select:
    print("\nHalf width at half height will be used to id boundaries", file=log)

  if params.soft_mask and sites_cart_all.size()>0:
    print("\nSoft mask will be applied to model-based mask", file=log)
  elif params.soft_mask:
    print ("\nSoft mask will be applied to outside of map box",file=log)
  if params.keep_map_size:
    print("\nEntire map will be kept (not cutting out region)", file=log)
  if params.restrict_map_size:
    print("\nOutput map will be within input map", file=log)
  if params.lower_bounds and params.upper_bounds:
    print("Bounds for cut out map are (%s,%s,%s) to (%s,%s,%s)" %(
     tuple(list(params.lower_bounds)+list(params.upper_bounds))), file=log)

  if mask_data:
    mask_data=mask_data.as_double()
  box = mmtbx.utils.extract_box_around_model_and_map(
    xray_structure   = xray_structure,
    map_data         = map_data.as_double(),
    mask_data        = mask_data,
    box_cushion      = params.box_cushion,
    selection        = selection,
    mask_select      = params.mask_select,
    density_select   = params.density_select,
    threshold        = params.density_select_threshold,
    get_half_height_width = params.get_half_height_width,
    mask_atoms       = params.mask_atoms,
    soft_mask        = params.soft_mask,
    soft_mask_radius = params.soft_mask_radius,
    mask_atoms_atom_radius = params.mask_atoms_atom_radius,
    value_outside_atoms = params.value_outside_atoms,
    keep_map_size         = params.keep_map_size,
    restrict_map_size     = params.restrict_map_size,
    lower_bounds          = params.lower_bounds,
    upper_bounds          = params.upper_bounds,
    bounds_are_absolute   = params.bounds_are_absolute,
    zero_outside_original_map   = params.zero_outside_original_map,
    extract_unique        = params.extract_unique,
    target_ncs_au_file    = params.target_ncs_au_file,
    regions_to_keep       = params.regions_to_keep,
    box_buffer            = params.box_buffer,
    soft_mask_extract_unique = params.soft_mask_extract_unique,
    mask_expand_ratio = params.mask_expand_ratio,
    keep_low_density      = params.keep_low_density,
    chain_type            = params.chain_type,
    sequence              = sequence,
    solvent_content       = params.solvent_content,
    molecular_mass        = params.molecular_mass,
    resolution            = params.resolution,
    ncs_object            = ncs_object,
    symmetry              = params.symmetry,
    half_map_data_list    = half_map_data_list,
    )

  ph_box = pdb_hierarchy.select(selection)
  ph_box.adopt_xray_structure(box.xray_structure_box)
  box.hierarchy=ph_box

  if params.mask_select:
    print("\nSolvent content used in mask_select: %.3f " %(
      box.get_solvent_content()),file=log)
  if (inputs and
    inputs.crystal_symmetry and inputs.ccp4_map and
    inputs.crystal_symmetry.unit_cell().parameters() and
     inputs.ccp4_map.unit_cell_parameters  ) and (
       inputs.crystal_symmetry.unit_cell().parameters() !=
       inputs.ccp4_map.unit_cell_parameters):
    print("\nNOTE: Input CCP4 map is only part of unit cell:", file=log)
    print("Full unit cell ('unit cell parameters'): "+\
      "(%.1f, %.1f, %.1f, %.1f, %.1f, %.1f) A" %tuple(
        inputs.ccp4_map.unit_cell_parameters), file=log)
    print("Size of CCP4 map 'map unit cell':        "+\
      "(%.1f, %.1f, %.1f, %.1f, %.1f, %.1f) A" %tuple(
       inputs.crystal_symmetry.unit_cell().parameters()), file=log)
    print("Full unit cell as grid units: (%s, %s, %s)" %(
      inputs.ccp4_map.unit_cell_grid), file=log)
    print("Map unit cell as grid units:  (%s, %s, %s)" %(
      map_data.all()), file=log)

    box.unit_cell_parameters_from_ccp4_map=inputs.ccp4_map.unit_cell_parameters
    box.unit_cell_parameters_deduced_from_map_grid=\
       inputs.crystal_symmetry.unit_cell().parameters()

  else:
    box.unit_cell_parameters_from_ccp4_map=None
    box.unit_cell_parameters_deduced_from_map_grid=None

  if box.pdb_outside_box_msg:
    print(box.pdb_outside_box_msg, file=log)

  # NOTE: box object is always shifted to place origin at (0,0,0)

  # NOTE ON ORIGIN SHIFTS:  The shifts are applied locally here. The box
  #  object is not affected and always has the origin at (0,0,0)
  #  output_box is copy of box with shift_cart corresponding to the output
  #   files. Normally this is the same as the original shift_cart. However
  #   if user has specified a new output origin it will differ.

  # For output files ONLY:
  #  keep_origin==False leave origin at (0,0,0)
  #  keep_origin==True: we shift everything back to where it was,
  #  output_origin_grid_units=10,10,10: output origin is at (10,10,10)

  # ncs_object is original
  #  box.ncs_object is shifted by shift_cart
  #  output_box.ncs_object is shifted back by -new shift_cart

  # Additional note on output unit_cell and grid_units.
  # The ccp4-style output map can specify the unit cell and grid units
  #  corresponding to that cell.  This can be separate from the origin and
  #  number of grid points in the map as written.  If specified, write these
  #  out to the output ccp4 map and also use this unit cell for writing
  #  any output PDB files

  from copy import deepcopy
  output_box=deepcopy(box)  # won't use box below here except to return it

  print("\nBox cell dimensions: (%.2f, %.2f, %.2f) A" %(
      box.box_crystal_symmetry.unit_cell().parameters()[:3]), file=log)

  if box.shift_cart:
     print("Working origin moved from grid position of"+\
        ": (%d, %d, %d) to (0,0,0) " %(
        tuple(box.origin_shift_grid_units(reverse=True))), file=log)
     print("Working origin moved from  coordinates of:"+\
        " (%.2f, %.2f, %.2f) A to (0,0,0)\n" %(
         tuple(-col(box.shift_cart))), file=log)

  if (params.keep_origin):
    print("\nRestoring original position for output files", file=log)
    print("Origin will be at grid position of"+\
        ": (%d, %d, %d) " %(
        tuple(box.origin_shift_grid_units(reverse=True))), file=log)
    print("\nOutput files will be in same location as original", end=' ', file=log)
    if not params.keep_map_size:
      print("just cut out.", file=log)
    else:
      print("keeping entire map", file=log)
    print("Note that output maps are only valid in the cut out region.\n", file=log)

  else:
    if origin_to_match:
      output_box.shift_cart=shift_cart_for_origin_to_match
      if params.output_origin_grid_units:
        print("Output map origin to be shifted to match target", file=log)
      print("Placing origin at grid point (%s, %s, %s)" %(
        origin_to_match)+"\n"+ \
        "Final coordinate shift for output files: (%.2f,%.2f,%.2f) A\n" %(
        tuple(col(output_box.shift_cart)-col(box.shift_cart))), file=log)
    elif box.shift_cart:
      output_box.shift_cart=(0,0,0) # not shifting back
      print("Final origin will be at (0,0,0)", file=log)
      print("Final coordinate shift for output files: (%.2f,%.2f,%.2f) A\n" %(
        tuple(col(output_box.shift_cart)-col(box.shift_cart))), file=log)
    else:
      print("\nOutput files are in same location as original and origin "+\
        "is at (0,0,0)\n", file=log)

  print("\nBox grid: (%s, %s, %s) " %(output_box.map_box.all()),file=log)
  ph_output_box_output_location = ph_box.deep_copy()
  if output_box.shift_cart:  # shift coordinates and NCS back by shift_cart
    # NOTE output_box.shift_cart could be different than box.shift_cart if
    #  there is a target position for the origin and it is not the same as the
    #  original origin.
    sites_cart = output_box.shift_sites_cart_back(
      output_box.xray_structure_box.sites_cart())
    xrs_offset = ph_output_box_output_location.extract_xray_structure(
        crystal_symmetry=output_box.xray_structure_box.crystal_symmetry()
          ).replace_sites_cart(new_sites = sites_cart)
    ph_output_box_output_location.adopt_xray_structure(xrs_offset)

    if output_box.ncs_object:
      output_box.ncs_object=output_box.ncs_object.coordinate_offset(
         tuple(-col(output_box.shift_cart)))
    shift_back=True
  else:
    shift_back=False

  if params.keep_input_unit_cell_and_grid and \
       (input_unit_cell_grid is not None) and \
       (input_unit_cell is not None):
    params.output_unit_cell=input_unit_cell
    params.output_unit_cell_grid=input_unit_cell_grid
    print("Setting output unit cell parameters and unit cell grid to"+\
      " match\ninput map file", file=log)

  if params.output_unit_cell: # Set output unit cell parameters
    from cctbx import crystal
    output_crystal_symmetry=crystal.symmetry(
      unit_cell=params.output_unit_cell, space_group="P1")
    output_unit_cell=output_crystal_symmetry.unit_cell()
    print("Output unit cell set to: %.2f, %.2f, %.2f, %.2f, %.2f, %.2f)" %tuple(
        output_crystal_symmetry.unit_cell().parameters()), file=log)
  else:
    output_crystal_symmetry=None

  # =============  Check/set output unit cell grid and cell parameters =======
  if params.output_unit_cell_grid or output_crystal_symmetry:
    if params.output_unit_cell_grid:
      output_unit_cell_grid=params.output_unit_cell_grid
    else:
      output_unit_cell_grid=output_box.map_box.all()
    print("Output unit cell grid set to: (%s, %s, %s)" %tuple(
        output_unit_cell_grid), file=log)

    expected_output_abc=[]
    box_spacing=[]
    output_spacing=[]
    box_abc=output_box.xray_structure_box.\
         crystal_symmetry().unit_cell().parameters()[:3]
    if output_crystal_symmetry:
      output_abc=output_crystal_symmetry.unit_cell().parameters()[:3]
    else:
      output_abc=[None,None,None]
    for a_box,a_output,n_box,n_output in zip(
        box_abc,
        output_abc,
        output_box.map_box.all(),
        output_unit_cell_grid):
      expected_output_abc.append(a_box*n_output/n_box)
      box_spacing.append(a_box/n_box)
      if output_crystal_symmetry:
        output_spacing.append(a_output/n_output)
      else:
        output_spacing.append(a_box/n_box)

    if output_crystal_symmetry: # make sure it is compatible...
      r0=expected_output_abc[0]/output_abc[0]
      r1=expected_output_abc[1]/output_abc[1]
      r2=expected_output_abc[2]/output_abc[2]
      from libtbx.test_utils import approx_equal
      if not approx_equal(r0,r1,eps=0.001) or not approx_equal(r0,r2,eps=0.001):
        print("WARNING: output_unit_cell and cell_grid will "+\
          "change ratio of grid spacing.\nOld spacings: "+\
         "(%.2f, %.2f, %.2f) A " %(tuple(box_spacing))+\
        "\nNew spacings:  (%.2f, %.2f, %.2f) A \n" %(tuple(output_spacing)), file=log)
    else:
      output_abc=expected_output_abc

    from cctbx import crystal
    output_crystal_symmetry=crystal.symmetry(
          unit_cell=list(output_abc)+[90,90,90], space_group="P1")
    print("Output unit cell will be:  (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f)\n"%(
       tuple(output_crystal_symmetry.unit_cell().parameters())), file=log)

  else:
    output_unit_cell_grid = output_box.map_box.all()
    output_crystal_symmetry=output_box.xray_structure_box.crystal_symmetry()
  # ==========  Done check/set output unit cell grid and cell parameters =====

  if write_output_files:
    # Write PDB file
    if ph_box.overall_counts().n_residues>0:

      if(params.output_file_name_prefix is None):
        file_name = "%s_box.pdb"%output_prefix
      else: file_name = "%s.pdb"%params.output_file_name_prefix
      ph_output_box_output_location.write_pdb_file(file_name=file_name,
          crystal_symmetry = output_crystal_symmetry)
      print("Writing boxed PDB with box unit cell to %s" %(
          file_name), file=log)

    # Write NCS file if NCS
    if output_box.ncs_object and output_box.ncs_object.max_operators()>0:
      if(params.output_file_name_prefix is None):
        output_symmetry_file = "%s_box.ncs_spec"%output_prefix
      else:
        output_symmetry_file = "%s.ncs_spec"%params.output_file_name_prefix
      output_box.ncs_object.format_all_for_group_specification(
          file_name=output_symmetry_file)

      print("\nWriting symmetry to %s" %( output_symmetry_file), file=log)

    # Write ccp4 map.
    if("ccp4" in params.output_format):
     if(params.output_file_name_prefix is None):
       file_name = "%s_box.ccp4"%output_prefix
     else: file_name = "%s.ccp4"%params.output_file_name_prefix
     from iotbx.mrcfile import create_output_labels
     if params.extract_unique:
       program_name='map_box using extract_unique'
       limitations=["extract_unique"]
     else:
       program_name='map_box'
       limitations=[]
     labels=create_output_labels(program_name=program_name,
       input_file_name=inputs.ccp4_map_file_name,
       input_labels=input_map_labels,
       limitations=limitations,
       output_labels=params.output_map_labels)
     output_box.write_ccp4_map(file_name=file_name,
       output_crystal_symmetry=output_crystal_symmetry,
       output_mean=params.output_ccp4_map_mean,
       output_sd=params.output_ccp4_map_sd,
       output_unit_cell_grid=output_unit_cell_grid,
       shift_back=shift_back,
       output_map_labels=labels,
       output_external_origin=params.output_external_origin)
     print("Writing boxed map "+\
          "to CCP4 formatted file:   %s"%file_name, file=log)
     if not params.half_map_list:
        params.half_map_list=[]
     if not output_box.map_box_half_map_list:
       output_box.map_box_half_map_list=[]
     if not half_map_labels_list:
       half_map_labels_list=len(output_box.map_box_half_map_list)*[None]
     for hm,labels,fn in zip(
       output_box.map_box_half_map_list,
       half_map_labels_list,
       params.half_map_list):  # half maps matching
       labels=create_output_labels(program_name=program_name,
         input_file_name=fn,
         input_labels=labels,
         limitations=limitations,
         output_labels=params.output_map_labels)
       hm_fn="%s_box.ccp4" %( ".".join(os.path.basename(fn).split(".")[:-1]))
       output_box.write_ccp4_map(file_name=hm_fn,
         map_data=hm,
         output_crystal_symmetry=output_crystal_symmetry,
         output_mean=params.output_ccp4_map_mean,
         output_sd=params.output_ccp4_map_sd,
         output_unit_cell_grid=output_unit_cell_grid,
         shift_back=shift_back,
         output_map_labels=labels,
         output_external_origin=params.output_external_origin)
       print ("Writing boxed half map to: %s " %(hm_fn),file=log)

    # Write xplor map.  Shift back if keep_origin=True
    if("xplor" in params.output_format):
     if(params.output_file_name_prefix is None):
       file_name = "%s_box.xplor"%output_prefix
     else: file_name = "%s.xplor"%params.output_file_name_prefix
     output_box.write_xplor_map(file_name=file_name,
         output_crystal_symmetry=output_crystal_symmetry,
         output_unit_cell_grid=output_unit_cell_grid,
         shift_back=shift_back,)
     print("Writing boxed map "+\
         "to X-plor formatted file: %s"%file_name, file=log)

    # Write mtz map coeffs.  Shift back if keep_origin=True
    if("mtz" in params.output_format):
     if(params.output_file_name_prefix is None):
       file_name = "%s_box.mtz"%output_prefix
     else: file_name = "%s.mtz"%params.output_file_name_prefix

     print("Writing map coefficients "+\
         "to MTZ file: %s"%file_name, file=log)
     if(map_coeff is not None):
       d_min = map_coeff.d_min()
     elif params.resolution is not None:
       d_min = params.resolution
     else:
       d_min = maptbx.d_min_from_map(map_data=output_box.map_box,
         unit_cell=output_box.xray_structure_box.unit_cell())
     output_box.map_coefficients(d_min=d_min,
       scale_max=params.scale_max,
       resolution_factor=params.resolution_factor, file_name=file_name,
       shift_back=shift_back)

  print(file=log)
  return box

# =============================================================================
# GUI-specific class for running command
from libtbx import runtime_utils
from wxGUI2 import utils

def validate_params(params):
  return True

class launcher(runtime_utils.target_with_save_result):
  def run(self):
    utils.safe_makedirs(self.output_dir)
    os.chdir(self.output_dir)
    result = run(args=self.args, log=sys.stdout)
    return result

# =============================================================================

if (__name__ == "__main__"):
  run(args=sys.argv[1:])


import os
import pandas
import urllib

FREEMONT_URL = 'https://data.seattle.gov/api/views/47yq-6ugv/rows.csv?accessType=DOWNLOAD'

def get_freemont_data(filename = 'Freemont.csv', url = FREEMONT_URL, force_download = False):
    if force_download or not os.path.exists(filename):
        urllib.request.urlretrieve(url, filename)

    data = pandas.read_csv(filename, index_col = 'Date')

    try:
        data.index = pandas.to_datetime(data.index, format='%m/%d/%Y %I:%M:%S %p')
    except TypeError:
        data.index = pandas.to_datetime(data.index)

    data.columns = ['Total', 'East', 'West']

    return(data)


import argparse
import collections
import imblearn
import joblib
import json
import numpy as np
import pandas as pd
import pickle
import skrebate
import sklearn
import sys
import xgboost
import warnings
from imblearn import under_sampling, over_sampling, combine
from scipy.io import mmread
from mlxtend import classifier, regressor
from sklearn.base import clone
from sklearn import (cluster, compose, decomposition, ensemble,
                     feature_extraction, feature_selection,
                     gaussian_process, kernel_approximation, metrics,
                     model_selection, naive_bayes, neighbors,
                     pipeline, preprocessing, svm, linear_model,
                     tree, discriminant_analysis)
from sklearn.exceptions import FitFailedWarning
from sklearn.model_selection._validation import _score, cross_validate
from sklearn.model_selection import _search, _validation

from galaxy_ml.utils import (SafeEval, get_cv, get_scoring, load_model,
                             read_columns, try_get_attr, get_module)

_fit_and_score = try_get_attr('galaxy_ml.model_validations', '_fit_and_score')
setattr(_search, '_fit_and_score', _fit_and_score)
setattr(_validation, '_fit_and_score', _fit_and_score)

N_JOBS = int(__import__('os').environ.get('GALAXY_SLOTS', 1))
CACHE_DIR = './cached'
NON_SEARCHABLE = ('n_jobs', 'pre_dispatch', 'memory', '_path',
                  'nthread', 'callbacks')
ALLOWED_CALLBACKS = ('EarlyStopping', 'TerminateOnNaN', 'ReduceLROnPlateau',
                     'CSVLogger', 'None')

def _eval_search_params(params_builder):
    search_params = {}

    for p in params_builder['param_set']:
        search_list = p['sp_list'].strip()
        if search_list == '':
            continue

        param_name = p['sp_name']
        if param_name.lower().endswith(NON_SEARCHABLE):
            print("Warning: `%s` is not eligible for search and was "
                  "omitted!" % param_name)
            continue

        if not search_list.startswith(':'):
            safe_eval = SafeEval(load_scipy=True, load_numpy=True)
            ev = safe_eval(search_list)
            search_params[param_name] = ev
        else:
            # Have `:` before search list, asks for estimator evaluatio
            safe_eval_es = SafeEval(load_estimators=True)
            search_list = search_list[1:].strip()
            # TODO maybe add regular express check
            ev = safe_eval_es(search_list)
            preprocessings = (
                preprocessing.StandardScaler(), preprocessing.Binarizer(),
                preprocessing.MaxAbsScaler(),
                preprocessing.Normalizer(), preprocessing.MinMaxScaler(),
                preprocessing.PolynomialFeatures(),
                preprocessing.RobustScaler(), feature_selection.SelectKBest(),
                feature_selection.GenericUnivariateSelect(),
                feature_selection.SelectPercentile(),
                feature_selection.SelectFpr(), feature_selection.SelectFdr(),
                feature_selection.SelectFwe(),
                feature_selection.VarianceThreshold(),
                decomposition.FactorAnalysis(random_state=0),
                decomposition.FastICA(random_state=0),
                decomposition.IncrementalPCA(),
                decomposition.KernelPCA(random_state=0, n_jobs=N_JOBS),
                decomposition.LatentDirichletAllocation(
                    random_state=0, n_jobs=N_JOBS),
                decomposition.MiniBatchDictionaryLearning(
                    random_state=0, n_jobs=N_JOBS),
                decomposition.MiniBatchSparsePCA(
                    random_state=0, n_jobs=N_JOBS),
                decomposition.NMF(random_state=0),
                decomposition.PCA(random_state=0),
                decomposition.SparsePCA(random_state=0, n_jobs=N_JOBS),
                decomposition.TruncatedSVD(random_state=0),
                kernel_approximation.Nystroem(random_state=0),
                kernel_approximation.RBFSampler(random_state=0),
                kernel_approximation.AdditiveChi2Sampler(),
                kernel_approximation.SkewedChi2Sampler(random_state=0),
                cluster.FeatureAgglomeration(),
                skrebate.ReliefF(n_jobs=N_JOBS),
                skrebate.SURF(n_jobs=N_JOBS),
                skrebate.SURFstar(n_jobs=N_JOBS),
                skrebate.MultiSURF(n_jobs=N_JOBS),
                skrebate.MultiSURFstar(n_jobs=N_JOBS),
                imblearn.under_sampling.ClusterCentroids(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.CondensedNearestNeighbour(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.EditedNearestNeighbours(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.RepeatedEditedNearestNeighbours(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.AllKNN(random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.InstanceHardnessThreshold(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.NearMiss(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.NeighbourhoodCleaningRule(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.OneSidedSelection(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.RandomUnderSampler(
                    random_state=0),
                imblearn.under_sampling.TomekLinks(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.ADASYN(random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.RandomOverSampler(random_state=0),
                imblearn.over_sampling.SMOTE(random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.SVMSMOTE(random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.BorderlineSMOTE(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.SMOTENC(
                    categorical_features=[], random_state=0, n_jobs=N_JOBS),
                imblearn.combine.SMOTEENN(random_state=0),
                imblearn.combine.SMOTETomek(random_state=0))
            newlist = []
            for obj in ev:
                if obj is None:
                    newlist.append(None)
                elif obj == 'all_0':
                    newlist.extend(preprocessings[0:35])
                elif obj == 'sk_prep_all':      # no KernalCenter()
                    newlist.extend(preprocessings[0:7])
                elif obj == 'fs_all':
                    newlist.extend(preprocessings[7:14])
                elif obj == 'decomp_all':
                    newlist.extend(preprocessings[14:25])
                elif obj == 'k_appr_all':
                    newlist.extend(preprocessings[25:29])
                elif obj == 'reb_all':
                    newlist.extend(preprocessings[30:35])
                elif obj == 'imb_all':
                    newlist.extend(preprocessings[35:54])
                elif type(obj) is int and -1 < obj < len(preprocessings):
                    newlist.append(preprocessings[obj])
                elif hasattr(obj, 'get_params'):       # user uploaded object
                    if 'n_jobs' in obj.get_params():
                        newlist.append(obj.set_params(n_jobs=N_JOBS))
                    else:
                        newlist.append(obj)
                else:
                    sys.exit("Unsupported estimator type: %r" % (obj))

            search_params[param_name] = newlist

    return search_params

def main(inputs, infile_estimator, infile1, infile2,
         outfile_result, outfile_object=None,
         outfile_weights=None, groups=None,
         ref_seq=None, intervals=None, targets=None,
         fasta_path=None):
    """
    Parameter
    ---------
    inputs : str
        File path to galaxy tool parameter

    infile_estimator : str
        File path to estimator

    infile1 : str
        File path to dataset containing features

    infile2 : str
        File path to dataset containing target values

    outfile_result : str
        File path to save the results, either cv_results or test result

    outfile_object : str, optional
        File path to save searchCV object

    outfile_weights : str, optional
        File path to save model weights

    groups : str
        File path to dataset containing groups labels

    ref_seq : str
        File path to dataset containing genome sequence file

    intervals : str
        File path to dataset containing interval file

    targets : str
        File path to dataset compressed target bed file

    fasta_path : str
        File path to dataset containing fasta file
    """
    warnings.simplefilter('ignore')

    with open(inputs, 'r') as param_handler:
        params = json.load(param_handler)

    # conflict param checker
    if params['outer_split']['split_mode'] == 'nested_cv' \
            and params['save'] != 'nope':
        raise ValueError("Save best estimator is not possible for nested CV!")

    if not (params['search_schemes']['options']['refit']) \
            and params['save'] != 'nope':
        raise ValueError("Save best estimator is not possible when refit "
                         "is False!")

    params_builder = params['search_schemes']['search_params_builder']

    with open(infile_estimator, 'rb') as estimator_handler:
        estimator = load_model(estimator_handler)
    estimator_params = estimator.get_params()

    # store read dataframe object
    loaded_df = {}

    input_type = params['input_options']['selected_input']
    # tabular input
    if input_type == 'tabular':
        header = 'infer' if params['input_options']['header1'] else None
        column_option = (params['input_options']['column_selector_options_1']
                         ['selected_column_selector_option'])
        if column_option in ['by_index_number', 'all_but_by_index_number',
                             'by_header_name', 'all_but_by_header_name']:
            c = params['input_options']['column_selector_options_1']['col1']
        else:
            c = None

        df_key = infile1 + repr(header)
        df = pd.read_csv(infile1, sep='\t', header=header,
                         parse_dates=True)
        loaded_df[df_key] = df

        X = read_columns(df, c=c, c_option=column_option).astype(float)
    # sparse input
    elif input_type == 'sparse':
        X = mmread(open(infile1, 'r'))

    # fasta_file input
    elif input_type == 'seq_fasta':
        pyfaidx = get_module('pyfaidx')
        sequences = pyfaidx.Fasta(fasta_path)
        n_seqs = len(sequences.keys())
        X = np.arange(n_seqs)[:, np.newaxis]
        for param in estimator_params.keys():
            if param.endswith('fasta_path'):
                estimator.set_params(
                    **{param: fasta_path})
                break
        else:
            raise ValueError(
                "The selected estimator doesn't support "
                "fasta file input! Please consider using "
                "KerasGBatchClassifier with "
                "FastaDNABatchGenerator/FastaProteinBatchGenerator "
                "or having GenomeOneHotEncoder/ProteinOneHotEncoder "
                "in pipeline!")

    elif input_type == 'refseq_and_interval':
        path_params = {
            'data_batch_generator__ref_genome_path': ref_seq,
            'data_batch_generator__intervals_path': intervals,
            'data_batch_generator__target_path': targets
        }
        estimator.set_params(**path_params)
        n_intervals = sum(1 for line in open(intervals))
        X = np.arange(n_intervals)[:, np.newaxis]

    # Get target y
    header = 'infer' if params['input_options']['header2'] else None
    column_option = (params['input_options']['column_selector_options_2']
                     ['selected_column_selector_option2'])
    if column_option in ['by_index_number', 'all_but_by_index_number',
                         'by_header_name', 'all_but_by_header_name']:
        c = params['input_options']['column_selector_options_2']['col2']
    else:
        c = None

    df_key = infile2 + repr(header)
    if df_key in loaded_df:
        infile2 = loaded_df[df_key]
    else:
        infile2 = pd.read_csv(infile2, sep='\t',
                              header=header, parse_dates=True)
        loaded_df[df_key] = infile2

    y = read_columns(
            infile2,
            c=c,
            c_option=column_option,
            sep='\t',
            header=header,
            parse_dates=True)
    if len(y.shape) == 2 and y.shape[1] == 1:
        y = y.ravel()
    if input_type == 'refseq_and_interval':
        estimator.set_params(
            data_batch_generator__features=y.ravel().tolist())
        y = None
    # end y

    optimizer = params['search_schemes']['selected_search_scheme']
    optimizer = getattr(model_selection, optimizer)

    # handle gridsearchcv options
    options = params['search_schemes']['options']

    if groups:
        header = 'infer' if (options['cv_selector']['groups_selector']
                                    ['header_g']) else None
        column_option = (options['cv_selector']['groups_selector']
                                ['column_selector_options_g']
                                ['selected_column_selector_option_g'])
        if column_option in ['by_index_number', 'all_but_by_index_number',
                             'by_header_name', 'all_but_by_header_name']:
            c = (options['cv_selector']['groups_selector']
                        ['column_selector_options_g']['col_g'])
        else:
            c = None

        df_key = groups + repr(header)
        if df_key in loaded_df:
            groups = loaded_df[df_key]

        groups = read_columns(
                groups,
                c=c,
                c_option=column_option,
                sep='\t',
                header=header,
                parse_dates=True)
        groups = groups.ravel()
        options['cv_selector']['groups_selector'] = groups

    splitter, groups = get_cv(options.pop('cv_selector'))
    options['cv'] = splitter
    options['n_jobs'] = N_JOBS
    primary_scoring = options['scoring']['primary_scoring']
    options['scoring'] = get_scoring(options['scoring'])
    if options['error_score']:
        options['error_score'] = 'raise'
    else:
        options['error_score'] = np.NaN
    if options['refit'] and isinstance(options['scoring'], dict):
        options['refit'] = primary_scoring
    if 'pre_dispatch' in options and options['pre_dispatch'] == '':
        options['pre_dispatch'] = None

    # del loaded_df
    del loaded_df

    # handle memory
    memory = joblib.Memory(location=CACHE_DIR, verbose=0)
    # cache iraps_core fits could increase search speed significantly
    if estimator.__class__.__name__ == 'IRAPSClassifier':
        estimator.set_params(memory=memory)
    else:
        # For iraps buried in pipeline
        for p, v in estimator_params.items():
            if p.endswith('memory'):
                # for case of `__irapsclassifier__memory`
                if len(p) > 8 and p[:-8].endswith('irapsclassifier'):
                    # cache iraps_core fits could increase search
                    # speed significantly
                    new_params = {p: memory}
                    estimator.set_params(**new_params)
                # security reason, we don't want memory being
                # modified unexpectedly
                elif v:
                    new_params = {p, None}
                    estimator.set_params(**new_params)
            # For now, 1 CPU is suggested for iprasclassifier
            elif p.endswith('n_jobs'):
                new_params = {p: 1}
                estimator.set_params(**new_params)
            # for security reason, types of callbacks are limited
            elif p.endswith('callbacks'):
                for cb in v:
                    cb_type = cb['callback_selection']['callback_type']
                    if cb_type not in ALLOWED_CALLBACKS:
                        raise ValueError(
                            "Prohibited callback type: %s!" % cb_type)

    param_grid = _eval_search_params(params_builder)
    searcher = optimizer(estimator, param_grid, **options)

    # do nested split
    split_mode = params['outer_split'].pop('split_mode')
    # nested CV, outer cv using cross_validate
    if split_mode == 'nested_cv':
        outer_cv, _ = get_cv(params['outer_split']['cv_selector'])

        if options['error_score'] == 'raise':
            rval = cross_validate(
                searcher, X, y, scoring=options['scoring'],
                cv=outer_cv, n_jobs=N_JOBS, verbose=0,
                error_score=options['error_score'])
        else:
            warnings.simplefilter('always', FitFailedWarning)
            with warnings.catch_warnings(record=True) as w:
                try:
                    rval = cross_validate(
                        searcher, X, y,
                        scoring=options['scoring'],
                        cv=outer_cv, n_jobs=N_JOBS,
                        verbose=0,
                        error_score=options['error_score'])
                except ValueError:
                    pass
                for warning in w:
                    print(repr(warning.message))

        keys = list(rval.keys())
        for k in keys:
            if k.startswith('test'):
                rval['mean_' + k] = np.mean(rval[k])
                rval['std_' + k] = np.std(rval[k])
            if k.endswith('time'):
                rval.pop(k)
        rval = pd.DataFrame(rval)
        rval = rval[sorted(rval.columns)]
        rval.to_csv(path_or_buf=outfile_result, sep='\t',
                    header=True, index=False)
    else:
        if split_mode == 'train_test_split':
            train_test_split = try_get_attr(
                'galaxy_ml.model_validations', 'train_test_split')
            # make sure refit is choosen
            # this could be True for sklearn models, but not the case for
            # deep learning models
            if not options['refit'] and \
                    not all(hasattr(estimator, attr)
                            for attr in ('config', 'model_type')):
                warnings.warn("Refit is change to `True` for nested "
                              "validation!")
                setattr(searcher, 'refit', True)
            split_options = params['outer_split']

            # splits
            if split_options['shuffle'] == 'stratified':
                split_options['labels'] = y
                X, X_test, y, y_test = train_test_split(X, y, **split_options)
            elif split_options['shuffle'] == 'group':
                if groups is None:
                    raise ValueError("No group based CV option was "
                                     "choosen for group shuffle!")
                split_options['labels'] = groups
                if y is None:
                    X, X_test, groups, _ =\
                        train_test_split(X, groups, **split_options)
                else:
                    X, X_test, y, y_test, groups, _ =\
                        train_test_split(X, y, groups, **split_options)
            else:
                if split_options['shuffle'] == 'None':
                    split_options['shuffle'] = None
                X, X_test, y, y_test =\
                    train_test_split(X, y, **split_options)
        # end train_test_split

        # shared by both train_test_split and non-split
        if options['error_score'] == 'raise':
            searcher.fit(X, y, groups=groups)
        else:
            warnings.simplefilter('always', FitFailedWarning)
            with warnings.catch_warnings(record=True) as w:
                try:
                    searcher.fit(X, y, groups=groups)
                except ValueError:
                    pass
                for warning in w:
                    print(repr(warning.message))

        # no outer split
        if split_mode == 'no':
            # save results
            cv_results = pd.DataFrame(searcher.cv_results_)
            cv_results = cv_results[sorted(cv_results.columns)]
            cv_results.to_csv(path_or_buf=outfile_result, sep='\t',
                              header=True, index=False)

        # train_test_split, output test result using best_estimator_
        # or rebuild the trained estimator using weights if applicable.
        else:
            scorer_ = searcher.scorer_
            if isinstance(scorer_, collections.Mapping):
                is_multimetric = True
            else:
                is_multimetric = False

            best_estimator_ = getattr(searcher, 'best_estimator_', None)
            if not best_estimator_:
                raise ValueError("GridSearchCV object has no "
                                 "`best_estimator_` when `refit`=False!")

            if best_estimator_.__class__.__name__ == 'KerasGBatchClassifier' \
                    and hasattr(estimator.data_batch_generator, 'target_path'):
                test_score = best_estimator_.evaluate(
                    X_test, scorer=scorer_, is_multimetric=is_multimetric)
            else:
                test_score = _score(best_estimator_, X_test,
                                    y_test, scorer_,
                                    is_multimetric=is_multimetric)

            if not is_multimetric:
                test_score = {primary_scoring: test_score}
            for key, value in test_score.items():
                test_score[key] = [value]
            result_df = pd.DataFrame(test_score)
            result_df.to_csv(path_or_buf=outfile_result, sep='\t',
                             header=True, index=False)

    memory.clear(warn=False)

    if outfile_object:
        best_estimator_ = getattr(searcher, 'best_estimator_', None)
        if not best_estimator_:
            warnings.warn("GridSearchCV object has no attribute "
                          "'best_estimator_', because either it's "
                          "nested gridsearch or `refit` is False!")
            return

        main_est = best_estimator_
        if isinstance(best_estimator_, pipeline.Pipeline):
            main_est = best_estimator_.steps[-1][-1]

        if hasattr(main_est, 'model_') \
                and hasattr(main_est, 'save_weights'):
            if outfile_weights:
                main_est.save_weights(outfile_weights)
            del main_est.model_
            del main_est.fit_params
            del main_est.model_class_
            del main_est.validation_data
            if getattr(main_est, 'data_generator_', None):
                del main_est.data_generator_

        with open(outfile_object, 'wb') as output_handler:
            pickle.dump(best_estimator_, output_handler,
                        pickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':
    aparser = argparse.ArgumentParser()
    aparser.add_argument("-i", "--inputs", dest="inputs", required=True)
    aparser.add_argument("-e", "--estimator", dest="infile_estimator")
    aparser.add_argument("-X", "--infile1", dest="infile1")
    aparser.add_argument("-y", "--infile2", dest="infile2")
    aparser.add_argument("-O", "--outfile_result", dest="outfile_result")
    aparser.add_argument("-o", "--outfile_object", dest="outfile_object")
    aparser.add_argument("-w", "--outfile_weights", dest="outfile_weights")
    aparser.add_argument("-g", "--groups", dest="groups")
    aparser.add_argument("-r", "--ref_seq", dest="ref_seq")
    aparser.add_argument("-b", "--intervals", dest="intervals")
    aparser.add_argument("-t", "--targets", dest="targets")
    aparser.add_argument("-f", "--fasta_path", dest="fasta_path")
    args = aparser.parse_args()

    main(args.inputs, args.infile_estimator, args.infile1, args.infile2,
         args.outfile_result, outfile_object=args.outfile_object,
         outfile_weights=args.outfile_weights, groups=args.groups,
         ref_seq=args.ref_seq, intervals=args.intervals,
         targets=args.targets, fasta_path=args.fasta_path)