# =============================================================================
# COPYRIGHT 2013-14 Brain Corporation.
# License under MIT license (see LICENSE file)
# =============================================================================

import argparse
import collections
import distutils.core
import fnmatch
import glob
import importlib
import logging
import multiprocessing
import os
import platform
import shutil
import subprocess
import sys
import tempfile
from detail import Requirement, RequirementException, read_requirement_file
from detail.requirement import remove_duplicate_requirements, expand_requirements_specifiers, generate_dependency_list
from detail.utility import ln, run_shell, download, safe_remove, unpack, get_single_char
import urllib2
# for doctests
import detail
import re


__version__ = '0.0.2'


class RobustusException(Exception):
    def __init__(self, message):
        Exception.__init__(self, message)


class Robustus(object):
    settings_file_path = '.robustus'
    cached_requirements_file_path = 'cached_requirements.txt'
    default_settings = {
        'cache': 'wheelhouse'
    }
    # FIXME: not so great to hardcode braincorp address here, but in other way
    # we need to modify other repositories use_repo.sh which use robustus
    default_package_locations = ['http://thirdparty-packages.braincorporation.net']

    def __init__(self, args):
        """
        Initialize robustus tool. Should be called if sys.executable is in robustus environment
        @param: args - command line arguments
        """
        if args.env is not None:
            self.env = os.path.abspath(args.env)
        else:
            self.env = os.path.abspath(os.path.join(sys.executable, os.pardir, os.pardir))

        # check if we are in robustus environment
        self.settings_file_path = os.path.join(self.env, Robustus.settings_file_path)
        if not os.path.isfile(self.settings_file_path):
            raise RobustusException('bad robustus environment ' + self.env + ': .robustus settings file not found')
        settings = eval(open(self.settings_file_path).read())
        self.settings = Robustus._override_settings(settings, args)
        if 'find_links' not in self.settings:
            self.settings['find_links'] = self.default_package_locations
        logging.info('Robustus will use the following cache folder: %s' % self.settings['cache'])

        self.python_executable = os.path.join(self.env, 'bin/python')
        if not os.path.isfile(self.python_executable):
            self.python_executable = os.path.join(self.env, 'bin/python27')
        if not os.path.isfile(self.python_executable):
            raise RobustusException('bad robustus environment ' + self.env + ': python not found')

        self.pip_executable = os.path.join(self.env, 'bin/pip')
        if not os.path.isfile(self.pip_executable):
            raise RobustusException('bad robustus environment ' + self.env + ': pip not found')

        self.easy_install_executable = os.path.join(self.env, 'bin/easy_install')
        if not os.path.isfile(self.easy_install_executable):
            raise RobustusException('bad robustus environment ' + self.env + ': easy_install not found')

        # make cached packages directory if necessary
        self.cache = os.path.join(self.env, self.settings['cache'])
        if not os.path.isdir(self.cache):
            os.mkdir(self.cache)

        # remove bad formatted rob files with '.' in version instead of '_'
        for rob_file in glob.iglob('%s/*.rob' % self.cache):
            rob_basename = os.path.basename(rob_file)
            if rob_basename.find('__') != -1:
                name, version = rob_basename[:-4].split('__')
                if '.' in version:
                    corrected_filename = os.path.join(self.cache,
                                                      '%s__%s.rob' % (name, version.replace('.', '_')))
                    logging.info('Corrected rob file version from %s to %s' % (rob_file, corrected_filename))
                    shutil.copy(rob_file, corrected_filename)
                    os.remove(rob_file)

        # read cached packages
        self.cached_packages = []
        for rob_file in glob.iglob('%s/*.rob' % self.cache):
            self.cached_packages.append(Requirement(rob_filename=rob_file))

    @staticmethod
    def _override_settings(settings, args):
        # override settings with command line arguments
        if args.cache is not None:
            settings['cache'] = args.cache
        settings['verbosity'] = args.verbosity
        settings['debug'] = args.debug

        # Set logging volume for debugging
        if settings['debug']:
            logging.getLogger('').setLevel(logging.DEBUG)
        return settings

    @staticmethod
    def env(args):
        """
        Create robustus environment.
        @param args: command line arguments
        """
        settings = dict()
        settings = Robustus._override_settings(settings, args)

        # create virtualenv
        python_executable = os.path.abspath(os.path.join(args.env, 'bin/python'))
        if os.path.isfile(python_executable):
            logging.info('Found virtualenv in ' + args.env)
        else:
            logging.info('Creating virtualenv')
            virtualenv_args = ['virtualenv', args.env, '--prompt', args.prompt]
            if args.python is not None:
                virtualenv_args += ['--python', args.python]
            if args.system_site_packages:
                virtualenv_args += ['--system-site-packages']
            run_shell(virtualenv_args, settings['verbosity'] >= 1)

        pip_executable = os.path.abspath(os.path.join(args.env, 'bin/pip'))
        if not os.path.isfile(pip_executable):
            raise RobustusException('failed to create virtualenv, pip not found')
        easy_install_executable = os.path.abspath(os.path.join(args.env, 'bin/easy_install'))
        if not os.path.isfile(easy_install_executable):
            raise RobustusException('failed to create virtualenv, easy_install not found')

        # http://wheel.readthedocs.org/en/latest/
        # wheel is binary packager for python/pip
        # we store all packages in binary wheel somewhere on the PC to avoid recompilation of packages

        # wheel needs pip>=1.4, setuptools>=0.8 and wheel packages for wheeling
        run_shell([pip_executable, 'install', 'pip==6.1.1', '--upgrade'], settings['verbosity'] >= 1)
        run_shell([pip_executable, 'install', 'wheel==0.24.0', '--upgrade'], settings['verbosity'] >= 1)
        # some sloppy maintained packages (like ROS) require outdated distribute for installation
        # and we need to install it before setuptools. For those there used to be the following lines here:
        # run_shell([pip_executable, 'install', 'distribute==0.7.3'], settings['verbosity'] >= 1)
        # run_shell([pip_executable, 'install', 'setuptools==1.1.6', '--upgrade'], settings['verbosity'] >= 1)
        # that are gone now, but please consider uncommenting those if new problems with ROS appear.
        # Currently we upgrade setuptools to the moder version:
        run_shell([pip_executable, 'install', 'setuptools==15.2', '--upgrade'], settings['verbosity'] >= 1)

        # linking BLAS and LAPACK libraries
        if os.path.isfile('/usr/lib64/libblas.so.3'):
            logging.info('Linking CentOS libblas to venv')
            blas_so = os.path.join(args.env, 'lib64/libblas.so')
            ln('/usr/lib64/libblas.so.3', blas_so, True)
            os.environ['BLAS'] = os.path.join(args.env, 'lib64')
        elif os.path.isfile('/usr/lib/libblas.so'):
            logging.info('Linking Ubuntu libblas to venv')
            blas_so = os.path.join(args.env, 'lib/libblas.so')
            ln('/usr/lib/libblas.so', blas_so, True)
            os.environ['BLAS'] = os.path.join(args.env, 'lib')

        if os.path.isfile('/usr/lib64/liblapack.so.3'):
            logging.info('Linking CentOS liblapack to venv')
            lapack_so = os.path.join(args.env, 'lib64/liblapack.so')
            ln('/usr/lib64/liblapack.so.3', lapack_so, True)
            os.environ['LAPACK'] = os.path.join(args.env, 'lib64')
        elif os.path.isfile('/usr/lib/liblapack.so'):
            logging.info('Linking Ubuntu liblapack to venv')
            lapack_so = os.path.join(args.env, 'lib/liblapack.so')
            ln('/usr/lib/liblapack.so', lapack_so, True)
            os.environ['LAPACK'] = os.path.join(args.env, 'lib')

        if sys.platform.startswith('darwin'):
            # on Mac, install readline before everything else
            # Ubuntu already has readline in the system python library
            run_shell([easy_install_executable, '-q', 'readline==6.2.2'], settings['verbosity'] >= 1)

        # compose settings file
        logging.info('Write .robustus config file')
        settings = Robustus._override_settings(Robustus.default_settings, args)
        with open(os.path.join(args.env, Robustus.settings_file_path), 'w') as file:
            file.write(str(settings))

        # Install Robustus in the Python virtual environment if its "setup.py" is available.
        # If Robustus has already been installed in the virtual environment, running "setup.py"
        # should be harmless.  This is required to pass the "test_robustus" test.
        cwd = os.getcwd()
        script_dir = os.path.dirname(os.path.realpath(__file__))
        setup_dir = os.path.abspath(os.path.join(script_dir, os.path.pardir))
        logging.info('python_executable = %s' % python_executable)
        logging.info('script_dir = %s' % script_dir)
        logging.info('setup_dir = %s' % setup_dir)
        os.chdir(setup_dir)
        if os.path.exists('setup.py'):
            run_shell([python_executable, 'setup.py', 'install'], settings['verbosity'] >= 1)
        else:
            logging.warn('Cannot find setup.py in %s.  Continuing...' % setup_dir)
        os.chdir(cwd)

        logging.info('Robustus initialized environment with cache located at %s' % settings['cache'])

    def install_satisfactory_requirement_from_remote(self, requirement_specifier):
        """
        If wheel for satisfactory requirement found on remote, install it.
        :param requirement_specifier: specifies package namd and package version string
        :return: True if package installed by function (according to pip return code);
        False otherwise.
        """
        logging.info('Attempting to install package from remote wheel')
        for find_link in self.settings['find_links']:
            find_links_url = find_link + '/python-wheels/index.html',  # TEMPORARY.
            dtemp_path = tempfile.mkdtemp()
            return_code = run_shell([self.pip_executable,
                                     'install',
                                     '--download-cache=%s' % dtemp_path,
                                     '--no-index',
                                     '--use-wheel',
                                     '--find-links=%s' % find_links_url,
                                     '--trusted-host=%s' % find_link.split("http://")[1],
                                     requirement_specifier.freeze()],
                                    verbose=self.settings['verbosity'] >= 2)
            if return_code == 0:
                # The following downloads the wheels of the requirment (and those of
                # dependencies) into a pip download cache and moves (renames) the downloaded
                # wheels into the local Robustus cache.  Regarding the need for this see "Wheels
                # for Dependencies" "http://lucumr.pocoo.org/2014/1/27/python-on-wheels/".
                for file_path in glob.glob(os.path.join(dtemp_path, 'http*.whl')):
                    if os.path.isfile(file_path):
                        file_name = os.path.basename(file_path)
                        file_name_new = file_name.rpartition('%2F')[-1]
                        file_path_new = os.path.join(self.cache, file_name_new)
                        shutil.move(file_path, file_path_new)  # NOTE: Allow overwrites.
                safe_remove(dtemp_path)
                return True
            else:
                logging.info('pip failed to install requirement %s from remote wheels cache %s.'
                             % (requirement_specifier.freeze(), find_links_url))
                safe_remove(dtemp_path)

        return False

    def install_through_wheeling(self, requirement_specifier, rob_file, ignore_index):
        """
        Check if package cache already contains package of specified version, if so install it.
        Otherwise make a wheel and put it into cache.
        Hope manual check for requirements file won't be necessary, waiting for pip 1.5 https://github.com/pypa/pip/issues/855
        :param package: package name
        :param version: package version string
        :return: None
        """
        # If wheelhouse doesn't contain necessary requirement attempt to install from remote wheel archive or make a wheel.
        installed = False
        if self.find_satisfactory_requirement(requirement_specifier) is None:
            # Pip does not download the wheels of dependencies unless it installs.
            installed = False
            if not self.settings['no_remote_cache']:
                installed = self.install_satisfactory_requirement_from_remote(requirement_specifier)
            if not installed:
                logging.info('Wheel not found, downloading package')
                cmd = [self.pip_executable, 'install']
                if len(self.settings['allow_external']) > 0:
                    cmd += ['--allow-external'] + self.settings['allow_external']
                if self.settings['allow_all_external']:
                    cmd.append('--allow-all-external')
                if len(self.settings['allow_unverified']) > 0:
                    cmd += ['--allow-unverified'] + self.settings['allow_unverified']
                cmd += ['--download', self.cache, requirement_specifier.freeze()]
                return_code = run_shell(cmd, verbose=self.settings['verbosity'] >= 2)
                if return_code != 0:
                    raise RequirementException('pip failed to download requirement %s' % requirement_specifier.freeze())
                logging.info('Done')
    
                logging.info('Building wheel')
                wheel_cmd = [self.pip_executable,
                             'wheel',
                             '--no-index',
                             '--find-links=%s' % self.cache,
                             '--wheel-dir=%s' % self.cache,
                             requirement_specifier.freeze()]
                # we probably sometimes will want to see build log
                for i in xrange(self.settings['verbosity']):
                    wheel_cmd.append('-v')
                return_code = run_shell(wheel_cmd, verbose=self.settings['verbosity'] >= 1)
                if return_code != 0:
                    raise RequirementException('pip failed to build wheel for requirement %s'
                                               % requirement_specifier.freeze())
                logging.info('Done')
    
        if not installed:
            # install from new prebuilt wheel
            logging.info('Installing package from wheel')
            return_code = run_shell([self.pip_executable,
                                     'install',
                                     '--no-index',
                                     '--use-wheel',
                                     '--find-links=%s' % self.cache,
                                     requirement_specifier.freeze()],
                                    verbose=self.settings['verbosity'] >= 2)
            if return_code != 0:
                raise RequirementException('pip failed to install requirement %s from wheels cache %s.'
                                           % (requirement_specifier.freeze(), self.cache))

    def _pip_install_requirement(self, requirement_specifier):
        command = ' '.join([self.pip_executable, 'install', requirement_specifier.freeze()])
        logging.info('Got url-based requirement. '
                     'Fall back to pip shell command:%s' % (command,))
        ret_code = run_shell(command, shell=True, verbose=self.settings['verbosity'] >= 1)
        return ret_code

    def install_requirement(self, requirement_specifier, ignore_index, tag):
        attempts = self.settings['attempts']
        logging.info('='*30)  # Nicely separate installation of different packages in console output
        for a in range(attempts):
            result = self._install_requirement_attempt(requirement_specifier, ignore_index, tag, a)
            if result:
                return

    def _install_requirement_attempt(self, requirement_specifier, ignore_index, tag, attempt_number):
        if attempt_number == 0:
            logging.info('Installing %s' % (requirement_specifier.freeze(),))
        else:
            logging.info('Installing %s (attempt %d)' % (requirement_specifier.freeze(), attempt_number + 1))

        if tag:
            logging.info('with tag %s, ignore_missing_refs %s' % (tag,
                                                                  self.settings['ignore_missing_refs']))

        if requirement_specifier.url is not None or requirement_specifier.path is not None:
            # install reqularly using pip
            if not self.settings['update_editables'] and \
                    requirement_specifier.url is not None and \
                    requirement_specifier.editable:
                editable_requirement_path = os.path.join(self.env, 'src', requirement_specifier.name)
                logging.info('Got url-based requirement. Checking if exists %s ' % (editable_requirement_path,))
                if tag is None and os.path.exists(editable_requirement_path):
                    logging.info('For safety reasons robustus will not proceed with requirement %s, '
                                 'because directories for installing this package already exists (%s). '
                                 'To update editable dependency, please remove folder and run again.' %
                                 (requirement_specifier.freeze(),
                                  os.path.join(self.env, 'src', requirement_specifier.name)))
                    return False

            # here we have to run via shell because requirement can be editable and then it will require
            # extra parsing to extract -e flag into separate argument.
            if tag and requirement_specifier.editable and requirement_specifier.path is None:
                logging.info('Overriding editable branch with tag %s' % tag)
                original_url = requirement_specifier.url
                requirement_specifier.override_branch(tag)

            ret_code = self._pip_install_requirement(requirement_specifier)
            
            # special case for path-based requirements - we need to call 'git checkout'
            if ret_code == 0 and tag and requirement_specifier.editable and requirement_specifier.path is not None:
                cwd = os.getcwd()
                os.chdir(requirement_specifier.path)
                logging.info('Checking out editable branch in directory "%s" with tag %s' % (requirement_specifier.path, tag))

                # Check if branch exists locally - should never be the case on Travis.
                if not subprocess.check_output('git branch --list %s' % tag, shell=True):
                    '''
                    If the branch doesn't exist locally, we fetch it.
                    Note: it may seem more natural to do:
                        
                        git checkout -b {branch} origin/{branch}
                    
                    but that produces a git error:
                    
                        fatal: git checkout: updating paths is incompatible with switching branches.
                        Did you intend to checkout 'origin/{branch}' which can not be resolved as commit?
                    
                    See http://stackoverflow.com/questions/945654/git-checkout-on-a-remote-branch-does-not-work
                    '''
                    os.system('git fetch origin {0}:{0}'.format(tag))

                local_checkout_code = os.system('git checkout {0}'.format(tag))
                os.chdir(cwd)
                if local_checkout_code!=0:
                    if self.settings['ignore_missing_refs']:
                        logging.info('Tag or branch doesnt exist for this package, using default')
                    else:
                        return False
                
            if ret_code != 0 and tag and self.settings['ignore_missing_refs']:
                logging.info('Tag or branch doesnt exist for this package, using default')
                requirement_specifier.url = original_url
                ret_code = self._pip_install_requirement(requirement_specifier)

            if ret_code != 0:
                return False  # do not print done, do not add package to the list of cached packages
        else:
            rob = os.path.join(self.cache, requirement_specifier.rob_filename())
            if os.path.isfile(rob):
                # package cached
                # open for reading so install script can read required information
                rob_file = open(rob, 'r')
            else:
                # package not cached
                # open for writing so install script can save required information
                rob_file = open(rob, 'w')

            try:
                try:
                    # try to use specific install script
                    install_module = importlib.import_module('robustus.detail.install_%s' % requirement_specifier.name.lower())
                    install_module.install(self, requirement_specifier, rob_file, ignore_index)
                except ImportError:
                    self.install_through_wheeling(requirement_specifier, rob_file, ignore_index)                
            except Exception as exc:
                logging.warn('Exception during installation: %s' % str(exc))
                rob_file.close()
                logging.warn('Robustus will delete the corresponding %s file in order '
                             'to recreate the wheel in the future. Please run again.' % str(rob))
                
                # remove specifier from cached packages
                self.cached_packages = [r for r in self.cached_packages
                                        if r.freeze() != requirement_specifier.freeze()]
                os.remove(rob)
                return False

        # add requirement to the list of cached packages
        if self.find_satisfactory_requirement(requirement_specifier) is None:
            self.cached_packages.append(requirement_specifier)
        logging.info('Done')
        return True

    def find_satisfactory_requirement(self, requirement_specifier):
        for requirement in self.cached_packages:
            if requirement_specifier.allows(requirement):
                return requirement

        return None

    def tag(self, args):
        tag_name = args.tag
        self._perrepo('git tag %s' % tag_name)
        self._perrepo('git push origin %s' % tag_name)

    def checkout(self, args):
        self._perrepo('git fetch origin')
        self._perrepo('git checkout -b {0} origin/{0}'.format(args.tag))

    def perrepo(self, args):
        # Use git to find the top-level working folder and run the command
        cmd_str = ' '.join(args.command)
        self._perrepo(cmd_str)

    def reset(self, args):
        if not args.force:
            # Make sure that the user is sure
            print 'Warning: this will delete any local changes you have made to the repos in this project. Press y to go ahead.'
            if get_single_char().lower() != 'y':
                print 'Aborting'
                return

        self._perrepo('git checkout -f')
        self._perrepo('git checkout master')
        self._perrepo('git pull origin master')

    def _perrepo(self, cmd_str):
        verbose = self.settings['verbosity'] > 0
        run_shell('cd "$(git rev-parse --show-toplevel)" && . "%s" && %s'
                  % (self._activate_path(), cmd_str), shell=True, verbose=verbose)

        for d in os.listdir(os.path.join(self.env, 'src')):
            full_path = os.path.join(self.env, 'src', d)
            if os.path.isdir(full_path):
                logging.info('Running command in %s' % full_path)
                run_shell('cd "%s" && . "%s" && %s' % (full_path, self._activate_path(),
                                                       cmd_str), shell=True, verbose=verbose)

    def _activate_path(self):
        """Return the path to the virtual env activate file."""
        return os.path.join(self.env, 'bin', 'activate')

    def install(self, args):
        # grab index locations
        
        logging.info('Starting Robustus install using robustus version %s' % __version__)
        
        if args.find_links is not None:
            self.settings['find_links'] = args.find_links

        # determine whether to do cloning of editable non-versioned requirements
        self.settings['update_editables'] = args.update_editables
        self.settings['no_remote_cache'] = args.no_remote_cache
        self.settings['ignore_missing_refs'] = args.ignore_missing_refs
        self.settings['allow_external'] = args.allow_external
        self.settings['allow_all_external'] = args.allow_all_external
        self.settings['allow_unverified'] = args.allow_unverified

        tag = args.tag
        if tag is not None:
            logging.info('Installing with tag %s ignore_missing_refs=%s' %
                         (tag, str(self.settings['ignore_missing_refs'])))

        self.settings['attempts'] = args.attempts

        # construct requirements list
        specifiers = args.packages
        if args.editable is not None:
            specifiers += ['-e ' + r for r in args.editable]
        # NOTE: If "tag=tag" is not passed to "expand_requirements_specifiers", then the
        # "requirements.txt" files expanded will be those on the default/"master" branch
        # (i.e., default kwarg "tag=None") not the branch/tag indicated by value of "tag".
        visited_sites = collections.OrderedDict()
        requirements = expand_requirements_specifiers(specifiers, tag=tag, visited_sites=visited_sites,
                                                      ignore_missing_refs=self.settings['ignore_missing_refs'])
        if args.requirement is not None:
            for requirement_file in args.requirement:
                requirements += read_requirement_file(requirement_file, tag,
                                                      ignore_missing_refs = self.settings['ignore_missing_refs'],
                                                      visited_sites=visited_sites)

        if len(requirements) == 0:
            raise RobustusException('You must give at least one requirement to install (see "robustus install -h")')

        requirements = remove_duplicate_requirements(requirements)

        logging.info('Here are all packages cached in robustus:\n' +
                     '\n'.join([r.freeze() for r in self.cached_packages]) + '\n')

        logging.info('Here are all the requirements robustus is going to install:\n' +
                     '\n'.join([r.freeze() for r in requirements]) + '\n')

        logging.info('These are all packages that were specified: \n' +
                     generate_dependency_list(visited_sites, selected_requirements=requirements))

        # workaround for xcode 5.1 upgrade. clang fails if there are unused arguments
        # specified during installation of some packages (cython, pygame, etc).
        if sys.platform.startswith('darwin'):
            os.environ['CFLAGS'] = '-Qunused-arguments'
            os.environ['CPPFLAGS'] = '-Qunused-arguments'
        
        # install
        for requirement_specifier in requirements:
            self.install_requirement(requirement_specifier, args.no_index, tag)

        # Display the branch of the currently installed repos.
        src_dirs = [os.path.join(os.getcwd(), 'venv', 'src', r.base_name().replace('_', '-')) for r in requirements if r.editable]
        logging.info('{0} Running on the following branches: {0}'.format('='*10))
        old_dir = os.getcwd()
        for directory in src_dirs:
            _, name_of_repo = os.path.split(directory)
            try:
                os.chdir(directory)
                msg =subprocess.check_output('git branch -rv --abbrev=40|grep $(git rev-parse HEAD)', shell=True)
                try:
                    active_branch |= re.search('\\*\w+master', msg).group(0)
                except Exception as err:
                    # Try a second option
                    pass
                active_branch = re.search('origin/\w*', msg).group(0)
                # http://stackoverflow.com/questions/6657690/python-getoutput-equivalent-in-subprocess
            except Exception as err:
                active_branch = ('<Could not find active branch - %s: %s>. Branch might be out-of-sync with origin. '
                                 'Try running git pull and git push.' % (err.__class__.__name__, err.message))
                
                # Debug what the hell is wrong
                commands_to_try = ['git status', 'git branch -r', 'git branch']
                for cmd in commands_to_try:
                    logging.info('-'*20)
                    logging.info('Trying command "%s"...' % cmd)
                    try: 
                        msg = subprocess.check_output(cmd, shell=True)
                        logging.info(msg)
                        logging.info('...Success')
                    except Exception as err:
                        logging.info('... Failed with error - %s: %s' % (err.__class__.__name__, err.message))
                logging.info('-'*20)
                
            logging.info('  %s: %s' % (name_of_repo, active_branch))
        os.chdir(old_dir)
        logging.info('='*56)

    def search_pkg_config_locations(self, locations=None):
        """
        Search for pkg-config files locations. Usually all libraries are going to '<env>/lib' folder, so
        search it recursively for *.pc files.
        """
        if locations is None:
            locations = [os.path.abspath(os.path.join(self.env, 'lib'))]

        pkg_files_dirs = set()
        for loc in locations:
            for root, dirnames, filenames in os.walk(loc):
                for filename in fnmatch.filter(filenames, '*.pc'):
                    pkg_files_dirs.add(root)

        return list(pkg_files_dirs)

    def install_cmake_package(self, requirement_specifier, cmake_options, ignore_index, clone_url=None, install_dir=None):
        """
        Build and install cmake package into cache & copy it to env.
        """
        pkg_cache_dir = os.path.abspath(os.path.join(self.cache, '%s-%s' % (requirement_specifier.name,
                                                                            requirement_specifier.version)))

        def in_cache():
            return os.path.isdir(pkg_cache_dir)

        if not in_cache() and not ignore_index:
            cwd = os.getcwd()
            archive = None
            download_dir = None
            try:
                if clone_url is not None:
                    retcode = run_shell(['git', 'clone', clone_url],
                                        verbose=self.settings['verbosity'] >= 1)
                    if retcode != 0:
                        raise RequirementException('Failed to clone %s' % clone_url)
                    download_dir = os.path.abspath(requirement_specifier.name)
                    os.chdir(download_dir)

                    # checkout specific version (branch)
                    if requirement_specifier.version is not None:
                        retcode = run_shell(['git', 'checkout', '-b', requirement_specifier.version,
                                             '--track', 'origin/%s' % requirement_specifier.version],
                                            verbose=self.settings['verbosity'] >= 1)
                        if retcode != 0:
                            raise RequirementException('Failed to checkout branch %s in %s' % (requirement_specifier.version,
                                                                                               requirement_specifier.name))

                    # update submodules if any
                    run_shell(['git', 'submodule', 'update', '--init'],
                              verbose=self.settings['verbosity'] >= 1)
                else:
                    archive = self.download(requirement_specifier.name, requirement_specifier.version)
                    download_dir = unpack(archive)
                    os.chdir(download_dir)

                logging.info('Building %s' % requirement_specifier.name)
                env = os.environ.copy()
                env['PKG_CONFIG_PATH'] = ','.join(self.search_pkg_config_locations())
                retcode = run_shell(['cmake', '.', '-DCMAKE_INSTALL_PREFIX=%s' % pkg_cache_dir] + cmake_options,
                                    env=env,
                                    verbose=self.settings['verbosity'] >= 1)
                if retcode != 0:
                    raise RequirementException('%s configure failed' % requirement_specifier.name)

                # on bStem we can build only in single thread, but bStem has 2 cores, thus
                # we are using this weird formula to determine number of threads for make
                retcode = run_shell(['make', '-j%i' % max(multiprocessing.cpu_count() / 2, 1)],
                                    verbose=self.settings['verbosity'] >= 1)
                if retcode != 0:
                    raise RequirementException('%s build failed' % requirement_specifier.name)
                retcode = run_shell(['make', 'install'],
                                    verbose=self.settings['verbosity'] >= 1)
                if retcode != 0:
                    raise RequirementException('%s "make install" failed' % requirement_specifier.name)
            except:
                safe_remove(pkg_cache_dir)
                raise
            finally:
                os.chdir(cwd)
                safe_remove(archive)
                safe_remove(download_dir)

        if in_cache():
            if install_dir is not None:
                if os.path.exists(install_dir):
                    shutil.rmtree(install_dir)
                shutil.copytree(pkg_cache_dir, install_dir)
            else:
                # install directly into venv
                install_dir = self.env
                distutils.dir_util._path_created = {}
                distutils.dir_util.copy_tree(pkg_cache_dir, install_dir)
        else:
            raise RequirementException('can\'t find %s-%s in robustus cache' % (requirement_specifier.name, requirement_specifier.version))

        return install_dir

    def freeze(self, args):
        for requirement in self.cached_packages:
            print requirement.freeze()

    def download(self, package, version):
        """
        Download package archive, look for locations specified using --find-links. Store archive in current
        working folder.
        :param package: package name
        :param version: package version
        :return: path to archive
        """
        logging.info('Searching for package archive %s-%s' % (package, version))
        archive_base_name = '%s-%s' % (package, version)
        extensions = ['.tar.gz', '.tar.bz2', '.zip']
        for index in self.settings['find_links']:
            for archive_name in [archive_base_name + ext for ext in extensions]:
                try:
                    download(os.path.join(index, archive_name), archive_name, verbose=self.settings['verbosity'] >= 2)
                    return os.path.abspath(archive_name)
                except urllib2.URLError:
                    pass

        raise RequirementException('Failed to find package archive %s-%s' % (package, version))

    def download_compiled_archive(self, package, version):
        """
        Download compiled package archive, look for locations specified using --find-links. Store archive in current
        working folder.
        :param package: package name
        :param version: package version
        :return: path to archive or None if not found
        """
        
        if self.settings['no_remote_cache']:
            return None

        if not platform.machine():
            logging.warn('Cannot determine architecture from "platform.machine()".')
            return None

        archive_base_name = '%s-%s-%s' % (package, version, platform.machine())
        logging.info('Searching for compiled package archive %s' % archive_base_name)
        extensions = ['.compiled.tar.gz', '.compiled.tar.bz2', '.compiled.zip']
        for index in self.settings['find_links']:
            for archive_name in [archive_base_name + ext for ext in extensions]:
                try:
                    download(os.path.join(index, archive_name), archive_name, verbose=self.settings['verbosity'] >= 2)
                    return os.path.abspath(archive_name)
                except urllib2.URLError:
                    pass

        logging.info('Failed to find compiled package archive %s' % archive_base_name)
        return None

    def download_cache_from_amazon(self, filename, bucket_name, key, secret):
        if filename is None or bucket_name is None:
            raise RobustusException('In order to download from amazon S3 you should specify filename,'
                                    'bucket, access key and secret access key, see "robustus download_cache -h"')

        try:
            import boto
            from boto.s3.key import Key

            # set boto lib debug to critical
            logging.getLogger('boto').setLevel(logging.CRITICAL)

            # connect to the bucket
            conn = boto.connect_s3(key, secret)
            bucket = conn.get_bucket(bucket_name)

            # go through the list of files
            cwd = os.getcwd()
            os.chdir(self.cache)
            for l in bucket.list():
                if str(l.key) == filename:
                    l.get_contents_to_filename(filename)
                    break
            os.chdir(cwd)
            if not os.path.exists(os.path.join(self.cache, filename)):
                raise RobustusException('Can\'t find file %s in amazon cloud bucket %s' % (filename, bucket_name))
        except ImportError:
            raise RobustusException('To use S3 cloud install boto library into robustus virtual')
        except Exception as e:
            raise RobustusException(e.message)

    def download_cache(self, args):
        """
        Download wheels (binary package archives) from wheelhouse_url and unzip them in wheelhouse
        @return: None
        """
        cwd = os.getcwd()
        os.chdir(self.cache)

        wheelhouse_archive = os.path.basename(args.url)
        try:
            if args.bucket is not None:
                self.download_cache_from_amazon(wheelhouse_archive, args.bucket, args.key, args.secret)
            else:
                logging.info('Downloading ' + args.url)
                subprocess.call(['rsync', '-r', '-l', args.url, '.'])
        except:
            os.chdir(cwd)
            raise

        wheelhouse_archive_lowercase = wheelhouse_archive.lower()
        if wheelhouse_archive_lowercase.endswith('.tar.gz'):
            logging.info('Unzipping')
            subprocess.call(['tar', '-xzvf', wheelhouse_archive])
        elif wheelhouse_archive_lowercase.endswith('.tar.bz'):
            logging.info('Unzipping')
            subprocess.call(['tar', '-xjvf', wheelhouse_archive])
        elif wheelhouse_archive_lowercase.endswith('.zip'):
            logging.info('Unzipping')
            subprocess.call(['unzip', wheelhouse_archive])

        if os.path.isfile(wheelhouse_archive):
            os.remove(wheelhouse_archive)
        os.chdir(cwd)
        logging.info('Done')

    def upload_cache_to_amazon(self, filename, bucket_name, key, secret, public):
        if filename is None or bucket_name is None or key is None or secret is None:
            raise RobustusException('In order to upload to amazon S3 you should specify filename,'
                                    'bucket, access key and secret access key, see "robustus upload_cache -h"')

        if os.path.isdir(filename):
            raise RobustusException('Can\'t upload directory to amazon S3, please specify archive name')

        try:
            import boto
            from boto.s3.key import Key

            # set boto lib debug to critical
            logging.getLogger('boto').setLevel(logging.CRITICAL)

            # connect to the bucket
            conn = boto.connect_s3(key, secret)
            bucket = conn.get_bucket(bucket_name)

            # create a key to keep track of our file in the storage
            k = Key(bucket)
            k.key = filename
            k.set_contents_from_filename(filename)
            if public:
                k.make_public()
        except ImportError:
            raise RobustusException('To use S3 cloud install boto library into robustus virtual')
        except Exception as e:
            raise RobustusException(e.message)

    def upload_cache(self, args):
        cwd = os.getcwd()
        os.chdir(self.cache)

        # compress cache
        cache_archive = os.path.basename(args.url)
        cache_archive_lowercase = cache_archive.lower()
        if cache_archive_lowercase.endswith('.tar.gz'):
            subprocess.call(['tar', '-zcvf', cache_archive] + os.listdir(os.getcwd()))
        elif cache_archive_lowercase.endswith('.tar.bz'):
            subprocess.call(['tar', '-jcvf', cache_archive] + os.listdir(os.getcwd()))
        elif cache_archive_lowercase.endswith('.zip'):
            subprocess.call(['zip', cache_archive] + os.listdir(os.getcwd()))

        try:
            if args.bucket is not None:
                self.upload_cache_to_amazon(cache_archive, args.bucket, args.key, args.secret, args.public)
            else:
                for file in glob.iglob('*'):
                    subprocess.call(['rsync', '-r', '-l', file, args.url])
        finally:
            if os.path.isfile(cache_archive):
                os.remove(cache_archive)
            os.chdir(cwd)

    @staticmethod
    def _create_args_parser():
        parser = argparse.ArgumentParser(description='Tool to make and configure python virtualenv,'
                                                     'setup necessary packages and cache them if necessary.',
                                         prog='robustus')
        parser.add_argument('--env', help='virtualenv to use')
        parser.add_argument('--cache', help='binary package cache directory')
        parser.add_argument('-v',
                            '--verbose',
                            default=0,
                            action='count',
                            dest="verbosity",
                            help='give more output, option is additive, and can be used up to 3 times')
        parser.add_argument('--debug',
                            action='store_true',
                            help="Take actions to assist with debugging such as not deleting packages which fail to build.")

        subparsers = parser.add_subparsers(help='robustus commands')

        env_parser = subparsers.add_parser('env', help='make robustus')
        env_parser.add_argument('env',
                                default='.env',
                                help='virtualenv arguments')
        env_parser.add_argument('-p', '--python',
                                help='python interpreter to use')
        env_parser.add_argument('--prompt',
                                default='(robustus)',
                                help='provides an alternative prompt prefix for this environment')
        env_parser.add_argument('--system-site-packages',
                                default=False,
                                action='store_true',
                                help='give access to the global site-packages dir to the virtual environment')
        env_parser.set_defaults(func=Robustus.env)

        install_parser = subparsers.add_parser('install', help='install packages')
        install_parser.add_argument('-r', '--requirement',
                                    action='append',
                                    help='install all the packages listed in the given'
                                         'requirements file, this option can be used multiple times.')
        install_parser.add_argument('packages',
                                    nargs='*',
                                    help='packages to install in format <package name>==version')
        install_parser.add_argument('-e', '--editable',
                                    action='append',
                                    help='installs package in editable mode')
        install_parser.add_argument('--no-index',
                                    action='store_true',
                                    help='ignore package index (only looking in robustus cache and at --find-links URLs)')
        install_parser.add_argument('-f', '--find-links',
                                    action='append',
                                    help='location where to find robustus packages, also is passed to pip')
        install_parser.add_argument('--update-editables',
                                    action='store_true',
                                    help='clone all editable non-versioned requirements inside venv '
                                         '(by default robustus skips editable requiterements)')
        install_parser.add_argument('--tag',
                                    action='store',
                                    help='Install editables using tag or branch')
        install_parser.add_argument('--ignore-missing-refs',
                                    action='store_true',
                                    help='Warn only but no error if a tag is missing (use with --tag)')
        install_parser.add_argument('--no-remote-cache',
                                    action='store_true',
                                    help='Do not use remote cache for downloading of wheels')
        install_parser.add_argument('--attempts',
                                    action='store',
                                    type=int,