# Copyright 2020 Nexenta by DDN, Inc. All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime
import hashlib
import ipaddress
import os
import posixpath

from eventlet import greenthread
from os_brick.remotefs import remotefs
from oslo_concurrency import processutils
from oslo_log import log as logging
from oslo_utils import strutils
from oslo_utils import timeutils
from oslo_utils import units
import six

from cinder import context
from cinder import coordination
from cinder.i18n import _
from cinder import interface
from cinder import objects
from cinder.privsep import fs
from cinder import utils as cinder_utils
from cinder.volume.drivers.nexenta import jsonrpc
from cinder.volume.drivers.nexenta import options
from cinder.volume.drivers.nexenta import utils as nexenta_utils
from cinder.volume.drivers import nfs

LOG = logging.getLogger(__name__)

DEFAULT_RETRY_COUNT = 10
DEFAULT_RETRY_DELAY = 30


@interface.volumedriver
class NexentaNfsDriver(nfs.NfsDriver):
    """Executes volume driver commands on Nexenta Appliance.

    Version history:

    .. code-block:: none

        1.0.0 - Initial driver version.
        1.1.0 - Auto sharing for enclosing folder.
        1.1.1 - Added caching for NexentaStor appliance 'volroot' value.
        1.1.2 - Ignore "folder does not exist" error in delete_volume and
                delete_snapshot method.
        1.1.3 - Redefined volume_backend_name attribute inherited from
                RemoteFsDriver.
        1.2.0 - Added migrate and retype methods.
        1.3.0 - Extend volume method.
        1.3.1 - Cache capacity info and check shared folders on setup.
        1.3.2 - Pass mount_point_base in init_conn to support host-based
                migration.
        1.4.0 - Refactored NFS driver.
              - Improved storage assisted volume migration.
              - Added revert to snapshot support.
              - Added volume multi-attach.
              - Fixed automatic mode for nexenta_rest_protocol.
              - Added deferred deletion for snapshots.
              - Added synchronization for NMS REST API calls.
              - Added configuration parameters for REST API connect/read
                timeouts, connection retries and backoff factor.
              - Disabled non-blocking mandatory locks.
              - Added informative exception messages for REST API.
              - Improved collection of backend statistics.
        1.4.1 - Added retries on timeouts, network connection errors,
              - SSL errors, proxy and NMS errors.
        1.4.2 - Added flag backend_state to report backend status.
              - Added retry on driver initialization failure.
        1.4.3 - Fixed concurrency issues.
              - Fixed issue with retries when mounting an NFS share.
    """

    VERSION = '1.4.3'
    CI_WIKI_NAME = 'Nexenta_CI'

    vendor_name = 'Nexenta'
    product_name = 'NexentaStor4'
    storage_protocol = 'NFS'
    driver_volume_type = 'nfs'

    def __init__(self, execute=processutils.execute, *args, **kwargs):
        self._remotefsclient = None
        super(NexentaNfsDriver, self).__init__(*args, **kwargs)
        if not self.configuration:
            message = (_('%(product_name)s %(storage_protocol)s '
                         'backend configuration not found')
                       % {'product_name': self.product_name,
                          'storage_protocol': self.storage_protocol})
            raise jsonrpc.NmsException(code='EINVAL', message=message)
        self.configuration.append_config_values(options.NEXENTASTOR4_NFS_OPTS)
        required_options = ['nas_host', 'nas_share_path', 'nexenta_user',
                            'nexenta_password']
        for option in required_options:
            if not self.configuration.safe_get(option):
                message = (_('%(product_name)s %(storage_protocol)s '
                             'backend configuration is missing '
                             'required option: %(option)s')
                           % {'product_name': self.product_name,
                              'storage_protocol': self.storage_protocol,
                              'option': option})
                raise jsonrpc.NmsException(code='EINVAL', message=message)
        root_helper = cinder_utils.get_root_helper()
        mount_point_base = self.configuration.nexenta_mount_point_base
        self.mount_point_base = os.path.realpath(mount_point_base)
        nfs_options = self.configuration.safe_get('nfs_mount_options')
        nas_options = self.configuration.safe_get('nas_mount_options')
        if nfs_options and nas_options:
            LOG.debug('Overriding NFS mount options: '
                      'nfs_mount_options = "%(nfs_options)s" with '
                      'nas_mount_options = "%(nas_options)s"',
                      {'nfs_options': nfs_options,
                       'nas_options': nas_options})
        if nas_options:
            self.mount_options = nas_options
        elif nfs_options:
            self.mount_options = nfs_options
        else:
            self.mount_options = None
        self._remotefsclient = remotefs.RemoteFsClient(
            self.driver_volume_type,
            root_helper, execute=execute,
            nfs_mount_point_base=self.mount_point_base,
            nfs_mount_options=self.mount_options)
        self.backend_name = self._get_backend_name()
        self.nms = None
        self.nas_stat = None
        self.nas_driver = self.__class__.__name__
        self.nas_host = self.configuration.nas_host
        self.nas_path = self.configuration.nas_share_path
        self.nas_pool = self.nas_path.split('/')[0]
        self.volume_compression = (
            self.configuration.nexenta_dataset_compression)
        self.volume_dedup = self.configuration.nexenta_dataset_dedup
        self.sparsed_volumes = self.configuration.nexenta_sparsed_volumes
        self.origin_snapshot_template = (
            self.configuration.nexenta_origin_snapshot_template)
        self.migration_snapshot_prefix = (
            self.configuration.nexenta_migration_snapshot_prefix)
        self.migration_service_prefix = (
            self.configuration.nexenta_migration_service_prefix)
        self.migration_throttle = (
            self.configuration.nexenta_migration_throttle)
        self.nbmand = 'on' if self.configuration.nexenta_nbmand else 'off'

    @staticmethod
    def get_driver_options():
        return options.NEXENTASTOR4_NFS_OPTS

    def do_setup(self, ctxt):
        while not self._do_setup():
            greenthread.sleep(DEFAULT_RETRY_DELAY)

    def _do_setup(self):
        try:
            self.nms = jsonrpc.NmsProxy(self.driver_volume_type,
                                        self.nas_path,
                                        self.configuration)
        except jsonrpc.NmsException as error:
            LOG.error('Failed to initialize RESTful API for backend '
                      '%(backend_name)s on host %(host)s: %(error)s',
                      {'backend_name': self.backend_name,
                       'host': self.host,
                       'error': error})
            return False
        return True

    def check_for_setup_error(self):
        while not self._check_for_setup_error():
            greenthread.sleep(DEFAULT_RETRY_DELAY)

    def _check_for_setup_error(self):
        """Check root filesystem, NFS service and NFS share."""
        fmri = 'svc:/network/nfs/server:default'
        try:
            self.nas_stat = self.nms.folder.get_child_props(self.nas_path, '')
        except jsonrpc.NmsException as error:
            LOG.error('Failed to get stat of SAN path %(nas_path)s: %(error)s',
                      {'nas_path': self.nas_path, 'error': error})
            return False
        if self.nas_stat['mountpoint'] == 'none':
            LOG.error('NAS path %(nas_path)s is not writable',
                      {'nas_path': self.nas_path})
            return False
        if self.nas_stat['mounted'] == 'no':
            LOG.error('NAS path %(nas_path)s is not mounted',
                      {'nas_path': self.nas_path})
            return False
        try:
            service_state = self.nms.netsvc.get_state(fmri)
        except jsonrpc.NmsException as error:
            LOG.error('Failed to get state of NFS service: %(error)s',
                      {'error': error})
            return False
        if service_state != 'online':
            LOG.error('NFS service is not online: %(state)s',
                      {'state': service_state})
            return False
        try:
            shared = self.nms.netstorsvc.is_folder_shared(fmri, self.nas_path)
        except jsonrpc.NmsException as error:
            LOG.error('Failed to get state of NFS share %(share)s: %(error)s',
                      {'share': self.nas_path, 'error': error})
            return False
        if not shared:
            LOG.error('NAS path %(nas_path)s is not shared',
                      {'nas_path': self.nas_path})
            return False
        if self.nas_stat['nbmand'] == self.nbmand:
            return True
        try:
            self.nms.folder.set_child_prop(self.nas_path, 'nbmand',
                                           self.nbmand)
        except jsonrpc.NmsException as error:
            LOG.error('Failed to set NAS path %(nas_path)s property nbmand '
                      'to value %(nbmand)s: %(error)s',
                      {'nas_path': self.nas_path, 'nbmand': self.nbmand,
                       'error': error})
            return False
        self.nas_stat['nbmand'] = self.nbmand
        return True

    def create_volume(self, volume):
        """Creates a volume.

        :param volume: volume reference
        :return: model update dict for volume reference
        """
        volume_path = self._get_volume_path(volume)
        volume_options = {}
        if self.volume_compression:
            volume_options['compression'] = self.volume_compression
        self.nms.folder.create_with_props(self.nas_path,
                                          volume['name'],
                                          volume_options)
        try:
            self._set_volume_acl(volume)
            self._mount_volume(volume)
            volume_file = self.local_path(volume)
            volume_size = volume['size']
            if self.sparsed_volumes:
                self._create_sparsed_file(volume_file, volume_size)
            else:
                compression = self.nms.folder.get_child_prop(volume_path,
                                                             'compression')
                if compression != 'off':
                    self.nms.folder.set_child_prop(volume_file,
                                                   'compression',
                                                   'off')
                self._create_regular_file(volume_file, volume_size)
                if compression != 'off':
                    self.nms.folder.set_child_prop(volume_path,
                                                   'compression',
                                                   compression)
        except jsonrpc.NmsException as create_error:
            try:
                self.nms.folder.destroy(volume_path, '-rf')
            except jsonrpc.NmsException as delete_error:
                LOG.debug('Failed to delete volume %(path)s: %(error)s',
                          {'path': volume_path, 'error': delete_error})
            raise create_error
        finally:
            self._unmount_volume(volume)

    @coordination.synchronized('{self.nms.lock}-{volume[id]}')
    def copy_image_to_volume(self, ctxt, volume, image_service, image_id):
        LOG.debug('Copy image %(image)s to volume %(volume)s',
                  {'image': image_id, 'volume': volume['name']})
        self._mount_volume(volume)
        super(NexentaNfsDriver, self).copy_image_to_volume(
            ctxt, volume, image_service, image_id)
        self._unmount_volume(volume)

    @coordination.synchronized('{self.nms.lock}-{image_meta[id]}')
    def copy_volume_to_image(self, ctxt, volume, image_service, image_meta):
        LOG.debug('Copy volume %(volume)s to image %(image)s',
                  {'volume': volume['name'], 'image': image_meta['id']})
        self._mount_volume(volume)
        super(NexentaNfsDriver, self).copy_volume_to_image(
            ctxt, volume, image_service, image_meta)
        self._unmount_volume(volume)

    def extend_volume(self, volume, new_size):
        """Extend an existing volume.

        :param volume: volume reference
        :param new_size: volume new size in GB
        """
        LOG.debug('Extend volume %(volume)s, new size: %(size)sGB',
                  {'volume': volume['name'], 'size': new_size})
        self._mount_volume(volume)
        volume_file = self.local_path(volume)
        if self.sparsed_volumes:
            self._execute('truncate', '-s',
                          '%dG' % new_size,
                          volume_file,
                          run_as_root=True)
        else:
            seek = volume['size'] * units.Ki
            count = (new_size - volume['size']) * units.Ki
            self._execute('dd',
                          'if=/dev/zero',
                          'of=%s' % volume_file,
                          'bs=%d' % units.Mi,
                          'seek=%d' % seek,
                          'count=%d' % count,
                          run_as_root=True)
        self._unmount_volume(volume)

    def delete_volume(self, volume):
        """Deletes a logical volume.

        :param volume: volume reference
        """
        volume_path = self._get_volume_path(volume)
        try:
            origin = self.nms.folder.get_child_prop(volume_path, 'origin')
        except jsonrpc.NmsException as error:
            if error.code == 'ENOENT':
                return
            raise
        self.nms.folder.destroy(volume_path, '-rf')
        if not origin:
            return
        template = self.origin_snapshot_template
        parent_path, snapshot_name = origin.split('@')
        if not nexenta_utils.match_template(template, snapshot_name):
            return
        try:
            self.nms.snapshot.destroy(origin, '-d')
        except jsonrpc.NmsException as error:
            LOG.error('Failed to delete origin snapshot %(origin)s '
                      'of volume %(volume)s: %(error)s',
                      {'origin': origin,
                       'volume': volume['name'],
                       'error': error})

    def create_volume_from_snapshot(self, volume, snapshot):
        """Create new volume from other's snapshot on appliance.

        :param volume: reference of volume to be created
        :param snapshot: reference of source snapshot
        """
        LOG.debug('Create volume %(volume)s from snapshot %(snapshot)s',
                  {'volume': volume['name'], 'snapshot': snapshot['name']})
        snapshot_path = self._get_snapshot_path(snapshot)
        clone_path = self._get_volume_path(volume)
        self.nms.folder.clone(snapshot_path, clone_path)
        if volume['size'] > snapshot['volume_size']:
            new_size = volume['size']
            volume['size'] = snapshot['volume_size']
            self.extend_volume(volume, new_size)
            volume['size'] = new_size

    def create_cloned_volume(self, volume, src_vref):
        """Creates a clone of the specified volume.

        :param volume: new volume reference
        :param src_vref: source volume reference
        """
        snapshot = {
            'name': self.origin_snapshot_template % volume['id'],
            'volume_id': src_vref['id'],
            'volume_name': src_vref['name'],
            'volume_size': src_vref['size']
        }
        self.create_snapshot(snapshot)
        try:
            self.create_volume_from_snapshot(volume, snapshot)
        except jsonrpc.NmsException as error:
            LOG.error('Failed to create clone %(clone)s '
                      'from volume %(volume)s: %(error)s',
                      {'clone': volume['name'],
                       'volume': src_vref['name'],
                       'error': error})
            raise
        finally:
            try:
                self.delete_snapshot(snapshot)
            except jsonrpc.NmsException as error:
                LOG.error('Failed to delete temporary snapshot '
                          '%(volume)s@%(snapshot)s: %(error)s',
                          {'volume': src_vref['name'],
                           'snapshot': snapshot['name'],
                           'error': error})

    def _ensure_share_unmounted(self, share):
        """Ensure that NFS share is unmounted on the host.

        :param share: share path
        """
        mntpoint = self._get_mount_point_for_share(share)
        mntpoints = self._remotefsclient._read_mounts()
        if mntpoint not in mntpoints:
            LOG.debug('NFS share %(share)s is not mounted at %(mntpoint)s',
                      {'share': share, 'mntpoint': mntpoint})
            return
        attempts = max(1, self.configuration.nfs_mount_attempts)
        for attempt in range(1, attempts + 1):
            try:
                fs.umount(mntpoint)
            except processutils.ProcessExecutionError as error:
                LOG.debug('Unmount attempt %(attempt)s failed: %(error)s, '
                          'retrying unmount NFS share %(share)s mounted '
                          'at %(mntpoint)s',
                          {'attempt': attempt, 'error': error,
                           'share': share, 'mntpoint': mntpoint})
                if attempt == attempts:
                    LOG.error('Failed to unmount NFS share %(share)s '
                              'mounted at %(mntpoint)s after %(attempt)s '
                              'attempts: %(error)s',
                              {'share': share, 'mntpoint': mntpoint,
                               'attempt': attempt, 'error': error})
                    raise
                greenthread.sleep(DEFAULT_RETRY_DELAY)
            else:
                LOG.debug('NFS share %(share)s has been unmounted at '
                          '%(mntpoint)s after %(attempt)s attempts',
                          {'share': share, 'mntpoint': mntpoint,
                           'attempt': attempt})
                break
        self._delete(mntpoint)

    def _delete(self, mntpoint):
        """Override parent method for safe remove mountpoint."""
        try:
            self._execute('rm', '-d', mntpoint, run_as_root=True)
        except processutils.ProcessExecutionError as error:
            LOG.debug('Failed to remove mountpoint %(mntpoint)s: %(error)s',
                      {'mntpoint': mntpoint, 'error': error})
        else:
            LOG.debug('The mountpoint %(mntpoint)s has been removed',
                      {'mntpoint': mntpoint})

    def _mount_volume(self, volume):
        """Ensure that volume is activated and mounted on the host."""
        share = self._get_volume_share(volume)
        self._ensure_share_mounted(share)

    def _unmount_volume(self, volume):
        """Ensure that volume is unmounted on the host."""
        try:
            share = self._get_volume_share(volume)
        except jsonrpc.NmsException as error:
            if error.code == 'ENOENT':
                return
        self._ensure_share_unmounted(share)

    def create_export(self, ctxt, volume, connector):
        """Driver entry point to get the export info for a new volume."""
        pass

    def ensure_export(self, ctxt, volume):
        """Driver entry point to get the export info for an existing volume."""
        pass

    @coordination.synchronized('{self.nms.lock}-{volume[id]}')
    def remove_export(self, ctxt, volume):
        """Driver entry point to remove an export for a volume."""
        self._unmount_volume(volume)

    def terminate_connection(self, volume, connector, **kwargs):
        """Terminate a connection to a volume.

        :param volume: a volume object
        :param connector: a connector object
        :returns: dictionary of connection information
        """
        pass

    @coordination.synchronized('{self.nms.lock}-{volume[id]}')
    def initialize_connection(self, volume, connector):
        """Allow connection to connector and return connection info.

        :param volume: volume reference
        :param connector: connector reference
        :returns: dictionary of connection information
        """
        LOG.debug('Initialize volume connection for %(volume)s',
                  {'volume': volume['name']})
        share = self._get_volume_share(volume)
        data = {
            'export': share,
            'name': 'volume'
        }
        if self.mount_options:
            data['options'] = '-o %s' % self.mount_options
        info = {
            'driver_volume_type': self.driver_volume_type,
            'mount_point_base': self.mount_point_base,
            'data': data
        }
        return info

    def _get_bound_host(self, host):
        """Get user@host:port from SSH bindings."""
        try:
            bindings = self.nms.appliance.ssh_list_bindings()
        except jsonrpc.NmsException as error:
            LOG.error('Failed to get SSH bindings: %(error)s',
                      {'error': error})
            return None
        for user_host_port in bindings:
            binding = bindings[user_host_port]
            if not (isinstance(binding, list) and len(binding) == 4):
                LOG.warning('Skip incompatible SSH binding: %(binding)s',
                            {'binding': binding})
                continue
            data = binding[2]
            items = data.split(',')
            for item in items:
                if host == item.strip():
                    return user_host_port
        return None

    def _svc_state(self, fmri, state):
        retries = DEFAULT_RETRY_COUNT
        while retries:
            greenthread.sleep(DEFAULT_RETRY_DELAY)
            retries -= 1
            try:
                status = self.nms.autosvc.get_state(fmri)
            except jsonrpc.NmsException as error:
                LOG.error('Failed to get state of migration '
                          'service %(fmri)s: %(error)s',
                          {'fmri': fmri, 'error': error})
                continue
            if status == 'uninitialized':
                continue
            elif status == state:
                return True
            if state == 'online':
                method = getattr(self.nms.autosvc, 'enable')
            elif state == 'disabled':
                method = getattr(self.nms.autosvc, 'disable')
            else:
                LOG.error('Request unknown service state: %(state)s',
                          {'state': state})
                return False
            try:
                method(fmri)
            except jsonrpc.NmsException as error:
                LOG.error('Failed to change state of migration service '
                          '%(fmri)s to %(state)s: %(error)s',
                          {'fmri': fmri, 'state': state, 'error': error})
        LOG.error('Unable to change state of migration service %(fmri)s '
                  'to %(state)s: maximum retries exceeded (%(retries)s)',
                  {'fmri': fmri, 'state': state, 'retries': retries})
        return False

    def _svc_progress(self, fmri):
        """Get progress for SMF service."""
        progress = 0
        try:
            estimations = self.nms.autosync.get_estimations(fmri)
        except jsonrpc.NmsException as error:
            LOG.error('Failed to get estimations for migration '
                      'service %(fmri)s: %(error)s',
                      {'fmri': fmri, 'error': error})
            return progress
        size = estimations.get('curt_siz')
        sent = estimations.get('curt_sen')
        try:
            size = float(size)
            sent = float(sent)
            if size > 0:
                progress = int(100 * sent / size)
        except (TypeError, ValueError) as error:
            LOG.error('Failed to parse estimations statistics '
                      '%(estimations)s for migration service '
                      '%(fmri)s: %(error)s',
                      {'estimations': estimations,
                       'fmri': fmri, 'error': error})
        return progress

    def _svc_result(self, fmri):
        try:
            props = self.nms.autosvc.get_child_props(fmri, '')
        except jsonrpc.NmsException as error:
            LOG.error('Failed to get properties of migration service '
                      '%(fmri)s: %(error)s',
                      {'fmri': fmri, 'error': error})
            return False
        history = props.get('zfs/run_history')
        if not history:
            LOG.error('Failed to get history of migration service '