from collections import namedtuple

from blivet.size import Size

from pyanaconda.anaconda_loggers import get_module_logger
from pyanaconda.core.i18n import CN_, CP_
from pyanaconda.modules.common.structures.storage import DeviceData
from pyanaconda.ui.lib.storage import apply_disk_selection, try_populate_devicetree, \
    filter_disks_by_names
from pyanaconda.modules.common.constants.objects import DISK_SELECTION, FCOE, ISCSI, DASD, \
    DEVICE_TREE
from pyanaconda.modules.common.constants.services import STORAGE

from pyanaconda.ui.gui.utils import timed_action, really_show, really_hide
from pyanaconda.ui.gui.spokes import NormalSpoke
from pyanaconda.ui.gui.spokes.advstorage.fcoe import FCoEDialog
from pyanaconda.ui.gui.spokes.advstorage.iscsi import ISCSIDialog
from pyanaconda.ui.gui.spokes.advstorage.zfcp import ZFCPDialog
from pyanaconda.ui.gui.spokes.advstorage.dasd import DASDDialog
from pyanaconda.ui.gui.spokes.advstorage.nvdimm import NVDIMMDialog
from pyanaconda.ui.gui.spokes.lib.cart import SelectedDisksDialog
from pyanaconda.ui.categories.system import SystemCategory

import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk

log = get_module_logger(__name__)

__all__ = ["FilterSpoke"]

PAGE_SEARCH = 0
PAGE_MULTIPATH = 1
PAGE_OTHER = 2
PAGE_NVDIMM = 3
PAGE_Z = 4

DiskStoreRow = namedtuple("DiskStoreRow", [
    "visible", "selected", "mutable",
    "name", "type", "model", "capacity",
    "vendor", "interconnect", "serial",
    "wwid", "paths", "port", "target",
    "lun", "ccw", "wwpn", "namespace", "mode"
])

def create_row(device_data, selected, mutable):
    """Create a disk store row for the given data.

    :param device_data: an instance of DeviceData
    :param selected: True if the device is selected, otherwise False
    :param mutable: False if the device is protected, otherwise True
    :return: an instance of DiskStoreRow
    """
    return DiskStoreRow(
        visible=True,
        selected=selected,
        mutable=mutable and not device_data.protected,
        name=device_data.name,
        type=device_data.type,
        model=device_data.attrs.get("model", ""),
        capacity=str(Size(device_data.size)),
        vendor=device_data.attrs.get("vendor", ""),
        interconnect=device_data.attrs.get("bus", ""),
        serial=device_data.attrs.get("serial", ""),
        wwid=device_data.attrs.get("path-id", ""),
        paths="\n".join(device_data.parents),
        port=device_data.attrs.get("port", ""),
        target=device_data.attrs.get("target", ""),
        lun=device_data.attrs.get("lun", "") or device_data.attrs.get("fcp-lun", ""),
        ccw=device_data.attrs.get("hba-id", ""),
        wwpn=device_data.attrs.get("wwpn", ""),
        namespace=device_data.attrs.get("namespace", ""),
        mode=device_data.attrs.get("mode", "")
    )

class FilterPage(object):
    """A FilterPage is the logic behind one of the notebook tabs on the filter
       UI spoke.  Each page has its own specific filtered model overlaid on top
       of a common model that holds all non-advanced disks.

       A Page is created once, when the filter spoke is initialized.  It is
       setup multiple times - each time the spoke is revisited.  When the Page
       is setup, it is given a complete view of all disks that belong on this
       Page.  This is because certain pages may require populating a combo with
       all vendor names, or other similar tasks.

       This class is just a base class.  One subclass should be created for each
       more specialized type of page.  Only one instance of each subclass should
       ever be created.
    """
    # Default value of a type combo.
    SEARCH_TYPE_NONE = 'None'

    def __init__(self, builder, model_name, combo_name):
        """Create a new FilterPage instance.

        :param builder: a instance of the Gtk.Builder
        :param model_name: a name of the filter model
        :param combo_name: a name of the type combo
        """
        self._builder = builder
        self._is_active = False

        self._model = self._builder.get_object(model_name)
        self._model.set_visible_func(self.visible_func)

        self._combo = self._builder.get_object(combo_name)

    @property
    def model(self):
        """The model."""
        return self._model

    @property
    def is_active(self):
        """Is the filter active?"""
        return self._is_active

    @is_active.setter
    def is_active(self, value):
        self._is_active = value

    def is_member(self, device_type):
        """Does device belong on this page?  This function should taken into
           account what kind of thing device is.  It should not be concerned
           with any sort of filtering settings.  It only determines whether
           device belongs.
        """
        return True

    def setup(self, store, disks, selected_names, protected_names):
        """Do whatever setup of the UI is necessary before this page can be
           displayed.  This function is called every time the filter spoke
           is revisited, and thus must first do any cleanup that is necessary.

           The setup function is passed a reference to the primary store, a list
           of names of disks the user has selected (either from a previous visit
           or via kickstart), and a list of all disk objects that belong on this
           page as determined from the is_member method.

           At the least, this method should add all the disks to the store.  It
           may also need to populate combos and other lists as appropriate.
        """
        pass

    def _setup_combo(self, combo, items):
        """Populate a given GtkComboBoxText instance with a list of items.

        The combo will first be cleared, so this method is suitable for calling
        repeatedly. The first item in the list will be empty to allow the combo
        box criterion to be cleared. The first non-empty item in the list will
        be selected by default.
        """
        combo.remove_all()
        combo.append_text('')

        # Remove duplicate and empty items and sort them.
        items = sorted(set(filter(None, items)))

        for i in items:
            combo.append_text(i)

        if items:
            combo.set_active(1)

    def _setup_search_type(self):
        """Set up the default search type."""
        self._combo.set_active_id(self.SEARCH_TYPE_NONE)
        self._combo.emit("changed")

    def clear(self):
        """Blank out any filtering-related fields on this page and return them
           to their defaults.  This is called when the Clear button is clicked.
        """
        pass

    def visible_func(self, model, itr, *args):
        """This method is called for every row (disk) in the store, in order to
           determine if it should be displayed on this page or not.  This method
           should take into account whether is_active is set, perhaps whether
           something in pyanaconda.flags is setup, and other settings to make
           a final decision.  Because filtering can be complicated, many pages
           will want to farm this decision out to another method.

           The return value is a boolean indicating whether the row is visible
           or not.
        """
        if not self._is_active:
            return True

        row = DiskStoreRow(*model[itr])
        if not self.is_member(row.type):
            return False

        log.debug("Filter %s with %s.", row.name, str(self))

        filter_by = self._combo.get_active_id()
        if filter_by == self.SEARCH_TYPE_NONE:
            return True

        return self._filter_func(filter_by, row)

    def _filter_func(self, filter_by, row):
        """Filter a row by the specified filter."""
        return True

    def __str__(self):
        """Get the name of the filter."""
        return self.__class__.__name__

class SearchPage(FilterPage):
    # Match these to searchTypeCombo ids in glade
    SEARCH_TYPE_PORT_TARGET_LUN = 'PTL'
    SEARCH_TYPE_WWID = 'WWID'

    def __init__(self, builder):
        super().__init__(builder, "searchModel", "searchTypeCombo")
        self._lun_entry = self._builder.get_object("searchLUNEntry")
        self._wwid_entry = self._builder.get_object("searchWWIDEntry")
        self._port_combo = self._builder.get_object("searchPortCombo")
        self._target_entry = self._builder.get_object("searchTargetEntry")

    def setup(self, store, disks, selected_names, protected_names):
        ports = set()

        for device_data in disks:
            ports.add(device_data.attrs.get("port"))

        self._setup_combo(self._port_combo, ports)
        self._setup_search_type()

    def clear(self):
        self._lun_entry.set_text("")
        self._port_combo.set_active(0)
        self._target_entry.set_text("")
        self._wwid_entry.set_text("")

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_PORT_TARGET_LUN:
            port = self._port_combo.get_active_text()
            if port and port != row.port:
                return False

            target = self._target_entry.get_text().strip()
            if target and target not in row.target:
                return False

            lun = self._lun_entry.get_text().strip()
            if lun and lun not in row.lun:
                return False

            return True

        if filter_by == self.SEARCH_TYPE_WWID:
            return self._wwid_entry.get_text() in row.wwid

        return False

class MultipathPage(FilterPage):
    # Match these to multipathTypeCombo ids in glade
    SEARCH_TYPE_VENDOR = 'Vendor'
    SEARCH_TYPE_INTERCONNECT = 'Interconnect'
    SEARCH_TYPE_WWID = 'WWID'

    def __init__(self, builder):
        super().__init__(builder, "multipathModel", "multipathTypeCombo")
        self._ic_combo = self._builder.get_object("multipathInterconnectCombo")
        self._vendor_combo = self._builder.get_object("multipathVendorCombo")
        self._wwid_entry = self._builder.get_object("multipathWWIDEntry")

    def is_member(self, device_type):
        return device_type == "dm-multipath"

    def setup(self, store, disks, selected_names, protected_names):
        vendors = set()
        interconnects = set()

        for device_data in disks:
            row = create_row(
                device_data,
                device_data.name in selected_names,
                device_data.name not in protected_names
            )

            store.append(list(row))
            vendors.add(device_data.attrs.get("vendor"))
            interconnects.add(device_data.attrs.get("bus"))

        self._setup_combo(self._vendor_combo, vendors)
        self._setup_combo(self._ic_combo, interconnects)
        self._setup_search_type()

    def clear(self):
        self._ic_combo.set_active(0)
        self._vendor_combo.set_active(0)
        self._wwid_entry.set_text("")

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_VENDOR:
            return row.vendor == self._vendor_combo.get_active_text()

        if filter_by == self.SEARCH_TYPE_INTERCONNECT:
            return row.interconnect == self._ic_combo.get_active_text()

        if filter_by == self.SEARCH_TYPE_WWID:
            return self._wwid_entry.get_text() in row.wwid

        return False

class OtherPage(FilterPage):
    # Match these to otherTypeCombo ids in glade
    SEARCH_TYPE_VENDOR = 'Vendor'
    SEARCH_TYPE_INTERCONNECT = 'Interconnect'
    SEARCH_TYPE_ID = 'ID'

    def __init__(self, builder):
        super().__init__(builder, "otherModel", "otherTypeCombo")
        self._ic_combo = self._builder.get_object("otherInterconnectCombo")
        self._id_entry = self._builder.get_object("otherIDEntry")
        self._vendor_combo = self._builder.get_object("otherVendorCombo")

    def is_member(self, device_type):
        return device_type == "iscsi" or device_type == "fcoe"

    def setup(self, store, disks, selected_names, protected_names):
        vendors = set()
        interconnects = set()

        for device_data in disks:
            row = create_row(
                device_data,
                device_data.name in selected_names,
                device_data.name not in protected_names
            )

            store.append([*row])
            vendors.add(device_data.attrs.get("vendor"))
            interconnects.add(device_data.attrs.get("bus"))

        self._setup_combo(self._vendor_combo, vendors)
        self._setup_combo(self._ic_combo, interconnects)
        self._setup_search_type()

    def clear(self):
        self._ic_combo.set_active(0)
        self._id_entry.set_text("")
        self._vendor_combo.set_active(0)

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_VENDOR:
            return self._vendor_combo.get_active_text() == row.vendor

        if filter_by == self.SEARCH_TYPE_INTERCONNECT:
            return self._ic_combo.get_active_text() == row.interconnect

        if filter_by == self.SEARCH_TYPE_ID:
            return self._id_entry.get_text().strip() in row.wwid

        return False

class ZPage(FilterPage):
    # Match these to zTypeCombo ids in glade
    SEARCH_TYPE_CCW = 'CCW'
    SEARCH_TYPE_WWPN = 'WWPN'
    SEARCH_TYPE_LUN = 'LUN'

    def __init__(self, builder):
        super().__init__(builder, "zModel", "zTypeCombo")
        self._ccw_entry = self._builder.get_object("zCCWEntry")
        self._wwpn_entry = self._builder.get_object("zWWPNEntry")
        self._lun_entry = self._builder.get_object("zLUNEntry")

    def clear(self):
        self._lun_entry.set_text("")
        self._ccw_entry.set_text("")
        self._wwpn_entry.set_text("")

    def is_member(self, device_type):
        return device_type == "zfcp" or device_type == "dasd"

    def setup(self, store, disks, selected_names, protected_names):
        """ Set up our Z-page, but only if we're running on s390x. """
        for device_data in disks:
            if device_data.type != "zfcp":
                continue

            row = create_row(
                device_data,
                device_data.name in selected_names,
                device_data.name not in protected_names
            )

            store.append([*row])

        self._setup_search_type()

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_CCW:
            return self._ccw_entry.get_text() in row.ccw

        if filter_by == self.SEARCH_TYPE_WWPN:
            return self._wwpn_entry.get_text() in row.wwpn

        if filter_by == self.SEARCH_TYPE_LUN:
            return self._lun_entry.get_text() in row.lun

        return False

class NvdimmPage(FilterPage):
    # Match these to nvdimmTypeCombo ids in glade
    SEARCH_TYPE_NAMESPACE = 'Namespace'
    SEARCH_TYPE_MODE = 'Mode'

    def __init__(self, builder):
        super().__init__(builder, "nvdimmModel", "nvdimmTypeCombo")
        self._tree_view = self._builder.get_object("nvdimmTreeView")
        self._mode_combo = self._builder.get_object("nvdimmModeCombo")
        self._namespace_entry = self._builder.get_object("nvdimmNamespaceEntry")

    def is_member(self, device_type):
        return device_type == "nvdimm"

    def setup(self, store, disks, selected_names, protected_names):
        modes = set()

        for device_data in disks:
            mode = device_data.attrs.get("mode")
            row = create_row(
                device_data,
                device_data.name in selected_names and mode == "sector",
                device_data.name not in protected_names or mode != "sector",
            )

            store.append([*row])
            modes.add(mode)

        self._setup_combo(self._mode_combo, modes)
        self._setup_search_type()

    def clear(self):
        self._mode_combo.set_active(0)
        self._namespace_entry.set_text("")

    def _filter_func(self, filter_by, row):
        if filter_by == self.SEARCH_TYPE_MODE:
            return self._mode_combo.get_active_text() == row.mode

        if filter_by == self.SEARCH_TYPE_NAMESPACE:
            return self._namespace_entry.get_text().strip() in row.namespace

        return False

    def get_selected_namespaces(self):
        namespaces = []
        selection = self._tree_view.get_selection()
        store, path_list = selection.get_selected_rows()

        for path in path_list:
            store_row = DiskStoreRow(*store[store.get_iter(path)])
            namespaces.append(store_row.namespace)

        return namespaces

class FilterSpoke(NormalSpoke):
    """
       .. inheritance-diagram:: FilterSpoke
          :parts: 3
    """
    builderObjects = ["diskStore", "filterWindow",
                      "searchModel", "multipathModel", "otherModel", "zModel", "nvdimmModel"]
    mainWidgetName = "filterWindow"
    uiFile = "spokes/advanced_storage.glade"
    category = SystemCategory
    title = CN_("GUI|Spoke", "_Installation Destination")

    @staticmethod
    def get_screen_id():
        """Return a unique id of this UI screen."""
        return "storage-advanced-configuration"

    def __init__(self, *args):
        super().__init__(*args)
        self.applyOnSkip = True

        self._pages = {}
        self._ancestors = []
        self._disks = []
        self._selected_disks = []
        self._protected_disks = []

        self._storage_module = STORAGE.get_proxy()
        self._device_tree = STORAGE.get_proxy(DEVICE_TREE)
        self._disk_selection = STORAGE.get_proxy(DISK_SELECTION)

        self._notebook = self.builder.get_object("advancedNotebook")
        self._store = self.builder.get_object("diskStore")
        self._reconfigure_nvdimm_button = self.builder.get_object("reconfigureNVDIMMButton")

    @property
    def indirect(self):
        return True

    # This spoke has no status since it's not in a hub
    @property
    def status(self):
        return None

    def apply(self):
        apply_disk_selection(self._selected_disks)

    def initialize(self):
        super().initialize()
        self.initialize_start()

        self._pages = {
            PAGE_SEARCH: SearchPage(self.builder),
            PAGE_MULTIPATH: MultipathPage(self.builder),
            PAGE_OTHER: OtherPage(self.builder),
            PAGE_NVDIMM: NvdimmPage(self.builder),
            PAGE_Z: ZPage(self.builder),
        }

        if not STORAGE.get_proxy(DASD).IsSupported():
            self._notebook.remove_page(PAGE_Z)
            self._pages.pop(PAGE_Z)

            self.builder.get_object("addZFCPButton").destroy()
            self.builder.get_object("addDASDButton").destroy()

        if not STORAGE.get_proxy(FCOE).IsSupported():
            self.builder.get_object("addFCOEButton").destroy()

        if not STORAGE.get_proxy(ISCSI).IsSupported():
            self.builder.get_object("addISCSIButton").destroy()

        # The button is sensitive only on NVDIMM page
        self._reconfigure_nvdimm_button.set_sensitive(False)

        # report that we are done
        self.initialize_done()

    def refresh(self):
        super().refresh()

        # Reset the scheduled partitioning if any to make sure that we
        # are working with the current system's storage configuration.
        # FIXME: Change modules and UI to work with the right device tree.
        self._storage_module.ResetPartitioning()

        self._disks = self._disk_selection.GetUsableDisks()
        self._selected_disks = self._disk_selection.SelectedDisks
        self._protected_disks = self._disk_selection.ProtectedDevices
        self._ancestors = self._device_tree.GetAncestors(self._disks)

        # Now all all the non-local disks to the store.  Everything has been set up
        # ahead of time, so there's no need to configure anything.  We first make
        # these lists of disks, then call setup on each individual page.  This is
        # because there could be page-specific setup to do that requires a complete
        # view of all the disks on that page.
        self._store.clear()

        disks_data = DeviceData.from_structure_list([
            self._device_tree.GetDeviceData(device_name)
            for device_name in self._disks
        ])

        for page in self._pages.values():
            disks = [
                d for d in disks_data
                if page.is_member(d.type)
            ]

            page.setup(
                self._store,
                disks,
                self._selected_disks,
                self._protected_disks
            )

        self._update_summary()

    def _update_summary(self):
        summary_button = self.builder.get_object("summary_button")
        label = self.builder.get_object("summary_button_label")

        # We need to remove ancestor devices from the count.  Otherwise, we'll
        # end up in a situation where selecting one multipath device could
        # potentially show three devices selected (mpatha, sda, sdb for instance).
        count = len([
            disk for disk in self._selected_disks
            if disk not in self._ancestors
        ])

        summary = CP_(
            "GUI|Installation Destination|Filter",
            "{} _storage device selected",
            "{} _storage devices selected",
            count
        ).format(count)

        if count > 0:
            really_show(summary_button)
            label.set_text(summary)
            label.set_use_underline(True)
        else:
            really_hide(summary_button)

    def on_back_clicked(self, button):
        self.skipTo = "StorageSpoke"
        super().on_back_clicked(button)

    def on_summary_clicked(self, button):
        disks = filter_disks_by_names(
            self._disks, self._selected_disks
        )
        dialog = SelectedDisksDialog(
            self.data, disks, show_remove=False, set_boot=False
        )

        with self.main_window.enlightbox(dialog.window):
            dialog.refresh()
            dialog.run()

    def on_clear_icon_clicked(self, entry, icon_pos, event):
        if icon_pos == Gtk.EntryIconPosition.SECONDARY:
            entry.set_text("")

    def on_page_switched(self, notebook, new_page, new_page_num, *args):
        # Disable all filters.
        for page in self._pages.values():
            page.is_active = False

        # Set up the new page.
        page = self._pages[new_page_num]
        page.is_active = True
        page.model.refilter()

        log.debug("Show the page %s.", str(page))

        # Set up the UI.
        notebook.get_nth_page(new_page_num).show_all()
        self._reconfigure_nvdimm_button.set_sensitive(new_page_num == 3)

    def on_row_toggled(self, button, path):
        if not path:
            return

        page_index = self._notebook.get_current_page()
        filter_model = self._pages[page_index].model
        model_itr = filter_model.get_iter(path)
        itr = filter_model.convert_iter_to_child_iter(model_itr)
        self._store[itr][1] = not self._store[itr][1]

        if self._store[itr][1] and self._store[itr][3] not in self._selected_disks:
            self._selected_disks.append(self._store[itr][3])
        elif not self._store[itr][1] and self._store[itr][3] in self._selected_disks:
            self._selected_disks.remove(self._store[itr][3])

        self._update_summary()

    @timed_action(delay=50, threshold=100)
    def on_refresh_clicked(self, widget, *args):
        log.debug("Refreshing...")
        try_populate_devicetree()
        self.refresh()

    def on_add_iscsi_clicked(self, widget, *args):
        log.debug("Add a new iSCSI device.")
        dialog = ISCSIDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_add_fcoe_clicked(self, widget, *args):
        log.debug("Add a new FCoE device.")
        dialog = FCoEDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_add_zfcp_clicked(self, widget, *args):
        log.debug("Add a new zFCP device.")
        dialog = ZFCPDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_add_dasd_clicked(self, widget, *args):
        log.debug("Add a new DASD device.")
        dialog = DASDDialog(self.data)
        self._run_dialog_and_refresh(dialog)

    def on_reconfigure_nvdimm_clicked(self, widget, *args):
        log.debug("Reconfigure a NVDIMM device.")
        namespaces = self._pages[PAGE_NVDIMM].get_selected_namespaces()
        dialog = NVDIMMDialog(self.data, namespaces)
        self._run_dialog_and_refresh(dialog)

    def _run_dialog_and_refresh(self, dialog):
        # Run the dialog.
        with self.main_window.enlightbox(dialog.window):
            dialog.refresh()
            dialog.run()

        # We now need to refresh so any new disks picked up by adding advanced
        # storage are displayed in the UI.
        self.refresh()

    @timed_action(delay=1200, busy_cursor=False)
    def on_filter_changed(self, *args):
        self._refilter_current_page()

    def on_search_type_changed(self, combo):
        self._set_notebook_page("searchTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_multipath_type_changed(self, combo):
        self._set_notebook_page("multipathTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_other_type_combo_changed(self, combo):
        self._set_notebook_page("otherTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_nvdimm_type_combo_changed(self, combo):
        self._set_notebook_page("nvdimmTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def on_z_type_combo_changed(self, combo):
        self._set_notebook_page("zTypeNotebook", combo.get_active())
        self._refilter_current_page()

    def _set_notebook_page(self, notebook_name, page_index):
        notebook = self.builder.get_object(notebook_name)
        notebook.set_current_page(page_index)
        self._refilter_current_page()

    def _refilter_current_page(self):
        index = self._notebook.get_current_page()
        page = self._pages[index]
        page.model.refilter()


import numpy as np

# Optimized for performance
# Note: not multi-thread safe
class DELAY_QUEUE(object):
	def __init__(self, delay: int, reserve: int = 2):
		super(DELAY_QUEUE, self).__init__()
		self.delay = delay if delay > 0 else 1
		self.n_transmission = 0
		# use the reserve trick to accelerate execution
		self.delay_channel = np.zeros(reserve, dtype=int)
		self.packet_length = np.zeros(reserve, dtype=int)
		self.packet_data = -np.ones(reserve, dtype=int)

	# Note: the model assumes that packets have no overlapping
	# if the previous transmitted packet longer than the gap between two transmissions
	# un-expectable behavior may happen
	# time spent is maintained by countdown
	def push(self, length: int, packet_data: int):
		# lack of space, enlarge
		if self.delay_channel.size <= self.n_transmission + 1:
			# double the size each time
			new_delay_channel = np.zeros(2 * self.delay_channel.size, dtype=int)
			new_packet_length = np.zeros(2 * self.packet_length.size, dtype=int)
			new_packet_data = -np.ones(2 * self.packet_data.size, dtype=int)
			# copy and push a new item
			new_delay_channel[1:self.delay_channel.size + 1] = self.delay_channel
			new_packet_length[1:self.packet_length.size + 1] = self.packet_length
			new_packet_data[1:self.packet_data.size + 1] = self.packet_data
			new_delay_channel[0] = self.delay
			new_packet_length[0] = length
			new_packet_data[0] = packet_data

			self.delay_channel = new_delay_channel
			self.packet_length = new_packet_length
			self.packet_data = new_packet_data
		else:
			# push an transmission to the queue
			self.delay_channel[1:self.n_transmission + 1] = self.delay_channel[:self.n_transmission]
			self.packet_length[1:self.n_transmission + 1] = self.packet_length[:self.n_transmission]
			self.packet_data[1:self.n_transmission + 1] = self.packet_data[:self.n_transmission]
			self.delay_channel[0] = self.delay
			self.packet_length[0] = length
			self.packet_data[0] = packet_data
		self.n_transmission += 1

	def top(self):
		if self.n_transmission > 0:
			return self.delay_channel[self.n_transmission - 1], self.packet_length[self.n_transmission - 1], self.packet_data[self.n_transmission - 1]
		else:
			return 0xffffffff, -1, -1

	def pop(self):
		# check the oldest packet in the queue
		time_remain, top_packet_len, packet_data = self.top()
		# if arrive, return 1
		if time_remain <= 0:
			# the last time slot of transmission, erase
			if time_remain + top_packet_len <= 1:
				# lazy erase
				self.n_transmission -= 1
				# only the finished can return data
				return -1, packet_data
			return 1, -1
		# if nothing arrive or empty queue, return 0
		return 0, -1

	# API for mobility update
	def update_delay(self, new_delay: int):
		self.delay = new_delay if new_delay > 0 else 1

	# API between CHANNEL and DELAY_QUEUE
	# Input: length-the transmission packet length at current sub time slot
	# 		 if no transmission, set length = 0
	# Output: if packet arrive at this time slot, return 1,
	#         if it is the finishing of the packet, return -1, otherwise 0
	# Note: use the cumulative absolute value of the return to get packet size
	#       use the -1 to indicate finish
	# Note: becuase the step behavior is push first, then time slot + 1, then pop
	#       therefore, the minimum of delay is 1,
	#       if 0 was set, cannot distinguish the difference between packet length 1 and 2
	# Note: the model can be treated as pushing at the begining edge of the time slot
	#       poping at the finishing edge of the time slot
	def step(self, length: int = 1, packet_data: int = -1):
		# only packet length larger than 0 means a transmit
		if length > 0:
			self.push(length, packet_data)
		self.delay_channel -= 1
		return self.pop()

# collection of all-nodes full-duplex channels
class CHANNEL(object):
	def __init__(self, n_nodes: int, delay: np.array):
		super(CHANNEL, self).__init__()
		# number of nodes of the system
		self.n_nodes = n_nodes
		# delay is an integer vector with n_nodes length
		# delay * sub time slot length * propagation speed = distance
		self.delay = delay
		# the memory used to track transmitting packets
		self.sink_receiving_counter = np.zeros(self.n_nodes, dtype=int)
		self.sink_throughput_trace = np.zeros(self.n_nodes, dtype=int)
		# because src->sink delay channel can exist more than one packet
		# use the delay queue to implement it
		# object dtype numpy array is less effieicnt than python list
		# used for action arrival
		self.src2sink_delay_channels = [DELAY_QUEUE(self.delay[i]) for i in range(self.n_nodes)]
		# used for observation update
		self.sink2src_delay_channels = [DELAY_QUEUE(self.delay[i]) for i in range(self.n_nodes)]
		# self.sink2src_delay_channels = [DELAY_QUEUE(1) for i in range(self.n_nodes)] # Xuan's configuration
		# because sink should 1 time slot slower than the source
		# use it to store previous time slot observation
		self.previous_obs = -1

	# TODO: add the translation that returns more comprehensive info
	def __str__(self):
		return str(self.sink_throughput_trace)

	def get_trace(self):
		return self.sink_throughput_trace

	# API for mobility update
	# It is inefficient becuase the organization of delay queues
	def update_delay(self, new_delays: np.array):
		for i in range(self.n_nodes):
			self.src2sink_delay_channels[i].update_delay(new_delays[i])
			self.sink2src_delay_channels[i].update_delay(new_delays[i])

	# Each call of this function will be a new sub time slot
	# Input: actions-the vector denotes actions of all source nodes in the network, 0/1 for trans/not
	# Return: Observation-the broadcast contents recieved for all source nodes
	#         Because different nodes has different delay, their observations are different
	# Note: The rewards is based on the observation, it is the responsibility of nodes
	def step(self, actions: np.array, packet_length: int = 1):
		uplink_channel_output = np.array([self.src2sink_delay_channels[i].step(actions[i] * packet_length)[0] for i in range(self.n_nodes)], dtype=int)
		# in a time slot, the observation to send is the same
		# but delay are different, so different nodes can see different obs
		Observations = np.array([i.step(1, self.previous_obs)[1] for i in self.sink2src_delay_channels], dtype=int)
		uplink_channel_throughput = np.abs(uplink_channel_output)
		self.sink_receiving_counter += uplink_channel_throughput
		obs = -1
		# from the sink's perspective, success includes nodes ID info
		success_trace = np.zeros(self.n_nodes, dtype=int)
		# for uplink, check sink state
		# Observations format: [collided, idle, successful]
		if uplink_channel_throughput.sum() == 1:
			# success
			if -1 in uplink_channel_output:
				# when a whole packet finished, the packet size is the effective payload
				# currently, the trace only the cumulative result, it can be extended to step trace
				self.sink_throughput_trace[uplink_channel_output == -1] += self.sink_receiving_counter[uplink_channel_output == -1]
				self.sink_receiving_counter[uplink_channel_output == -1] = 0
				success_trace[uplink_channel_output == -1] = 1
				# only a success whole transmission lead to success observation
				obs = 2
			else:
				obs = 1
		elif uplink_channel_throughput.sum() == 0:
			# idle
			obs = 1
		else:
			# collision
			self.sink_receiving_counter[uplink_channel_throughput == 1] = 0
			obs = 0
		self.previous_obs = obs
		return Observations, success_trace

	# intelligent sink API
	# Each call of this function will be a new sub time slot
	# Input: actions is all source nodes action
	#        broadcast is the source node ID used to broadcast
	# Return: Observation-output of uplinks (target, always only one +-1)
	#         src_instruction-source nodes listened contents
	# Note: The rewards is based on the observation, agents calculate by themselves
	def cycle(self, actions: np.array, broadcast: np.array, packet_length: int = 1):
		Observations = np.array([self.src2sink_delay_channels[i].step(actions[i] * packet_length)[0] for i in range(self.n_nodes)], dtype=int)
		abs_obs = np.abs(Observations)
		src_instruction = np.array([i.step(1, broadcast)[1] for i in self.sink2src_delay_channels], dtype=int)
		success_trace = np.zeros(self.n_nodes, dtype=int)
		self.sink_receiving_counter += abs_obs
		if -1 in Observations and abs_obs.sum() == 1:
			self.sink_throughput_trace[Observations == -1] += self.sink_receiving_counter[Observations == -1]
			self.sink_receiving_counter[Observations == -1] = 0
			success_trace[Observations == -1] = 1
		elif abs_obs.sum() > 1:
			self.sink_receiving_counter[abs_obs == 1] = 0
		return Observations, src_instruction, success_trace

# Normal nodes decision making simulation
class ENVIRONMENT(object):
	def __init__(self,
				 n_agents: int,
				 n_others: int,
				 nodes_mask: np.array,
				 packet_length: int = 1,
				 guard_length: int = 1,
				 sub_slot_length: int = 1e7,
				 frame_length: int = 10,
				 tdma_occupancy: int = 2,
				 aloha_prob: float = 0.2,
				 window_size: int = 2,
				 max_backoff: int = 2,
				 env_mode: int = 0,
				 mac_mode: int = 0,
				 sink_mode: int = 0,
				 nodes_delay: np.array = None,
				 num_sub_slot: int = 1,
				 movable: bool = False,
				 move_freq: float = 0,
				 save_trace: bool = False,
				 n_iter: int = 1000,
				 log_name: str = '',
				 config_name: str = '',
				 ):
		super(ENVIRONMENT, self).__init__()
		# Most important parameters
		# Ruoyu: agents are user-defined MAC nodes
		# if n_agents == 0, it is the pure conventional network simulation
		self.n_agents = n_agents
		self.n_others = n_others
		self.n_nodes = self.n_agents + self.n_others
		assert self.n_nodes < 1024
		# Ruoyu: Support for hybrid network
		# 0-agent, 1-TDMA, 2-qALOHA, 3-FW_ALOHA, 4-EB_ALOHA
		self.nodes_mask = nodes_mask
		assert self.nodes_mask.size == self.n_nodes

		# Ruoyu: simulation modes selection
		# env_mode 0: RF, env_mode others: UAN
		self.env_mode = env_mode
		# mac_mode 0: sync, mac_mode others: async
		self.mac_mode = mac_mode
		# sink_mode 0: src-agent, sink_mode other: sink-agent
		self.sink_mode = sink_mode

		self.packet_length = 1 if self.env_mode == 0 else packet_length
		self.guard_length = 0 if self.env_mode == 0 else guard_length
		self.sending_counter = np.zeros(self.n_nodes, dtype=int)
		# sub time slot length, unit in nano second (10^-9 s)
		# default 0.01 s
		self.sub_slot_length = sub_slot_length

		# conventional nodes decision making parameters
		self.node_actions = np.zeros(self.n_nodes, dtype=int)
		self.frame_length = frame_length
		self.tdma_occupancy = tdma_occupancy
		self.tdma = np.zeros(self.frame_length, dtype=int)
		self.tdma[np.random.choice(self.frame_length, self.tdma_occupancy)] = 1
		self.tdma_counter = 0
		self.aloha_prob = aloha_prob
		self.window_size = window_size
		self.max_backoff = max_backoff
		self.eb_collision_count = np.zeros(self.n_nodes, dtype=int)
		self.window = np.random.randint(0, self.window_size * 2**self.eb_collision_count, dtype=int)

		# based on simulation mode, use different step function to ensure the APIs are the same
		self.step = self.source_agent if self.sink_mode == 0 else self.sink_agent
		# delay is the propagation delay measured by number of sub time slots
		self.nodes_delay = np.ones(self.n_nodes, dtype=int) if self.env_mode == 0 else nodes_delay
		assert self.nodes_delay.size == self.n_nodes
		self.num_sub_slot = 1 if self.env_mode == 0 else num_sub_slot
		self.channel = CHANNEL(self.n_nodes, self.nodes_delay)
		self.n_padding = 2 * self.nodes_delay.max()

		self.previous_action = np.zeros(self.num_sub_slot, dtype=int)

		# mobile simulator need to be initlized outside
		self.movable = movable
		if self.movable:
			self.move_freq = move_freq

		# Ruoyu: Track the system throughput is the environment responsibility
		# it has large overhead of memory, but we need that data
		self.save_trace = save_trace
		self.trace_counter = 0
		if self.save_trace:
			self.n_iter = n_iter
			self.log_name = log_name
			self.config_name = config_name
			# the addtional rows are for the drain reward
			self.transmission_logs = np.zeros((self.n_iter + 1, self.n_nodes), dtype=int)
		else:
			self.transmission_logs = np.zeros(self.n_nodes, dtype=int)

	# attach the mobile simulator
	def attach_spatial(self, spatial):
		self.spatial = spatial
		self.move_counter = 0
		new_distribution = self.spatial.get_distance()
		new_delay = self.distance2delay(new_distribution)
		self.update_delay(new_delay)

	def reset(self):
		self.node_actions = np.zeros(self.n_nodes, dtype=int)
		self.tdma = np.zeros(self.frame_length, dtype=int)
		self.tdma[np.random.choice(self.frame_length, self.tdma_occupancy)] = 1
		self.tdma_counter = 0
		self.eb_collision_count = np.zeros(self.n_nodes, dtype=int)
		self.window = np.random.randint(0, self.window_size * 2**self.eb_collision_count, dtype=int)
		self.channel = CHANNEL(self.n_nodes, self.nodes_delay, self.num_sub_slot)
		self.previous_action = np.zeros(self.num_sub_slot, dtype=int)
		self.trace_counter = 0
		if self.movable:
			self.move_counter = 0

	# conventional nodes decision making
	# granularity is time slot, call it at the beginning of a time slot
	# action representation is consistent between sync and async mode
	# 0: no transmission, 1: transmit at the first sub time slot, so on so forth
	def get_actions(self):
		# TDMA, always transmit at the beginning of the time slot
		self.node_actions[self.nodes_mask == 1] = self.tdma[self.tdma_counter]

		# q-ALOHA, sync mode transmits at the beginning, async will random choose transmit start time
		self.node_actions[self.nodes_mask == 2] = (np.random.uniform(0, 1, self.n_nodes) < self.aloha_prob)[self.nodes_mask == 2]
		if self.mac_mode:
			self.node_actions[self.nodes_mask == 2] *= np.random.randint(1, self.num_sub_slot + 1 - self.guard_length, self.n_nodes)[self.nodes_mask == 2]

		# FW/EB-ALOHA
		self.node_actions[self.nodes_mask == 3] = (self.window[self.nodes_mask == 3] == 0)
		self.node_actions[self.nodes_mask == 4] = (self.window[self.nodes_mask == 4] == 0)

	# call the function at the end of a time slot
	# this function will update all internal counters
	def update_counters(self):
		# update tdma period counter
		self.tdma_counter = (self.tdma_counter + 1) % self.frame_length
		self.window -= 1
		self.eb_collision_count = np.minimum(self.eb_collision_count, self.max_backoff)
		self.window[self.window < 0] = np.random.randint(0, self.window_size * 2**self.eb_collision_count)[self.window < 0]

	# post process observations
	# if the observation has more complex architecture
	# add the decoding part here
	def decode_obs(self, Observations):
		# treat no data as idle
		Observations[Observations == -1] = 1
		# shift [0, 1, 2] to [-1, 0, 1]
		Observations -= 1
		return Observations

	def distance2delay(self, distance):
		# RF propagation delay can be neglect
		if self.env_mode == 0:
			return np.ones(self.n_nodes, dtype=int)
		else:
			# distance should in unit of meter
			# UAN propagation speed 1500m/s
			# ceil rather than floor
			return -(((distance / 1500) * 1e9) // -self.sub_slot_length).astype(int)

	# for spatial initialization based on the current delay
	def delay2distance(self):
		# only the UAN needs to concern this concept
		return 1500 * (self.nodes_delay.astype(float) * self.sub_slot_length * 1e-9)

	def update_delay(self, delay):
		self.nodes_delay = delay
		self.channel.update_delay(self.nodes_delay)

	# src-agent ENV API
	# there is no more than 1 agents, the action is an integer
	# this function is compatible to non-agent simulation
	def source_agent(self, action=0):
		# get the action of this time slot
		self.get_actions()
		self.node_actions[self.nodes_mask == 0] = action
		Observations = np.zeros((self.num_sub_slot, self.n_nodes), dtype=int)
		Success_trace = np.zeros((self.num_sub_slot, self.n_nodes), dtype=int)
		for i in range(self.num_sub_slot):
			sub_slot_action = np.zeros(self.n_nodes, dtype=int)
			sub_slot_action[self.node_actions == 1] = 1
			# eliminate overlapping sending case
			sub_slot_action[self.sending_counter != 0] = 0
			self.sending_counter[sub_slot_action == 1] = self.packet_length
			# for async mode, when action == 1, it is time to go
			self.node_actions -= 1
			self.sending_counter -= 1
			self.sending_counter[self.sending_counter < 0] = 0
			# Note: obs may contain -1, represent noting
			Observations[i, :], Success_trace[i, :] = self.channel.step(sub_slot_action, self.packet_length)

			# mobility update
			if self.movable and self.move_freq != 0:
				self.move_counter += 1
				if (1 / self.move_counter) <= self.move_freq:
					self.spatial.step()
					new_distribution = self.spatial.get_distance()
					new_delay = self.distance2delay(new_distribution)
					self.update_delay(new_delay)
					self.move_counter = 0

		# how to use depends on the agent
		Observations = self.decode_obs(Observations)

		# update internal counters
		self.eb_collision_count[(Observations < 0).sum(0) > 0] += 1
		self.update_counters()

		if self.save_trace:
			self.transmission_logs[self.trace_counter, :] = Success_trace.sum(0)
		else:
			self.transmission_logs += Success_trace.sum(0)
		self.trace_counter += 1

		return Observations[:, self.nodes_mask == 0]

	# sink-agent ENV API
	# main part in the CHANNEL (sink)
	# ENVIRONMENT just provides APIs for agent
	# dimensions: action: # sub * 1
	#             obs: # sub * n_nodes
	#             rewards: 1 * n_nodes
	def sink_agent(self, broadcast):
		Observations = np.zeros((self.num_sub_slot, self.n_nodes), dtype=int)
		Success_trace = np.zeros((self.num_sub_slot, self.n_nodes), dtype=int)
		nodes_idx = np.array([i for i in range(self.n_nodes)], dtype=int)
		for i in range(self.num_sub_slot):
			sub_slot_action = np.zeros(self.n_nodes, dtype=int)
			sub_slot_action[nodes_idx == self.previous_action] = 1
			# eliminate overlapping sending case
			sub_slot_action[self.sending_counter != 0] = 0
			self.sending_counter[sub_slot_action == 1] = self.packet_length
			self.sending_counter -= 1
			self.sending_counter[self.sending_counter < 0] = 0
			Observations[i, :], self.previous_action, Success_trace[i, :] = self.channel.cycle(sub_slot_action, broadcast[i], self.packet_length)

		if self.save_trace:
			self.transmission_logs[self.trace_counter, :] = Success_trace.sum(0)
		else:
			self.transmission_logs += Success_trace.sum(0)
		self.trace_counter += 1

		return Observations

	# drain the delay queue
	# store logs and configs
	# return statistic throughput
	def finalize(self):
		zero_padding_actions = np.zeros(self.n_nodes, dtype=int)
		Success_trace = np.zeros((self.n_padding, self.n_nodes), dtype=int)
		nodes_idx = np.array([i for i in range(self.n_nodes)], dtype=int)
		for i in range(self.n_padding):
			if self.sink_mode == 0:
				_, Success_trace[i, :] = self.channel.step(zero_padding_actions, self.packet_length)
			else:
				zero_padding_actions[nodes_idx == self.previous_action] = 1
				zero_padding_actions[nodes_idx != self.previous_action] = 0
				_, self.previous_action, Success_trace[i, :] = self.channel.cycle(zero_padding_actions, 1023, self.packet_length)
			# mobility update
			if self.movable and self.move_freq != 0:
				self.move_counter += 1
				if (1 / self.move_counter) <= self.move_freq:
					self.spatial.step()
					new_distribution = self.spatial.get_distance()
					new_delay = self.distance2delay(new_distribution)
					self.update_delay(new_delay)
					self.move_counter = 0

		if self.movable and self.move_freq != 0:
			self.spatial.finalize()

		if self.save_trace:
			self.transmission_logs[self.trace_counter, :] = Success_trace.sum(0)
			np.savetxt(self.log_name, self.transmission_logs[:self.trace_counter + 1, :], fmt='%d')
			with open(self.config_name, 'a') as f:
				config_list = ['n_agents', 'n_others', 'nodes_mask', 'packet_length', 'guard_length', 'frame_length', 'tdma_occupancy',
							   'tdma', 'aloha_prob', 'window_size', 'max_backoff', 'env_mode', 'mac_mode', 'sink_mode',
							   'nodes_delay', 'num_sub_slot', 'movable', 'move_freq']
				f.write('\n======= ENVIRONMENT =======\n')
				f.write('\n'.join([f'{config_key}: {self.__dict__[config_key]}' for config_key in config_list if config_key in self.__dict__]))
				f.write('\n')
				f.close()
		else:
			self.transmission_logs += Success_trace.sum(0)
			print(f'{self.n_agents + self.n_others}-nodes system throughput: {(self.transmission_logs / self.trace_counter + 1).sum()}')
		return self.channel.get_trace()


def damerau_levenshtein_distance(s1, s2):
    # İki boş string arasındaki mesafe 0'dır
    if len(s1) == 0:
        return len(s2)
    if len(s2) == 0:
        return len(s1)

    # İki stringin de uzunluğunu alıyoruz
    len_s1 = len(s1)
    len_s2 = len(s2)

    # Mesafeyi hesaplamak için bir matris oluşturuyoruz
    dist = [[0] * (len_s2 + 1) for _ in range(len_s1 + 1)]

    # İlk satırı ve sütunu başlangıç değerleriyle dolduruyoruz
    for i in range(len_s1 + 1):
        dist[i][0] = i
    for j in range(len_s2 + 1):
        dist[0][j] = j

    # Damerau-Levenshtein mesafesi hesaplamak için dinamik programlama yöntemini kullanıyoruz
    for i in range(1, len_s1 + 1):
        for j in range(1, len_s2 + 1):
            cost = 0 if s1[i - 1] == s2[j - 1] else 1
            dist[i][j] = min(dist[i - 1][j] + 1,            # Silme
                             dist[i][j - 1] + 1,            # Ekleme
                             dist[i - 1][j - 1] + cost)     # Değiştirme

            if i > 1 and j > 1 and s1[i - 1] == s2[j - 2] and s1[i - 2] == s2[j - 1]:
                dist[i][j] = min(dist[i][j], dist[i - 2][j - 2] + cost)  # Transpozisyon
    return dist[len_s1][len_s2]
while True:
    ilk=input("First word:")
    ikinci=input("Second word:")

    print("Distance between two words:",damerau_levenshtein_distance(ilk,ikinci))

    if(ilk=="0" or ikinci==0):
        break


from read_image_data import retrieve_image_data
from read_labels import retrieve_labels
import numpy as np
import tensorflow as tf

# Retrieve the training data
training_images = retrieve_image_data("./../data/train-images.idx3-ubyte", range(0, 60000))
training_labels = retrieve_labels("./../data/train-labels.idx1-ubyte", range(0, 60000))

# Retrieve the test data
test_images = retrieve_image_data("./../data/t10k-images.idx3-ubyte", range(0, 10000))
test_labels = retrieve_labels("./../data/t10k-labels.idx1-ubyte", range(0, 10000))

# Declare hyperparameters
learning_rate = .0001
lamb = 0.01
epochs = 10
batch_size = 100

# Declare the training data placeholders
x = tf.compat.v1.placeholder(tf.float32, [None, 784]) # 28 * 28 = 784 pixels
y = tf.compat.v1.placeholder(tf.float32, [None, 10]) # 10 digits

# Reshape the input data to prepare for convolution
x_reshaped = tf.reshape(x, [-1, 28, 28, 1])

def create_conv_layer(input_data, num_input_channels, num_filters, filter_shape, pool_shape, name):
	"""
	Return a new convolutional layer called "name" that is applied to "input_data". Specify depth of
	the input and output with "num_input_channels" and "num_filters". Also specify the shape of the
	convolutional filters and max pooling dimensions with "filter_shape" and "pool_shape"
	"""

	# Set up the shape of the convolutional filters
	conv_shape = [filter_shape[0], filter_shape[1], num_input_channels, num_filters]

	# Initialize weights and bias for the convolutional filters
	W = tf.Variable(tf.random.normal(conv_shape, stddev = 0.03), name = name + '_W')
	b = tf.Variable(tf.random.normal([num_filters]), name = name + '_b')

	# Set up the convolutional layer operation and add the bias
	output_layer = tf.nn.conv2d(input_data, W, [1, 1, 1, 1], padding = "SAME") + b

	# Apply ReLU activation function
	output_layer = tf.nn.relu(output_layer)

	# Perform 2 x 2 max pooling
	ksize_and_strides = [1, pool_shape[0], pool_shape[1], 1] # no overlap so dimensions and strides are equal
	output_layer = tf.nn.max_pool2d(output_layer, ksize = ksize_and_strides, strides = ksize_and_strides, padding = "SAME")

	return output_layer

# Create new convolutional layers
conv_layer1 = create_conv_layer(x_reshaped, 1, 32, [5, 5], [2, 2], "layer1")
conv_layer2 = create_conv_layer(conv_layer1, 32, 64, [5, 5], [2, 2], "layer2")

# Flatten the output of the convolutional layers to a 1d array
x_flattened = tf.reshape(conv_layer2, [-1, 7 * 7 * 64])

# Declare weights connecting last convolutional layer to dense layer
W1 = tf.Variable(tf.random.normal([7 * 7 * 64, 1000], stddev = 0.03), name = "W1")
b1 = tf.Variable(tf.random.normal([1000]), name = "b1")

# Declare weights connecting dense layer to output layer
W2 = tf.Variable(tf.random.normal([1000, 10], stddev = 0.03), name = "W2")
b2 = tf.Variable(tf.random.normal([10]), name = "b2")

# Calculate output of dense layer
dense_output = tf.add(tf.matmul(x_flattened, W1), b1)
dense_output = tf.nn.relu(dense_output)

# Calculate output of the final layer and clip it to avoid values less than zero or greater than one
y_pred = tf.nn.softmax(tf.add(tf.matmul(dense_output, W2), b2))
y_pred = tf.clip_by_value(y_pred, 1e-10, 0.9999999)

# Define cost function using cross entropy function an l2 regularization
cross_entropy = - tf.reduce_mean(tf.reduce_sum(y * tf.math.log(y_pred) + (1 - y) * tf.math.log(1 - y_pred), axis = 1))
l2_regularization = (lamb / 2) * (tf.reduce_mean(tf.reduce_sum(tf.math.square(W1), axis = 1)) + tf.reduce_mean(tf.reduce_sum(tf.math.square(W2), axis = 1)))
cost_function = cross_entropy + l2_regularization

# Create optimizer
optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate = learning_rate).minimize(cost_function)

# Setup the initialization operator
init = tf.compat.v1.global_variables_initializer()

# Define an accuracy assessment function
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_pred, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

def next_batch(batch_size, x, y):
	'''
	Return "batch_size" random samples and labels
	'''

	indices = np.arange(0, len(x))
	np.random.shuffle(indices)
	indices = indices[:batch_size]
	batch_x = [x[index] for index in indices]
	batch_y = [y[index] for index in indices]

	return np.asarray(batch_x), np.asarray(batch_y)

# Start the session
with tf.compat.v1.Session() as sess:
	# Initialize the variables
	sess.run(init)

	# Calculate number of batches to be run
	num_batches = int(len(training_images) / batch_size)

	for epoch in range(epochs):
		avg_cost = 0
		for i in range(num_batches):
			batch_x, batch_y = next_batch(batch_size, training_images, training_labels)
			_, cost = sess.run([optimizer, cost_function], feed_dict = {x : batch_x, y : batch_y})
			avg_cost += (cost / num_batches)
		print("Epoch: %d cost = %1.3f" % (epoch + 1, avg_cost))

	# Run test data through model
	print("Accuracy: %1.4f" % (sess.run(accuracy, feed_dict = {x : test_images, y : test_labels})))


import importlib
import inspect
from typing import Dict

from deeppavlov.core.commands.utils import expand_path, get_deeppavlov_root, set_deeppavlov_root
from deeppavlov.core.common.file import read_json
from deeppavlov.core.common.registry import get_model, cls_from_str
from deeppavlov.core.common.errors import ConfigError
from deeppavlov.core.common.log import get_logger
from deeppavlov.core.models.component import Component

log = get_logger(__name__)

_refs = {}

def _resolve(val):
    if isinstance(val, str) and val.startswith('#'):
        component_id, *attributes = val[1:].split('.')
        try:
            val = _refs[component_id]
        except KeyError:
            e = ConfigError('Component with id "{id}" was referenced but not initialized'
                            .format(id=component_id))
            log.exception(e)
            raise e
        attributes = ['val'] + attributes
        val = eval('.'.join(attributes))
    return val

def _init_param(param, mode):
    if isinstance(param, str):
        param = _resolve(param)
    elif isinstance(param, (list, tuple)):
        param = [_init_param(p, mode) for p in param]
    elif isinstance(param, dict):
        if {'ref', 'name', 'class', 'config_path'}.intersection(param.keys()):
            param = from_params(param, mode=mode)
        else:
            param = {k: _init_param(v, mode) for k, v in param.items()}
    return param

def from_params(params: Dict, mode: str = 'infer', **kwargs) -> Component:
    """Builds and returns the Component from corresponding dictionary of parameters."""
    # what is passed in json:
    config_params = {k: _resolve(v) for k, v in params.items()}

    # get component by reference (if any)
    if 'ref' in config_params:
        try:
            return _refs[config_params['ref']]
        except KeyError:
            e = ConfigError('Component with id "{id}" was referenced but not initialized'
                            .format(id=config_params['ref']))
            log.exception(e)
            raise e

    elif 'config_path' in config_params:
        from deeppavlov.core.commands.infer import build_model_from_config
        deeppavlov_root = get_deeppavlov_root()
        refs = _refs.copy()
        _refs.clear()
        config = read_json(expand_path(config_params['config_path']))
        model = build_model_from_config(config, as_component=True)
        set_deeppavlov_root({'deeppavlov_root': deeppavlov_root})
        _refs.clear()
        _refs.update(refs)
        return model

    elif 'class' in config_params:
        cls = cls_from_str(config_params.pop('class'))
    else:
        cls_name = config_params.pop('name', None)
        if not cls_name:
            e = ConfigError('Component config has no `name` nor `ref` or `class` fields')
            log.exception(e)
            raise e
        cls = get_model(cls_name)

    # find the submodels params recursively
    config_params = {k: _init_param(v, mode) for k, v in config_params.items()}

    try:
        spec = inspect.getfullargspec(cls)
        if 'mode' in spec.args+spec.kwonlyargs or spec.varkw is not None:
            kwargs['mode'] = mode

        component = cls(**dict(config_params, **kwargs))
        try:
            _refs[config_params['id']] = component
        except KeyError:
            pass
    except Exception:
        log.exception("Exception in {}".format(cls))
        raise

    return component


import asyncio
import json
import logging
import os
import ssl
from asyncio import StreamReader, StreamWriter
from asyncio.exceptions import IncompleteReadError
from typing import List

from . import forwarders, utils

_logger = logging.getLogger()

RequiredFields = ("level", "pid", "message", "created_at", "created_by")

class LogTokenFileError(Exception):
    pass

class LogServer:
    """Logging server which can accept logs from the JSONSocketHandler. Received
    logs are passed to the standard python log. This allows to pass the logs further
    with other logging handlers."""

    def __init__(
        self,
        host: str,
        port: int,
        forwarder: forwarders.Forwarder,
        ssl_context: ssl.SSLContext = None,
        token_file: str = None,
        use_auth: bool = True,
    ):
        self.host = host
        self.port = port
        self.ssl_context = ssl_context
        self.use_auth = use_auth
        self.tokens = {}
        self.token_file = token_file
        self.token_mtime = None
        self.forwarder = forwarder

    def add_token(self, token: str, **kwargs) -> None:
        """Add a token and store additional information about the client"""
        if self.token_file:
            raise LogTokenFileError("Token file is used")

        self.tokens[token] = kwargs

    def delete_token(self, token: str) -> dict:
        """Delete a token from the storage. It's not possible if a token_file is used"""
        if self.token_file:
            raise LogTokenFileError("Token file is used")

        return self.tokens.pop(token, None)

    def _update_tokens(self) -> None:
        """Update the token store if the file changed"""
        if not self.token_file:
            return

        stat = os.stat(self.token_file)
        if self.token_mtime != stat.st_mtime:
            with open(self.token_file) as fp:
                self.tokens = json.load(fp)

    def auth_client(self, auth: dict) -> dict:
        """Evaluate the auth message and return the fitting client"""
        if not isinstance(auth, dict):
            return None

        token = auth.get("token")
        if not token:
            return None

        self._update_tokens()
        client = self.tokens.get(token)
        if client is None:
            return None

        client["name"] = client.get("name", token)
        return client

    async def _read_message(self, reader: StreamReader) -> dict:
        """Read a message from the reader and evaluate it"""
        try:
            (length,) = await utils.receive_struct(reader, ">L")
            if length <= 0:
                return None

            message = json.loads(await reader.readexactly(length))
            return message
        except (json.JSONDecodeError, IncompleteReadError):
            return None

    def _validate_message(self, message: dict, required: List[str]) -> dict:
        """Validate a message against a list of required keys"""
        if not isinstance(message, dict):
            return None

        return message if set(message).issuperset(required) else None

    async def _process_message(self, message: dict, client_name: str = None) -> None:
        """Forward the log to the next server or database"""
        if not message.get("host") and client_name:
            message["host"] = client_name

        _logger.debug(f"Forwarding: {message}")
        await self.forwarder.put(message)

    async def _stop(self, reader: StreamReader, writer: StreamWriter) -> None:
        """Stop the reader and writer"""
        reader.feed_eof()
        writer.close()
        await writer.wait_closed()

    async def _accept(self, reader: StreamReader, writer: StreamWriter) -> None:
        """Accept new clients and wait for logs to process them"""
        if self.use_auth:
            message = await self._read_message(reader)
            message = self._validate_message(message, ["token"])
            client = self.auth_client(message)

            if not client:
                await self._stop(reader, writer)
                return

            name = client["name"]
            _logger.info(f"Client '{name}' connected")
        else:
            client = {}
            name = None

        while True:
            message = await self._read_message(reader)
            data = self._validate_message(message, RequiredFields)
            if not isinstance(data, dict):
                break

            await self._process_message(data, name)

        await self._stop(reader, writer)

    async def run(self) -> None:
        """Start the server and listen for logs"""
        if self.forwarder:
            _logger.info(f"Starting forwarder to {self.forwarder}")
            asyncio.create_task(self.forwarder.process())

        _logger.info(f"Starting log server on {self.host}:{self.port}")
        self.sock = await asyncio.start_server(
            self._accept,
            self.host,
            self.port,
            ssl=self.ssl_context,
        )

        async with self.sock:
            await self.sock.serve_forever()

    def start(self) -> None:
        """Start the log server as asyncio task"""
        asyncio.run(self.run())

    async def stop(self) -> None:
        """Stop the LogServer and close the socket"""
        self.sock.close()
        await self.sock.wait_closed()


import socket
from btcp.lossy_layer import LossyLayer
from btcp.btcp_socket import BTCPSocket
from btcp.constants import *
import random
from queue import Queue, Empty

#flags[0] = ACK     1
#flags[1] = SYN     2
#flags[2] = FIN     4
#ACK, SYN           3
#ACK, FIN           5
#SYN, FIN           6
#ACK, SYN, FIN      8

# The bTCP server socket
# A server application makes use of the services provided by bTCP by calling accept, recv, and close
class BTCPServerSocket(BTCPSocket):
    def __init__(self, window, timeout):
        super().__init__(window, timeout)
        self._lossy_layer = LossyLayer(self, SERVER_IP, SERVER_PORT, CLIENT_IP, CLIENT_PORT)
        self._queue = Queue()
        self._sq = 0#last sequence number sent
        self._recv = 0#last sequence number received
        self._lastack = 0#last acknowledgement number received
        self._clientwindow = 0
        self._padding = bytearray(bytes(1008))
        self._message = bytearray(bytes(0))

    # Wait for the client to initiate a three-way handshake
    def accept(self):
        segment = self._queue.get()
        self._recv = int.from_bytes(segment[:2], "big")
        self._sq = random.randint(0, 65535)
        packet = BTCPSocket.create_packet(self, self._sq, self._recv + 1, 3, self._window, 0, 0, self._padding)
        LossyLayer.send_segment(self._lossy_layer, packet)
        self._lastack = self._recv +1
        segment = self._queue.get()
        x = int.from_bytes(segment[:2], "big")
        y = int.from_bytes(segment[2:4], "big")
        self._clientwindow = int.from_bytes(segment[5:6], "big")
        if self._lastack == x and self._sq + 1 == y:
            self._lastack = y
            self._recv = x
        #a client has successfully connected

    # Send any incoming data to the application layer
    def recv(self):
        #so I guess in here we need to check for the FIN flag to close the connection
        try:
            segment = self._queue.get()
            sq = int.from_bytes(segment[:2], "big")
            self._lastack = int.from_bytes(segment[2:4], "big")
            flags = int.from_bytes(segment[4:5], "big")
            if flags >= 4:  # so if FIN flag is set (maybe also other flags)
                self._sq += 1
                packet = BTCPSocket.create_packet(self, self._sq, sq + 1, 5, self._window, 0, 0,self._padding)
                LossyLayer.send_segment(self._lossy_layer, packet)
                return False
            else:  # every normal packet that does not finish the connection
                length = int.from_bytes(segment[6:8], "big")
                self._message += segment[10:(10+length)]
                self._sq += 1
                packet = BTCPSocket.create_packet(self, self._sq, sq +1, flags, self._window, 0, 0, self._padding)
                LossyLayer.send_segment(self._lossy_layer, packet)
                return True
        except Empty:
            pass

    # Clean up any state
    def close(self):
        self._lossy_layer.destroy()

    # Called by the lossy layer from another thread whenever a segment arrives
    def lossy_layer_input(self, segment, addr):
        # received the clients request and sends its response back
        self._queue.put(segment)


'''
Created on May 24, 2019

@author: hzhang0418
'''

import os
import sys
import random

import pandas as pd

import v6.data_io
import v6.label_debugger
import v6.feature_selection

import utils.myconfig

def run(detector='both', confusion=False, top_k=20):
    config_file=r'/scratch/hzhang0418/projects/datasets/mono/beer.config'
    #config_file=r'/scratch/hzhang0418/projects/datasets/mono/bike.config'
    #config_file=r'/scratch/hzhang0418/projects/datasets/mono/books1.config'
    #config_file=r'/scratch/hzhang0418/projects/datasets/mono/citations.config'
    #config_file=r'/scratch/hzhang0418/projects/datasets/mono/movies1.config'
    #config_file=r'/scratch/hzhang0418/projects/datasets/mono/restaurants4.config'

    #config_file = r'/scratch/hzhang0418/projects/datasets/labeldebugger/fodors_zagats.config'

    #config_file=r'/scratch/hzhang0418/projects/datasets/mono/cora_large.config'
    #config_file = r'/scratch/hzhang0418/projects/datasets/labeldebugger/cora_new.config'

    #mayuresh
    config_file=r'/export/da/mkunjir/LabelDebugger/config/cora.config'

    params = utils.myconfig.read_config(config_file)

    basedir = params['basedir']
    hpath = os.path.join(basedir, params['hpath'])
    gpath = os.path.join(basedir, 'golden.csv')

    exclude_attrs = ['_id', 'ltable.id', 'rtable.id']

    features, labels, pair2index, index2pair = v6.data_io.read_feature_file(hpath, exclude_attrs)
    pair2golden = v6.data_io.read_golden_label_file(gpath) if os.path.exists(gpath) else {p:labels[i] for p, i in pair2index.items()} ## mayuresh: if no golden labels, then only new errors will be injected

    params['fs_alg'] = 'xgboost'
    params['max_list_len'] = 40
    params['detectors'] = detector
    params['confusion'] = confusion

    params['num_cores'] = 4
    params['num_folds'] = 5

    params['min_con_dim'] = 1
    params['counting_only'] = False
    #debugger = v6.label_debugger.LabelDebugger(features, labels, params)

    # label errors
    all_errors = []
    false_nonmatch_errors = 0
    false_nonmatch_indices = []
    for index, p in index2pair.items():
        if labels[index]!=pair2golden[p]:
            #print('Error found at: ', index, ' the pair is: ', p)
            all_errors.append(index)
            if labels[index]==0:
               false_nonmatch_errors += 1
               false_nonmatch_indices.append(index)

    '''
    # TEMP: mayuresh: eliminate errors from labeled data
    #mpath = os.path.join(basedir, 'labeled_data.csv')
    new_hpath = os.path.join(basedir, 'feature_vector_newerrors.csv')
    new_gpath = os.path.join(basedir, 'golden_newerrors.csv')
    fv_df = pd.read_csv(hpath)
    g_df = pd.read_csv(gpath)
    #fv_df.drop(fv_df.index[all_errors], inplace=True)
    #fv_df.to_csv(new_hpath, index=False)
    #exit()
    '''

    # randomly insert errors
    seed = 0 #random.randrange(sys.maxsize)
    rng = random.Random(seed)
    print("Seed was:", seed)

    perc = 0.1 #rng.randint(5, 15)/100.0
    print("Error rate:", perc)

    num_err = int(len(labels)*perc - len(all_errors)) # accounting for existing errors
    #num_err = min(800, num_err) # 800 is the maximum errors we can check in the given budget

    while num_err < 0: ## correct some existing errors to make them equal to the required number
      i = 0#rng.randint(0, len(all_errors)-1)
      index = all_errors[i]
      labels[index] = 1 - labels[index]
      if labels[index]==1:
         false_nonmatch_errors -= 1
         false_nonmatch_indices.remove(index)
      all_errors.remove(index)
      num_err += 1

    error_indices = set(all_errors)
    new_error_indices = set()
    for _ in range(num_err*10):
        index = rng.randint(0, len(labels)-1)
        if index in error_indices or index in new_error_indices:
            continue
        new_error_indices.add(index)
        if labels[index]==1:
            false_nonmatch_errors += 1
            false_nonmatch_indices.append(index)
        labels[index] = 0 if labels[index]==1 else 1
        if len(new_error_indices)>=num_err:
            break

    '''
    ## mayuresh: update feature vector with new errors and new golden labels
    print('New errors inserted: ', len(new_error_indices))
    #g_df = g_df.iloc[0:0] # dropping current golden labels
    for cnt, index in enumerate(new_error_indices):
      pair = index2pair[index]
      fv_df.at[index, 'label'] = labels[index]
      #print('Changed fv location: ', index)
      g_df.at[index, 'golden'] = 1-labels[index] # = g_df.append({'_id':cnt, 'ltable.id':pair[0], 'rtable.id':pair[1], 'golden':1-labels[index]}, ignore_index=True)
    g_df.drop(fv_df.index[all_errors], inplace=True) #delete old errors
    g_df.to_csv(new_gpath, index=False)
    fv_df.drop(fv_df.index[all_errors], inplace=True) #delete old errors
    fv_df.to_csv(new_hpath, index=False)
    exit()
    '''

    print("Total number of errors: ", len(error_indices) + len(new_error_indices))
    print("Of which total false nonmatches: ", false_nonmatch_errors)
    all_errors = list(error_indices)
    all_errors.extend(list(new_error_indices))
    false_match_errors = len(all_errors) - false_nonmatch_errors

    print(params['dataset_name'])

    #index = 324
    #print( labels[index], pair2golden[index2pair[index]] )

    ## mayuresh: read selected features if available in file or else store them
    #del params['spath']
    if 'spath' in params:
       selected_features_path = os.path.join(basedir, params['spath'])
       selected_features = pd.read_csv(selected_features_path).to_numpy()
    else:
       selected_features = v6.feature_selection.select_features(features, labels, params['fs_alg'])
       selected_features_path = os.path.join(basedir, 'selected_features.csv')
       pd.DataFrame(selected_features).to_csv(selected_features_path, index=False)
    print('Selected features of dim: ', selected_features.shape, ' from the original features: ', features.shape)

    params['fs_alg'] = 'none' # disabling feature selection now that we are done selecting
    debugger = v6.label_debugger.LabelDebugger(selected_features, labels, params)

    all_detected_errors, detected_false_nonmatches = debug_labels(debugger, index2pair, pair2golden, top_k)

    print("Total number of label errors: ", len(all_errors))
    print("Number of iterations: ", debugger.iter_count)
    print("Number of checked pairs: ", len(debugger.verified_indices))
    print("Number of detected errors: ", len(all_detected_errors))
    print("Of which the number of false nonmatches: ", len(detected_false_nonmatches))
    print("False match detection recall: ", 100.0 * (len(all_detected_errors) - len(detected_false_nonmatches)) / false_match_errors)
    print("False non-match detection recall: ", 100.0 * len(detected_false_nonmatches) / false_nonmatch_errors)
    print("False match detection prec: ", 100.0 * (len(all_detected_errors) - len(detected_false_nonmatches)) / (len(debugger.verified_indices) * false_match_errors / len(all_errors)))
    #print("False nonmatch detection prec: ", 100.0 * (len(detected_false_nonmatches)) / (len(debugger.verified_indices) * false_nonmatch_errors / len(all_errors)))

    apath = os.path.join(basedir, params['apath'])
    bpath = os.path.join(basedir, params['bpath'])

    table_A = pd.read_csv(apath)
    table_B = pd.read_csv(bpath)

    '''
    #show all errors
    table_A['id'] = table_A['id'].astype(str)
    table_B['id'] = table_B['id'].astype(str)
    all_error_pairs = []
    for index in all_errors:
        p = index2pair[index]
        label = labels[index]
        left = table_A.loc[ table_A['id'] == str(p[0])]
        right = table_B.loc[ table_B['id'] == str(p[1])]
        tmp = {}
        for col in left:
            tmp['ltable.'+col] = left.iloc[0][col]
        for col in right:
            tmp['rtable.'+col] = right.iloc[0][col]
            tmp['label'] = label
        all_error_pairs.append(tmp)

    if len(all_error_pairs)>0:
        df = pd.DataFrame(all_error_pairs)
        output_file = params['dataset_name']+'_all_errors.csv'
        df.to_csv(output_file, index=False)

    # show missed errors
    missed_error_pairs = []
    for index in all_errors:
        if index not in all_detected_errors:
            p = index2pair[index]
            label = labels[index]
            left = table_A.loc[ table_A['id'] == int(p[0])]
            right = table_B.loc[ table_B['id'] == int(p[1])]
            tmp = {}
            for col in left:
                tmp['ltable.'+col] = left.iloc[0][col]
            for col in right:
                tmp['rtable.'+col] = right.iloc[0][col]
                tmp['label'] = label
            missed_error_pairs.append(tmp)

    if len(missed_error_pairs)>0:
        df = pd.DataFrame(missed_error_pairs)
        output_file = params['dataset_name']+'missed_errors.csv'
        df.to_csv(output_file, index=False)
    '''

def debug_labels(debugger, index2pair, pair2golden, top_k=20):

    num_iter_without_errors = 0
    all_detected_errors = set()
    all_detected_match_errors = set() # labels mislabeled as matches
    total_num_iters = 0
    while True:
        top_suspicious_indices = debugger.find_suspicious_labels(top_k)

        # find their correct labels
        index2correct_label = { index:pair2golden[ index2pair[index] ]  for index in top_suspicious_indices}
        iter_count, num_errors, error_indices, error_indices_matches, det_error_poses  = debugger.analyze(index2correct_label)
        #print('Iteration: ', iter_count)
        #print("Number of errors found: ", num_errors)
        #print("Error indices: ", error_indices)
        #print("Detector performance: ")
        #for n, (count, pos) in enumerate(det_error_poses):
        #    print("Detector ", n, "found ", count, " errors")
        #    print("Positions: ", pos)

        all_detected_errors.update(error_indices)
        all_detected_match_errors.update(error_indices_matches)

        if num_errors==0:
            num_iter_without_errors += 1
        else:
            num_iter_without_errors = 0

        if num_iter_without_errors>=3:
            break

        debugger.correct_labels(index2correct_label)

        total_num_iters += 1

        if total_num_iters>=5000:
            break

    return all_detected_errors, all_detected_match_errors

run('fpfn', False, 20)
#run('mono', False, 20)
#run('both', False, 20)
#run('both', False, 10)
#run('both', False, 15)
#run('cleanlab', False)
run('cleanlab', True)


'''Module for the detection of the cut plane'''
import json
import logging
import itertools as it
import operator
from pathlib import Path
import warnings
from typing import List, Union

import neurom as nm
import numpy as np

from neurom import geom
from neurom.core import Section, Morphology
from scipy import optimize, special

from neuror.cut_plane import legacy_detection, planes
from neuror.exceptions import NeuroRError

L = logging.getLogger(__name__)

class CutPlane(planes.HalfSpace):
    '''The cut plane class.

    It is composed of a HalfSpace and a morphology
    The morphology is part of the HalfSpace, the cut space is
    the complementary HalfSpace.
    '''

    def __init__(self, coefs: List[float],
                 upward: bool,
                 morphology: Union[None, str, Path, Morphology],
                 bin_width: float):
        '''Cut plane ctor.

        Args:
            coefs: the [abcd] coefficients of a plane equation: ``a X + b Y + c Z + d = 0``
            upward: if ``True``, the morphology points satisfy: ``a X + b Y + c Z + d > 0``
                    else, they satisfy: ``a X + b Y + c Z + d < 0``
            morphology: the morphology
            bin_width: the bin width
        '''
        super().__init__(coefs[0], coefs[1], coefs[2], coefs[3], upward)

        if isinstance(morphology, Morphology):
            self.morphology = morphology
        elif isinstance(morphology, (str, Path)):
            self.morphology = nm.load_morphology(morphology)
        elif morphology is not None:
            raise NeuroRError(f'Unsupported morphology type: {type(morphology)}')

        self.bin_width = bin_width
        self.cut_leaves_coordinates = None
        self.status = None
        self.minus_log_prob = None
        if morphology is not None:
            self._compute_cut_leaves_coordinates()
            self._compute_probabilities()
            self._compute_status()

    @classmethod
    def from_json(cls, cut_plane_obj, morphology=None):
        '''Factory constructor from a JSON file.

        Args:
            cut_plane_obj (dict|str|pathlib.Path): a cut plane
                It can be a python dictionary or a path to a json file that contains one
            morphology (~neurom.core.morphology.Morphology): the morphology passed to the
                :class:`CutPlane` object
        '''
        assert isinstance(cut_plane_obj, (str, dict, Path))

        if not isinstance(cut_plane_obj, dict):
            with open(cut_plane_obj) as f:
                cut_plane_obj = json.load(f)

        data = cut_plane_obj['cut-plane']
        obj = CutPlane([data['a'], data['b'], data['c'], data['d']],
                       data['upward'],
                       morphology,
                       cut_plane_obj['details']['bin-width'])

        obj.cut_leaves_coordinates = np.array(cut_plane_obj['cut-leaves'])
        obj.status = cut_plane_obj['status']
        obj.minus_log_prob = cut_plane_obj['details']['-LogP']
        return obj

    # pylint: disable=arguments-differ
    @classmethod
    def from_rotations_translations(cls, transformations, morphology, bin_width):
        plane = planes.PlaneEquation.from_rotations_translations(transformations)
        return cls(plane.coefs, True, morphology, bin_width)

    @classmethod
    def find(cls, neuron, bin_width=3,
             searched_axes=('X', 'Y', 'Z'),
             searched_half_spaces=(-1, 1),
             fix_position=None):
        """Find and return the cut plane that is oriented along X, Y or Z.

        6 potential positions are considered: for each axis, they correspond
        to the coordinate of the first and last point of the neuron.

        Description of the algorithm:

        1) The distribution of all points along X, Y and Z is computed
           and put into 3 histograms.
        2) For each histogram we look at the first and last empty bins
           (ie. the last bin before the histogram starts rising,
           and the first after it reaches zero again). Under the assumption
           that there is no cut plane, the posteriori probability
           of observing this empty bin given the value of the not-empty
           neighbour bin is then computed.
        3) The lowest probability of the 6 probabilities (2 for each axes)
           corresponds to the cut plane

        Args:
            neuron (~neurom.core.morphology.Morphology|str|pathlib.Path): a morphology
            bin_width: The size of the binning

            display: where or not to display the control plots
                     Note: It is the user responsability to call matplotlib.pyplot.show()

            searched_axes: x, y or z. Specify the planes for which to search the cut plane

            searched_half_spaces: A negative value means the morphology lives
                on the negative side of the plane, and a positive one the opposite.

            fix_position: If not None, this is the position
                for which to search the cut plane. Only the orientation will be searched for.
                This can be useful to find the orientation of a plane whose position is known.

        Returns:
            A cut plane object
        """
        warnings.warn("This method will be deprecated in favor or cut_leaves.find_cut_leaves"
                      "it also has bugs if one uses +1 in searched_half_spaces and multiple"
                      "combinations of searched_arguments.", DeprecationWarning)
        if not isinstance(neuron, Morphology):
            neuron = nm.load_morphology(neuron)

        # pylint: disable=invalid-unary-operand-type
        coef_d = -fix_position if fix_position is not None else 0

        cut_planes = [CutPlane([int(axis.upper() == 'X'), int(axis.upper() == 'Y'),
                                int(axis.upper() == 'Z'), coef_d],
                               upward=(side > 0),
                               morphology=neuron,
                               bin_width=bin_width)
                      for axis, side in it.product(searched_axes, searched_half_spaces)]

        best_cut_plane = max(cut_planes, key=operator.attrgetter('minus_log_prob'))

        if fix_position is None:
            _, bins = best_cut_plane.histogram()
            # The orientation of the plane is defined such as the morphology lives
            # on the positive side once coordinates are projected
            # (see HalfSpace.project_on_directed_normal)
            # So the first bin is where we look for the cut plane
            best_cut_plane.coefs[3] = bins[0]
        else:
            best_cut_plane.coefs[3] = coef_d
        return CutPlane(
            best_cut_plane.coefs, best_cut_plane.upward, neuron, best_cut_plane.bin_width)

    @classmethod
    def find_legacy(cls, neuron, axis):
        '''Find the cut points according to the legacy algorithm

        As implemented in:
        https://bbpgitlab.epfl.ch/nse/morphologyrepair/BlueRepairSDK/-/blob/main/BlueRepairSDK/src/repair.cpp#L263
        '''
        if not isinstance(neuron, Morphology):
            neuron = nm.load_morphology(neuron)

        cut_leaves, side = legacy_detection.internal_cut_detection(neuron, axis)

        plane = cls([int(axis.upper() == 'X'), int(axis.upper() == 'Y'),
                     int(axis.upper() == 'Z'), 0],
                    upward=(side < 0),
                    morphology=None,
                    bin_width=0)
        plane.morphology = nm.load_morphology(neuron)
        plane.cut_leaves_coordinates = cut_leaves
        return plane

    @property
    def cut_sections(self):
        '''Returns sections that ends within the cut plane'''
        leaves = np.array([leaf
                           for neurite in self.morphology.neurites
                           for leaf in nm.iter_sections(neurite, iterator_type=Section.ileaf)])
        leaves_coord = [leaf.points[-1, nm.COLS.XYZ] for leaf in leaves]
        return leaves[self.distance(leaves_coord) < self.bin_width]

    def _compute_cut_leaves_coordinates(self):
        '''Returns cut leaves coordinates'''
        leaves = [leaf.points[-1, nm.COLS.XYZ] for leaf in self.cut_sections]
        self.cut_leaves_coordinates = np.vstack(leaves) if leaves else np.array([])

    def to_json(self):
        '''Return a dictionary with the following items:

            - status: 'ok' if everything went right, else an informative string
            - cut_plane: a tuple (plane, position) where 'plane' is 'X', 'Y' or 'Z'
                       and 'position' is the position
            - cut_leaves: an np.array of all termination points in the cut plane
            - figures: if 'display' option was used, a dict where values are tuples (fig, ax)
                     for each figure
            - details: A dict currently only containing -LogP of the bin where the cut plane was
                     found
        '''
        return {'cut-leaves': self.cut_leaves_coordinates,
                'status': self.status,
                'details': {'-LogP': self.minus_log_prob, 'bin-width': self.bin_width},
                'cut-plane': super().to_json()}

    def histogram(self):
        '''Get the point distribution projected along the normal to the plane

        Returns:
            a numpy.histogram
        '''
        points = _get_points(self.morphology)
        projected_points = self.project_on_directed_normal(points)
        min_, max_ = np.min(projected_points, axis=0), np.max(projected_points, axis=0)
        binning = np.arange(min_, max_ + self.bin_width, self.bin_width)
        return np.histogram(projected_points, bins=binning)

    def _compute_probabilities(self):
        '''Returns -log(p) where p is the a posteriori probabilities of the observed values
        in the bins X min, X max, Y min, Y max, Z min, Z max

        Parameters:
            hist: a dict of the X, Y and Z 1D histograms

        Returns: a dict of -log(p) values'''

        hist = self.histogram()
        if not hist[0].size:
            self.minus_log_prob = np.nan
        else:
            self.minus_log_prob = get_minus_log_p(0, hist[0][0])
        self._compute_status()
        return self.minus_log_prob

    def _compute_status(self):
        '''Returns ok if the probability that there is a cut plane is high enough'''
        _THRESHOLD = 50
        if np.isnan(self.minus_log_prob):
            self.status = 'The proba is NaN, something went wrong'
        elif self.minus_log_prob < _THRESHOLD:
            self.status = ('The probability that there is in fact NO '
                           f'cut plane is high: -log(p) = {self.minus_log_prob}!')
        else:
            self.status = 'ok'

def _success_function(params, points, bin_width):
    '''The success function is low (=good) when the difference of points
    on the left side and right side of the plane is high'''
    plane = planes.PlaneEquation.from_rotations_translations(params)
    n_left, n_right = plane.count_near_plane(points, bin_width)
    res = -abs(n_left - n_right)
    return res

def _minimize(x0, points, bin_width):
    '''Returns a tuple of the optimized values of
    (rot_x, rot_y, rot_z, transl_x, transl_y, transl_z)'''
    delta_angle = 10  # in degrees
    delta_transl = 10
    delta = np.array([delta_angle, delta_angle, delta_angle,
                      delta_transl, delta_transl, delta_transl])
    bounds_min = x0 - delta
    bounds_max = x0 + delta
    bounds = list(zip(bounds_min, bounds_max))
    result = optimize.minimize(_success_function,
                               x0=x0,
                               args=(points, bin_width),
                               bounds=bounds,
                               method='Nelder-Mead')

    if result.status:
        raise NeuroRError(result.message)
    return result.x

def get_minus_log_p(k, mu):
    '''Compute -Log(p|k) where p is the a posteriori probability to observe k counts
    in bin given than the mean value was "mu":
    demo: p(k|mu) = exp(-mu) * mu**k / k!
    '''
    return mu - k * np.log(mu) + np.log(special.factorial(k))

def _get_points(neuron):
    return np.array([point
                     for neurite in (neuron.neurites or [])
                     for section in nm.iter_sections(neurite)
                     for point in section.points])

def plot(neuron, result, inline=False):
    '''Plot the neuron, the cut plane and the cut leaves.

    Args:
        neuron (~neurom.core.morphology.Morphology): the neuron to be plotted
        result (dict): the cut plane object in dictionary form
        inline (bool): if True, plot as an interactive plot (for example in a Jupyter notebook)
    '''
    try:
        from plotly_helper.neuron_viewer import NeuronBuilder
        from plotly_helper.object_creator import scatter
        from plotly_helper.shapes import line
    except ImportError as e:
        raise ImportError(
            'neuror[plotly] is not installed.'
            ' Please install it by doing: pip install neuror[plotly]') from e

    bbox = geom.bounding_box(neuron)

    plane = result['cut-plane']

    for display_plane, idx in [('xz', 0), ('yz', 1), ('3d', None)]:
        builder = NeuronBuilder(neuron, display_plane, inline=inline,
                                line_width=4, title=str(neuron.name))
        if idx is not None:
            if plane['a'] == 0 and plane['b'] == 0:
                builder.helper.add_shapes([
                    line(bbox[0][idx], -plane['d'], bbox[1][idx], -plane['d'], width=4)
                ])
            builder.helper.add_data({'a': scatter(result['cut-leaves'][:, [idx, 2]],
                                                  showlegend=False, width=5)})
        else:
            builder.helper.add_data({'a': scatter(result['cut-leaves'], width=2)})
        builder.plot()


from django.contrib.auth import get_user_model
from django.db import models

CATEGORY_CHOICES = [('Drinks', 'Напитки'), ('Sweets', 'Сладости'), ('Bakery', 'Выпечка')]
RATING_CHOICES = [('1', '1'), ('2', '2'), ('3', '3'), ('4', '4'), ('5', '5')]

class BaseModel(models.Model):
    created_time = models.DateTimeField(auto_now_add=True, verbose_name="Дата создания")
    updated_time = models.DateTimeField(auto_now=True, verbose_name="Дата изменения")

    class Meta:
        abstract = True

class Product(BaseModel):
    name = models.CharField(max_length=35, null=False, blank=False, verbose_name="Название продукта")
    category = models.CharField(max_length=45, null=False, blank=False, choices=CATEGORY_CHOICES,
                                default=CATEGORY_CHOICES[0][0],
                                verbose_name='Категория продукта')
    description = models.TextField(max_length=3000, verbose_name="Описание продукта")
    image = models.ImageField(upload_to='images', null=True, blank=True,
                              verbose_name="Изображение")

    def __str__(self):
        return f"{self.id}. {self.name}."

    class Meta:
        db_table = "products"
        verbose_name = "Продукт"
        verbose_name_plural = "Продукты"
        permissions = [
            ('create_product', 'Создать товар'),
            ('update_product', 'Редактировать товар'),
            ('remove_product', 'Удалить товар'),
        ]

class Review(BaseModel):
    author = models.ForeignKey(get_user_model(), related_name="reviews", verbose_name="Автор", default=1,
                               on_delete=models.SET_DEFAULT)
    product = models.ForeignKey("webapp.Product", on_delete=models.CASCADE, related_name="products",
                                verbose_name="Продукт")
    review_text = models.TextField(max_length=3000, null=False, blank=False, verbose_name="Отзыв")
    rating = models.CharField(max_length=1, null=False, blank=False, choices=RATING_CHOICES,
                              default=RATING_CHOICES[0][0],
                              verbose_name="Оценка продукта")
    review_status = models.BooleanField(default=False)

    def str(self):
        return f"{self.id}. {self.author}"

    class Meta:
        db_table = "review"
        verbose_name = "Отзыв"
        verbose_name_plural = "Отзывы"
        permissions = [
            ('create_review', 'Создать отзыв'),
            ('update_review', 'Редактировать отзыв'),
            ('remove_review', 'Удалить отзыв'),
        ]