import logging

from os.path import join as opj

from datalad.interface.base import Interface
from datalad.interface.common_opts import annex_copy_opts, recursion_flag, \
    recursion_limit, git_opts, annex_opts
from datalad.support.param import Parameter
from datalad.support.constraints import EnsureStr
from datalad.support.constraints import EnsureNone
from datalad.support.annexrepo import AnnexRepo
from datalad.support.exceptions import InsufficientArgumentsError
from datalad.support.exceptions import IncompleteResultsError
from datalad.dochelpers import exc_str
from datalad.utils import assure_list

from .dataset import EnsureDataset
from .dataset import Dataset
from .dataset import datasetmethod
from .dataset import require_dataset

__docformat__ = 'restructuredtext'

lgr = logging.getLogger('datalad.distribution.publish')

# TODO: make consistent configurable output

def _log_push_info(pi_list):
    from git.remote import PushInfo as PI

    if pi_list:
        for push_info in pi_list:
            if (push_info.flags & PI.ERROR) == PI.ERROR:
                lgr.error(push_info.summary)
            else:
                lgr.info(push_info.summary)
    else:
        lgr.warning("Nothing was pushed.")

class Publish(Interface):
    """Publish a dataset to a known :term:`sibling`.

    This makes the last saved state of a dataset available to a sibling
    or special remote data store of the dataset which must already exist
    and be known to the dataset.

    .. note::
      Power-user info: This command uses :command:`git push`, and :command:`git annex copy`
      to publish a dataset. Publication targets are either configured remote
      Git repositories, or git-annex special remotes (if their support data
      upload).
    """
    # TODO: Figure out, how to tell about tracking branch/upstream
    #      (and the respective remote)
    #      - it is used, when no destination is given
    #      - it is configured to be the given destination, if there was no
    #        upstream set up before, so you can use just "datalad publish" next
    #        time.

    # TODO: Doc!

    _params_ = dict(
        dataset=Parameter(
            args=("-d", "--dataset"),
            metavar='DATASET',
            doc="""specify the dataset to publish. If no dataset is given, an
            attempt is made to identify the dataset based on the current
            working directory""",
            constraints=EnsureDataset() | EnsureNone()),
        to=Parameter(
            args=("--to",),
            metavar='LABEL',
            doc="""sibling name identifying the publication target. If no
            destination is given an attempt is made to identify the target
            based on the dataset's configuration (i.e. a set up tracking
            branch)""",
            # TODO: See TODO at top of class!
            constraints=EnsureStr() | EnsureNone()),
        since=Parameter(
            args=("--since",),
            constraints=EnsureStr() | EnsureNone(),
            doc="""When publishing dataset(s), specifies commit (treeish, tag, etc)
            from which to look for changes
            to decide either updated publishing is necessary for this and which children.
            If empty argument is provided, then we will always run publish command.
            By default, would take from the previously published to that remote/sibling
            state (for the current branch)"""),
        # since: commit => .gitmodules diff to head => submodules to publish

        skip_failing=Parameter(
            args=("--skip-failing",),
            action="store_true",
            doc="skip failing sub-datasets (incombination with `recursive`) "
                "instead of failing altogether"),
        path=Parameter(
            args=("path",),
            metavar='PATH',
            doc="path(s), that may point to file handle(s) to publish including "
                "their actual content or to subdataset(s) to be published. If a "
                "file handle is published with its data, this implicitly means "
                "to also publish the (sub)dataset it belongs to. '.' as a path "
                "is treated in a special way in the sense, that it is passed "
                "to subdatasets in case `recursive` is also given.",
            constraints=EnsureStr() | EnsureNone(),
            nargs='*'),
        recursive=recursion_flag,
        recursion_limit=recursion_limit,
        git_opts=git_opts,
        annex_opts=annex_opts,
        annex_copy_opts=annex_copy_opts
    )

    @staticmethod
    @datasetmethod(name='publish')
    def __call__(
            path=None,
            dataset=None,
            to=None,
            since=None,
            skip_failing=False,
            recursive=False,
            recursion_limit=None,
            git_opts=None,
            annex_opts=None,
            annex_copy_opts=None):
        # shortcut
        ds = require_dataset(dataset, check_installed=True, purpose='publication')
        assert(ds.repo is not None)

        path = assure_list(path)

        # figure out, what to publish from what (sub)dataset:
        publish_this = False   # whether to publish `ds`
        publish_files = []     # which files to publish by `ds`

        expl_subs = set()      # subdatasets to publish explicitly
        publish_subs = dict()  # collect what to publish from subdatasets

        if not path:
            # publish `ds` itself, if nothing else is given:
            publish_this = True
        else:
            for p in path:
                subdatasets = ds.get_subdatasets()
                if p in subdatasets:
                    # p is a subdataset, that needs to be published itself
                    expl_subs.add(p)
                else:
                    try:
                        d = ds.get_containing_subdataset(p)
                    except ValueError as e:
                        # p is not in ds => skip:
                        lgr.warning(str(e) + " - Skipped.")
                        continue
                    if d == ds:
                        # p needs to be published from ds
                        publish_this = True
                        publish_files.append(p)
                    else:
                        # p belongs to subds `d`
                        if not publish_subs[d.path]:
                            publish_subs[d.path] = dict()
                        if not publish_subs[d.d.path]['files']:
                            publish_subs[d.d.path]['files'] = list()
                        publish_subs[d.path]['dataset'] = d
                        publish_subs[d.path]['files'].append(p)

        if publish_this:
            # Note: we need an upstream remote, if there's none given. We could
            # wait for git push to complain, but we need to explicitly figure it
            # out for pushing annex branch anyway and we might as well fail
            # right here.

            track_remote, track_branch = None, None

            # keep `to` in case it's None for passing to recursive calls:
            dest_resolved = to
            if to is None:
                # TODO: If possible, avoid resolution herein and rely on git
                # (or GitRepo respectively), meaning: Just pass `None`
                # ATM conflicts with _get_changed_datasets => figure it out

                track_remote, track_branch = ds.repo.get_tracking_branch()
                if track_remote:
                    dest_resolved = track_remote
                else:
                    # we have no remote given and no upstream => fail
                    raise InsufficientArgumentsError(
                        "No known default target for "
                        "publication and none given.")

        subds_prev_hexsha = {}
        if recursive:
            all_subdatasets = ds.get_subdatasets(fulfilled=True)

            # TODO: dest_resolved => to?
            # Note: This is a bug anyway, since in actual recursive call `to` is
            # passed in order to be resolved by the subdatasets themselves
            # (might be None), but when considering what subdatasets to be
            # published, we assume `dest_resolved` is the same for all of them.

            # ==> TODO: RF to consider `since` only for the current ds and then go on
            # recursively.

            subds_to_consider = \
                Publish._get_changed_datasets(
                    ds.repo, all_subdatasets, dest_resolved, since=since) \
                if publish_this \
                else all_subdatasets
            # if we were returned a dict, we got subds_prev_hexsha
            if isinstance(subds_to_consider, dict):
                subds_prev_hexsha = subds_to_consider
            for subds_path in subds_to_consider:
                if path and '.' in path:
                    # we explicitly are passing '.' to subdatasets in case of
                    # `recursive`. Therefore these datasets are going into
                    # `publish_subs`, instead of `expl_subs`:
                    sub = Dataset(opj(ds.path, subds_path))
                    publish_subs[sub.path] = dict()
                    publish_subs[sub.path]['dataset'] = sub
                    publish_subs[sub.path]['files'] = ['.']
                else:
                    # we can recursively publish only, if there actually
                    # is something
                    expl_subs.add(subds_path)

        published, skipped = [], []

        for dspath in sorted(expl_subs):
            # these datasets need to be pushed regardless of additional paths
            # pointing inside them
            # due to API, this may not happen when calling publish with paths,
            # therefore force it.
            # TODO: There might be a better solution to avoid two calls of
            # publish() on the very same Dataset instance
            ds_ = Dataset(opj(ds.path, dspath))
            try:
                # we could take local diff for the subdataset
                # but may be we could just rely on internal logic within
                # subdataset to figure out what it needs to publish.
                # But we need to pass empty string one inside as is
                pkw = {}
                if since == '':
                    pkw['since'] = since
                else:
                    # pass previous state for that submodule if known
                    pkw['since'] = subds_prev_hexsha.get(dspath, None)
                published_, skipped_ = ds_.publish(to=to, recursive=recursive, **pkw)
                published += published_
                skipped += skipped_
            except Exception as exc:
                if not skip_failing:
                    raise
                lgr.warning("Skipped %s: %s", ds.path, exc_str(exc))
                skipped += [ds_]

        for d in publish_subs:
            # recurse into subdatasets

            # TODO: need to fetch. see above
            publish_subs[d]['dataset'].repo.fetch(remote=to)

            published_, skipped_ = publish_subs[d]['dataset'].publish(
                to=to,
                path=publish_subs[d]['files'],
                recursive=recursive,
                annex_copy_opts=annex_copy_opts)
            published += published_
            skipped += skipped_

        if publish_this:

            # is `to` an already known remote?
            if dest_resolved not in ds.repo.get_remotes():
                # unknown remote
                raise ValueError("No sibling '{0}' found for {1}."
                                 "".format(dest_resolved, ds))

            # in order to be able to use git's config to determine what to push,
            # we need to annex merge first. Otherwise a git push might be
            # rejected if involving all matching branches for example.
            # Once at it, also push the annex branch right here.

            # Q: Do we need to respect annex-ignore here? Does it make sense to
            # publish to a remote without pushing the annex branch
            # (if there is any)?
            if isinstance(ds.repo, AnnexRepo):
                ds.repo.fetch(remote=dest_resolved)
                ds.repo.merge_annex(dest_resolved)
                _log_push_info(ds.repo.push(remote=dest_resolved,
                                            refspec="git-annex:git-annex"))

            # upstream branch needed for update (merge) and subsequent push,
            # in case there is no.
            # no tracking branch yet?
            set_upstream = track_branch is None

            # publishing of `dest_resolved` might depend on publishing other
            # remote(s) first:
            # define config var name for potential publication dependencies
            depvar = 'remote.{}.datalad-publish-depends'.format(dest_resolved)
            for d in ds.config.get(depvar, []):
                lgr.info("Dependency detected: '%s'" % d)
                # Note: Additional info on publishing the dep. comes from within
                # `ds.publish`.
                ds.publish(path=path,
                           to=d,
                           since=since,
                           skip_failing=skip_failing,
                           recursive=recursive,
                           recursion_limit=recursion_limit,
                           git_opts=git_opts,
                           annex_opts=annex_opts,
                           annex_copy_opts=annex_copy_opts)

            lgr.info("Publishing {0} to {1}".format(ds, dest_resolved))

            # we now know where to push to:
            # TODO: what to push? default: git push --mirror if nothing configured?
            # consider also: --follow-tags, --tags, --atomic

            # Note: git's push.default is 'matching', which possibly doesn't
            # work for first
            # time publication (a branch, that doesn't exist on remote yet)
            # But if we want to respect remote.*.push entries, etc. we need to
            # not pass a specific refspec (like active branch) to `git push`
            # by default.

            _log_push_info(ds.repo.push(remote=dest_resolved,
                                        refspec=ds.repo.get_active_branch(),
                                        set_upstream=set_upstream))

            published.append(ds)

            if publish_files or annex_copy_opts:
                if not isinstance(ds.repo, AnnexRepo):
                    # incomplete, since `git push` was done already:
                    raise IncompleteResultsError(
                        (published, skipped),
                        failed=publish_files,
                        msg="Cannot publish content of something, that is not "
                            "an annex. ({0})".format(ds))
                if ds.config.get('remote.{}.annex-ignore', False):
                    # Q: Do we need a --force option here? annex allows to
                    # ignore the ignore setting
                    raise IncompleteResultsError(
                        (published, skipped),
                        failed=publish_files,
                        msg="Sibling '{0}' of {1} is configured to be ignored "
                            "by annex. No content was published."
                            % (dest_resolved, ds))

                lgr.info("Publishing data of dataset {0} ...".format(ds))
                published += ds.repo.copy_to(files=publish_files,
                                             remote=dest_resolved,
                                             options=annex_copy_opts)

        return published, skipped

    @staticmethod
    def result_renderer_cmdline(results, args):
        from datalad.ui import ui
        for res, res_label in zip(results, ('published', 'skipped')):
            if not res:
                if res_label == 'published':
                    ui.message("Nothing was %s" % res_label)
                continue
            msg = "{n} {obj} {res_label}:\n".format(
                obj='items were' if len(res) > 1 else 'item was',
                n=len(res),
                res_label=res_label)
            for item in res:
                if isinstance(item, Dataset):
                    msg += "Dataset: %s\n" % item.path
                else:
                    msg += "File: %s\n" % item
            ui.message(msg)

    @staticmethod
    def _get_changed_datasets(repo, all_subdatasets, to, since=None):
        if since == '' or not all_subdatasets:
            # we are instructed to publish all
            return all_subdatasets

        if since is None:  # default behavior - only updated since last update
            # so we figure out what was the last update
            # XXX here we assume one to one mapping of names from local branches
            # to the remote
            # TODO: This seems to be the only thing left, that we need to know the
            # remote `to` for (if not explicitly specified anyway). Otherwise
            # we could figure it out at GitRepo level instead, which makes
            # things easier, cleaner and more in line with git push.
            active_branch = repo.get_active_branch()
            since = '%s/%s' % (to, active_branch)

            if since not in repo.get_remote_branches():
                # we did not publish it before - so everything must go
                return all_subdatasets

        lgr.debug("Checking diff since %s for %s" % (since, all_subdatasets))
        diff = repo.repo.commit().diff(since, all_subdatasets)
        for d in diff:
            # not sure if it could even track renames of subdatasets
            # but let's "check"
            assert(d.a_path == d.b_path)
        return dict((d.b_path, d.b_blob.hexsha if d.b_blob else None) for d in diff)


from __future__ import division
from __future__ import print_function
from sentence_transformers import SentenceTransformer

from nltk import tokenize
import sys
import numpy as np

from pyod.models.copod import COPOD
from pyod.utils.data import evaluate_print

# 提取dataset中的评论并且
import csv

import nltk
nltk.download('punkt')

data = []
# with open('./paper.json', 'r') as file:
#     papers = json.load(file)
#     data = [paper["title"] for paper in papers]

with open('./papers.csv', 'rt',encoding='utf-8') as file:
    csv_reader = csv.reader(file)
    for idx, row in enumerate(csv_reader):
        if idx > 601:
            break
        if idx > 101:
            data.append(row[4])

print("end csv reading")

data.append("Recurrent topic-transition gan for visual paragraph generation.")
data.append("Modelling one‐and two‐dimensional solid‐state NMR spectra ")
data.append("Storygan: A sequential conditional gan for story visualization")
data.append("Attngan: Fine-grained text to image generation with attentional generative adversarial networks")
data.append("A perpendicular-anisotropy CoFeB–MgO magnetic tunnel junction")
data.append("Tidal ventilation at low airway pressures can augment lung injury.")

data_vecs = []
model = SentenceTransformer('sentence-transformers/multi-qa-mpnet-base-dot-v1')
for i, s in enumerate(data):
    sys.stdout.write("\r%d/%d" % (i, len(data)))
    sys.stdout.flush()
    r = model.encode(tokenize.sent_tokenize(s)).mean(axis=0)
    data_vecs.append(r.tolist())

print("end model encoding")

data_vecs = np.array(data_vecs)
print(np.shape(data_vecs))

clf = COPOD()
clf.fit(data_vecs)

print(np.shape(clf.labels_))

res = np.c_[clf.labels_, clf.decision_scores_].tolist()
for i, r in enumerate(res):
    r.append(data[i])

with open("new.csv", "w+",encoding = 'utf-8') as my_csv:
    csvWriter = csv.writer(my_csv, delimiter=',')
    csvWriter.writerows(res)


#! python3

'''
Overview: pubCrawl.py is a python script designed to "scrape" the webpage of a provided URL for "actionable items"(URLs, redirects, resource references, etc).
'''

#*IMPORT BLOCK BEGIN*
from bs4 import BeautifulSoup as bs
from selenium import webdriver
from selenium.webdriver.firefox.options import Options as firefoxOptions
from urllib.parse import urlparse
import datetime
import json
import os.path
import random
import requests
import re
import string
import sys
#*IMPORT BLOCK END*

#*FUNCTION BLOCK BEGIN*
#**STUB BLOCK BEGIN**
#***FUTURE VERSION BLOCK BEGIN***

#***FUTURE VERSION BLOCK END***

#**STUB BLOCK END**

def userInputHandler(systemInput,defaultURL = "https://www.instagram.com/",testMode = False,argValue=" "):
	'''
	Function for accepting & parsing user input from the console line.  Will return a default value(defaultURL) if no value is given by the user.
	:param systemInput: Input Data
	:type systemInput: str
	:param defaultURL: URL provided to the function to indicate the URL that is to be returned if no value is supplied by the user during initialization of the program
	:type defaultURL: str
	:param testMode: Testing value; value to inform the function whether it is operating in a function testing scenario or not.  Default is set to "False" to help assure that the function operates with the assumption of a live environment unless instructed otherwise.
	:type testMode: boolean
	:param argValue: Testing value; value to be provided when testing the function that provides a value in place of "sys.argv[1]" which will not be supplied when running unit tests from test_pubcrawl.py
	:type argValue: str
	:return: str
	'''

	try:
		if(len(systemInput) <= 1):
			cleanedURL = defaultURL
		else:
			if testMode == False:
				argValue = sys.argv[1]
			else:
				argValue = argValue
			providedURL = argValue
			urlCleaner = pubCrawl(providedURL)
			cleanedURL = urlCleaner.userInputParser(providedURL)
		return(cleanedURL)
	except Exception as e:
		logger = pubCrawl(systemInput)
		logger.logHandler(e, 'userInputHandler', 'ERROR')

class pubCrawl:
	'''
	Class designed scrape target site/URL's code for visible URLs.  It will then take the results & write them to a text file for review.
	'''
	#*STUB BLOCK BEGIN**
	#**FUTURE VERSION BLOCK BEGIN***
	def recursiveDepthManager(self,intDepth = 1):
		'''
		#!STUB(Future Feature)
		Function for managing & orchestrating the recursive function of user defined depths of scraping on found resources.
		:param intDepth: value that defines how far down the program should look for URLs within the URLs that it does find. Will be adjusted & implemented in later version.
		:type intDepth: int
		'''
		if(intDepth == 1):
			pass
		else:
			pass

	def siteRepCheck(self):
		'''
		#!STUB(Future Feature)
		Function for integrating API calls on malicious site checkers(Brightcloud, Virustotal, etc) to automatically check provided URLs for malicious activity/poor community reputation.
		'''
		pass

	#**FUTURE VERSION BLOCK END***
	#*STUB BLOCK END**

	#Lambda designed to read str pool of data & output it as a dictionary.  Expected output: dict
	dataPoolSorter = lambda self, data : json.loads(data)

	#Lambda designed to provide ease of operating system filepath construction.  Expected output: str
	pathCombiner = lambda self, firstFileValue, secondFileValue : os.path.join(firstFileValue, secondFileValue)

	def __init__(self, inputURL,scanDepth = 0, testMode = False):
		'''
		Function for initializing the pubCrawl class when it is called.
		:param inputURL: URL submitted for scraping by the program
		:type inputURL: str
		:param testMode: value to inform the class if it is operating under testing conditions or not
		:type testMode: boolean
		'''
		self.inputURL = inputURL
		self.scanDepth = scanDepth
		self.testMode = testMode

	def userInputParser(self,providedURL):
		'''
		Function for attempting to clean up user's input & parsing it properly for future connection/handling.
		:param providedURL: URL passed to the function to be evaluated & parsed to assure that the working URL is constructed in a predictable, more absolute way
		:type providedURL: str
		:return: str
		'''

		print(f"userInputParser's providedURL: {providedURL}")
		try:
			parsedURL = urlparse(providedURL)
			if parsedURL[1] != '':	#Check to determine if the value in parsedURL's netloc(parsedURL[1]) is populated.
				checkedURL = self.urlNetLocCreator(parsedURL)
			else:
				checkedURL = self.schemeIterator(parsedURL)

			return checkedURL

		except Exception as e:
			self.logHandler(e, 'userInputParser', 'ERROR')

	def schemeIterator(self,providedRelativeURL, schemeOptions = ['https','http']):
		'''
		Function for attempting to regulate & massage user inputted URLs for better functionality, primarily while working with the requests library.
		:param providedRelativeURL: Value to define the URL that will have different schemes applied to it to assure that it will lead to a functional URL.
		:type providedRelativeURL: str
		:param schemeOptions: list of options to be iterated through to be applied to the providedRelativeURL if no schemes have already been applied to providedRelativeURL when it was submitted.
		:type schemeOptions: list of strings that define URL schemes to be tested. This will be in order to assure that the URL provided will resolve properly.
		:return str:
		'''

		try:
			for scheme in schemeOptions:
				urlToCheck = f"{scheme}://{providedRelativeURL[2]}/" #Line that reconstructs the value in providedURL's path to add the current tested scheme to form "{SCHEME}://{PATH}"({http}://{scanme.nmap.org}" excluding curley braces)
				try: #Block to test the relative provided URL with the applied scheme
					request = requests.get(urlToCheck)
					checkedURL = urlToCheck
					break
				except:
					continue

			return checkedURL

		except Exception as e:
			self.logHandler(e, 'schemeIterator', 'ERROR')

	def urlNetLocCreator(self, providedURL):
		'''
		Function for taking strings provided by calling function, evaluating, & processing it so that netlocations(netloc) can be seperated more easily from paths(path).
		:param providedURL: Value to define the URL that will be evaluated & rebuilt so it more cleanly provides an absolute URL instead of a relative URL.
		:type providedURL: str
		:return: str
		'''

		try:
			checkedURL = f"{providedURL[0]}://{providedURL[1]}/" #Line that reconstructs the provided URL to include only the URL's scheme(parsedURL[0]) & it's netloc(parsedURL[1]) in form "{SCHEME}://{NETLOC}/"("{http}://{scanme.nmap.org}/" excluding curley braces)
			return checkedURL

		except Exception as e:
			self.logHandler(e, 'urlNetLocCreator', 'ERROR')

	def scrapeDynamicSite(self,inputURL):
		'''
		Function designed to gather & return URLs from websites that require activation before they can be properly scraped for URLs.
		:param inputURL: URL of the target to be scraped
		:type inputURL: str
		:return: list
		'''
		try:
			dynamicURLList = self.dynamicSiteGrabber(self.dynamicConnectorFirefox(),inputURL)
			return(dynamicURLList)
		except Exception as e:
			logHandlerReturn = self.logHandler(e, 'scrapeDynamicSite', 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)

	def scrapeStaticSite(self,inputURL):
		'''
		Function designed to gather & return URLs from websites that are static & have HTML available to be scraped without dynamic code activitation.
		:param inputURL: URL of the target to be scraped
		:type inputURL: str
		:return: list
		'''
		try:
			targetResponse = self.siteConnector(inputURL)
			siteHTML = bs(targetResponse, 'html.parser')
			staticURLList = self.staticSiteScraper(siteHTML)
			return staticURLList
		except Exception as e:
			logHandlerReturn = self.logHandler(e, "scrapeStaticSite", 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)

	def logHandler(self, loggingData, functionSource, logType = 'INFORMATION',writeLocation1 = "output",writeLocation2 = "logs"):
		'''
		Function for gathering error information & recording it to a central location.
		:param loggingData: Exception data provided by the calling function that is to be recorded.
		:type loggingData: str
		:param functionSource: Name of the function that is calling logHandler() so it can be recorded for logs.
		:type functionSource: str
		:param logType: Exception type provided by calling function to be recorded.
		:type logType: str
		:param writeLocation1: String indicating the parent folder of where logHandler() should write its log results
		:type writeLocation1: str
		:param writeLocation2: String indicating the child folder of where logHandler() should write its log results
		:type writeLocation2: str
		'''
		testMode = self.testMode
		if testMode == False:
			loggingData = str(loggingData)
			functionSource = str(functionSource)
			logType = str(logType)
			logTime = datetime.datetime.now()
			logTime = str(logTime)
			fileName = logType + '_LOGS' + '.txt'
			logFilePath = self.pathCombiner(writeLocation1,writeLocation2)
			fullFilePath = self.pathCombiner(logFilePath,fileName)
			openMode = 'a+'
			with open(fullFilePath, openMode) as writeFile:
				writeFile.write(logTime +  " (" + functionSource + "): " + loggingData + ";\n")
			if logType != "TESTING":
				print(f"During runtime {functionSource} met a logable event type {logType}")
				print(f"Log written to: {fullFilePath}")
			return None
		else:
			return loggingData

	def fileNamer(self):
		'''
		Feature for generating a unique filename for each of the pubCrawl outputs.
		:return: str in format ADJECTIVENOUN####
		'''
		adjectiveFilePath = self.pathCombiner("fileNames","adjectives.txt")
		adjectiveDataPool = self.fileReader(adjectiveFilePath)
		adjectives = self.dataPoolSorter(adjectiveDataPool)
		nounFilePath = self.pathCombiner("fileNames","nouns.txt")
		nounDataPool = self.fileReader(nounFilePath)
		nouns = self.dataPoolSorter(nounDataPool)

		try:
			adjectiveKey = random.choice(list(adjectives))
			adjectivePool = adjectives[adjectiveKey]
			adjectiveSelection = random.choice(list(adjectivePool)).capitalize()
			nounKey = random.choice(list(nouns))
			nounPool = nouns[nounKey]
			nounSelection = random.choice(list(nounPool)).capitalize()
			randomDigits = ''.join(random.choice(string.digits) for _ in range(4))
			fileName = adjectiveSelection + nounSelection + randomDigits
			return(fileName)
		except Exception as e:
			logHandlerReturn = self.logHandler(e, "fileNamer", 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)

	def fileReader(self, fullFileName):
		'''
		Function for reading exterior files & returning their content to requestor.
		:param fullFileName: file location of data to be read & returned
		:type fullFileName: str
		:return: file data in str format
		'''
		try:
			with open(fullFileName) as file:
				data = file.read()
			return data
		except Exception as e:
			logHandlerReturn = self.logHandler(e, 'fileReader', 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)

	def dataParser(self, targetSite, providedData, providedFileName, parentFolder = "output", fileType = ".txt"):
		'''
		Function for parsing the data that is found from the staticSiteScraper & saving it to a data file.
		:param targetSite: Value to indicate what site is being scraped for URLs.
		:type targetSite: str
		:param providedData: Data provided by calling function that will be written to the file
		:type providedData: str
		:param providedFileName: File name that is provided by calling function that the data will be written to.
		:type providedFileName: str
		:param parentFolder: Value that indicates what folder data should be written to.
		:type parentFolder: str
		:param fileType: the filetype of the file that the data will be written to.
		:type fileType: str
		'''
		mode = 'w'
		try:
			fileName = providedFileName + fileType
			fullFilePath = self.pathCombiner(parentFolder,fileName)
			with open(fullFilePath, mode) as fileWriter:
				fileWriter.write(f'URL Scraped: {targetSite}\n')
				for link in providedData:
					fileWriter.write(f'{link}\n')
		except Exception as e:
			logHandlerReturn = self.logHandler(e, 'dataParser', 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)
			return(testResolve)

	def staticSiteScraper(self, providedHTML):
		'''
		Function for scraping the URL provided by the user for usable information that is stored in the HTML(via hrefs).
		:param providedHTML: HTML provided by the calling function that will be searched for available URLs
		:type providedHTML: str
		:return: modified list
		'''
		urlList = []
		try:
			for link in providedHTML.find_all(attrs={'href': re.compile("^https?://")}):
				#print("\'href\': " + link.get('href'))
				urlList.append("%s" % link.get('href'))
			return(urlList)
		except Exception as e:
			self.logHandler(e, 'staticSiteScraper', 'ERROR')

	def siteConnector(self, workingURL):
		'''
		Function for negotiating & connecting to user supplied URL.
		:param workingURL: URL provided by calling function that will be connected to by siteConnector()
		:type workingURL: str
		:return: str
		'''
		try:
			response = requests.get(workingURL)
			return(response.text)
		except Exception as e:
			logHandlerReturn = self.logHandler(e, "siteConnector", 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)

	def dynamicConnectorFirefox(self):
		'''
		Function for creating a Selenium web browser driver in Firefox
		:return: selenium.webdriver.firefox.webdriver.WebDriver
		'''
		try:
			options = firefoxOptions()
			options.add_argument("--headless") #This line is to assure that a browser window does not open on the user's screen.  This line can be commented out if you want to open n number of browser windows where n is equal to the number of URLs being scraped.
			driver = webdriver.Firefox(options=options)
			return driver
		except Exception as e:
			logHandlerReturn = self.logHandler(e, 'dynamicConnectorFirefox', 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)

	def dynamicSiteGrabber(self,providedDriver,providedURL,searchingElement1 = 'tag name',searchingElement2 = 'a',retrievingAttribute = 'href'):
		'''
		Function for running & grabbing HTML elements from a site after it has been summoned.  Function then searches for elements on the page to be returned.
		:param providedDriver: Driver that has been generated via selenium for calling on the website
		:type providedDriver: selenium.webdriver.firefox.webdriver.WebDriver
		:param providedURL: Provided URL that will be summoned & scraped for information
		:type providedURL: str
		:param searchingElement1: The type of element that is to be searched for in resulting HTML once the site has been summoned.
		:type searchingElement1: str
		:param searchingElement2: Subtype of element that is to be searched for in resulting HTML when the site has been summoned.
		:type searchingElement2: str
		:param retrievingAttribute: Type of element that will be scanned for & retrieved to be returned to the calling function
		:type retrievingAttribute: str
		:return: list
		'''
		try:
			foundURLs = []
			driver = providedDriver
			driver.get(providedURL)
			allLinks = driver.find_elements(searchingElement1,searchingElement2)
			for elem in allLinks:
				foundURLs.append(elem.get_attribute(retrievingAttribute))
			return foundURLs
		except Exception as e:
			logHandlerReturn = self.logHandler(e, 'dynamicSiteGrabber', 'ERROR')
			testResolve = self.testHandler(logHandlerReturn,e)

	def testHandler(self,logHandlerValue,exceptionValue):
		'''
		Function for assisting the class to determine what steps to complete based on the values it recieves
		:param logHandlerValue: Value that may be passed to the fuction that was supplied by logHandler() if the class is running in testMode
		:type logHandlerValue: str
		:param exceptionValue: Value that is supplied to the function & returned if it is determined that the logHandlerValue contains nothing(None)
		:type exceptionValue: TypeError
		:return: TypeError
		'''
		if logHandlerValue == None:
			sys.exit("Exiting")
		else:
			return exceptionValue

	def scrapeSite(self,targetType='HYBRIDSITE'):
		'''
		Function for activating & operating the pubCrawl class in the anticipated order & manner if the user is planning on performing both static & dynamic scraping.
		:param targetType: Value that indicates the type of site that is going to be scraped.  DYNAMICSITE indicates a site that should be scraped for only URLs that arrive after scripts are allowed to load.  STATICSITE indicates a site that should only be scraped for URLS before scripts are allowed to load.  HYBRIDSITE indicates a site that should be scraped for URLS both before & after scripts are allowed to load.
		:type targetType: str
		'''
		workingURL = self.inputURL
		targetType = targetType.upper()
		depthCounter = 0
		urlList = []
		while(depthCounter <= self.scanDepth):
			print(depthCounter)
			depthCounter += 1
		print(f"URL to be scraped: {workingURL}")
		print(f"Depth Set: {str(self.scanDepth)}")
		if(targetType == 'HYBRIDSITE'):
			staticURLList = self.scrapeStaticSite(workingURL)
			dynamicURLList = self.scrapeDynamicSite(workingURL)
			uniqueURLSet = set.union(set(staticURLList),set(dynamicURLList))
		elif(targetType == 'DYNAMICSITE'):
			uniqueURLList = self.scrapeDynamicSite(workingURL)
			uniqueURLSet = set(uniqueURLList)
		elif(targetType == 'STATICSITE'):
			uniqueURLList = self.scrapeStaticSite(workingURL)
			uniqueURLSet = set(uniqueURLList)
		else:
			sys.exit(f'{targetType} is not a valid targetType value. Try again with one of the following:\n\n\tHYBRIDSITE\n\tDYNAMICSITE\n\tSTATICSITE\n')

		for url in uniqueURLSet:
			urlList.append(url)

		fileName = self.fileNamer()
		self.dataParser(workingURL,uniqueURLSet, fileName)
		print(f"Results stored to {fileName}")
		print(urlList)

def main():
	'''
	Function for the primary orchestration & running of the code
	'''

	workingURL = userInputHandler(sys.argv)
	pub_crawl = pubCrawl(workingURL)
	pub_crawl.scrapeSite()

#*FUNCTION BLOCK END*

if __name__ == '__main__':
	main()


import pandas as pd
import os
import re
import datetime

folderPath = './'
videoPath = folderPath + 'Videos/'

# main function
def main():
    timestamps, labels = readCSVFile()
    df = readFileAndCreateDataFrame(timestamps,labels)
    df.to_csv('./result.csv', index=False)

def readCSVFile():
    df = pd.read_csv(folderPath + 'label_data.csv', encoding="ISO-8859-1")
    header = df.columns.tolist()
    firstColumn = df[header[0]]
    secondColumn = df[header[1]]
    return firstColumn, secondColumn

def calculateMinuteToSec(start, end):
    sec = 0
    startDate = datetime.datetime.strptime(start,
                               '%Y%m%d%H%M%S')
    endDate = datetime.datetime.strptime(end,
                               '%Y-%m-%d %H:%M:%S')
    duration = str(endDate - startDate)
    for value in duration.split(':'):
        sec = 60 * sec + int(value)
    return sec

def sortAscendingOrder(array):
    # loop to access each array element
    for i in range(len(array)):

        # loop to compare array elements
        for j in range(0, len(array) - i - 1):

            # compare two adjacent elements
            if array[j]['video'][6:-4] > array[j + 1]['video'][6:-4]:

                # swapping elements if elements are not in the intended order
                temp = array[j]
                array[j] = array[j+1]
                array[j+1] = temp
    return array

def createDataFrame(loca, vid, first, second, seconds):
    data = {'timestamp': first, 'label': second, 'location': loca, 'video_file': vid, 'seconds_into': seconds}
    df = pd.DataFrame(data)
    return df

def readFileAndCreateDataFrame(timestamps, labels):
    locations = []
    video_files = []
    videos_dict= []
    seconds_into = []

    subfolders = os.listdir(folderPath + 'Videos')
    for subfolder in subfolders:
        for root,dirs,filenames in os.walk(videoPath + subfolder):
            # save video and folder name as dict
            for file in filenames:
                video_dict = {}
                video_dict['video'] = file
                video_dict['location'] = subfolder
                videos_dict.append(
                    video_dict
                )
    sortedDictArray = sortAscendingOrder(videos_dict)

    for index, value in enumerate(sortedDictArray):

        if index < len(sortedDictArray) - 1:
            cur_element = value['video'][6:-4]
            next_element = sortedDictArray[index + 1]['video'][6:-4]

            for file in timestamps:
                labeledTimestamp = re.sub('\ |\:|\-', '', str(file))
                # check labeled timestamp is within first video timestamp and second video timestamp
                if labeledTimestamp>= cur_element and labeledTimestamp< next_element:
                    locations.append(value['location'])
                    video_files.append(value['video'])
                    seconds_into.append(calculateMinuteToSec(cur_element, file))
                else:
                    pass
        else:
            for file in timestamps:
                labeledTimestamp = re.sub('\ |\:|\-', '', str(file))
                # check the record is greater than the last video or not
                # (for not losing the data which timestamp is greater than the last video)
                if labeledTimestamp>= next_element:
                    locations.append(value['location'])
                    video_files.append(value['video'])
                    seconds_into.append(calculateMinuteToSec(cur_element, file))
    df = createDataFrame(locations, video_files, timestamps, labels, seconds_into)
    return df

if __name__=="__main__":
    main()


import click
import os

from crawler.crawler import Crawler
from configs import configs
from utils.constants import HELP_CFGS, HELP_URL

# Some random defaults for configuration options.
default_config = configs.DEFAULT_CONFIGS
default_agent = default_config['user_agent']
default_search_method = default_config['traversal']
default_max_depth = default_config['max_depth']
default_max_pages = default_config['max_total']

# The entry point for our CLI.
@click.group()
def main():
    pass

# Sub-command for actually crawling the website.
@main.command(name='crawl')
# @click.option('--a', default=default_agent, help=HELP_AGENT)
# @click.option('--m', default=default_search_method, help=HELP_SEARCH_METHOD)
# @click.option('--depth', default=default_max_depth, help=HELP_MAX_DEPTH)
# @click.option('--pages', default=default_max_pages, help=HELP_MAX_PAGES)
@click.argument('url', type=click.STRING, metavar='<url>', required=True)
@click.argument('cfgs', type=click.STRING, metavar='<cfgs>', required=True)
def crawl(url, cfgs):
    click.secho("Crawler will begin on '{url}' with below settings:\n".format(url=url), fg='green')
    config = configs.load_config_section(config_section=cfgs)
    if config is None:
        print(f"Invalid config {cfgs}. Switching to DEFAULT.")
        config = configs.load_config_section(config_section='DEFAULT')
    else:
        print(f"Config set {cfgs} loaded.")
    click.echo()
    crawler = Crawler()
    print(f"Target URL = {url}")
    crawler.crawl(url, config['traversal'], config['user_agent'], int(config['max_depth']), int(config['max_total']))

# Sub-command that displays information about the project.
@main.command(name='info')
def info():
    # Gets the width of the terminal.
    terminal_width = os.get_terminal_size().columns

    # Project information and metadata.
    title = 'kumo [クモ]'.center(terminal_width)[:-2]
    description = 'A configurable, efficient web crawler and form brute-forcer written in Python.\n'.center(terminal_width)
    version = 'version: 0.1'.center(terminal_width)
    authors = 'authors: Johnny So, Andy Liang, Stanley Lim, Mankirat Gulati'.center(terminal_width)
    github = 'github: http://github.com/devhid/kumo'.center(terminal_width)

    # Output to STDOUT with ANSI coloring.
    click.secho(title, fg='green')
    click.secho(description, fg='blue')
    click.secho(version, fg='yellow')
    click.secho(authors, fg='yellow')
    click.secho(github, fg='yellow')

if __name__ == "__main__":
    main()


import datetime
from sqlalchemy.sql.schema import Column
from sqlalchemy import Integer
from sqlalchemy.sql.sqltypes import String, Text, SmallInteger
from herovii.models.base import Base

__author__ = 'bliss'

class NewsOrg(Base):
    __tablename__ = 'hisihi_organization_notice'
    __bind_key__ = 'csu'

    id = Column(Integer, primary_key=True)
    organization_id = Column(Integer, default=0)

    # 新闻类型
    tag = Column(String(45))

    # 新闻标题
    title = Column(String(100), nullable=True)
    content = Column(Text)

    # 新闻是否推送到
    push_to_organization = Column(SmallInteger, default=1)
    create_time = Column(Integer, default=int(datetime.datetime.now().timestamp()))
    update_time = Column(Integer, default=int(datetime.datetime.now().timestamp()))
    status = Column(Integer, default=1)

    def keys(self):
        return (
            'id', 'tag', 'title', 'content',
            'create_time', 'update_time'
        )


# https://leetcode.com/problems/number-of-islands/
# TC: O(M*N)
# SC: O(M*N)

class Solution:
    from collections import deque

    def _is_valid(self, grid, r, c):
        if r < 0 or c < 0 or r > len(grid) - 1 or c > len(grid[0]) - 1:
            return False
        return True

    def dfs(self, grid, x, y):
        DIRECTIONS = [(1, 0), (-1, 0), (0, 1), (0, -1)]
        grid[x][y] = '0'
        for d in DIRECTIONS:
            new_x = x + d[0]
            new_y = y + d[1]
            if self._is_valid(grid, new_x,
                              new_y) and grid[new_x][new_y] == '1':
                self.dfs(grid, new_x, new_y)

    def bfs(self, grid, x, y):
        DIRECTIONS = [(1, 0), (-1, 0), (0, 1), (0, -1)]
        grid[x][y] = '0'
        queue = collections.deque()
        queue.append((x, y))

        while queue:
            x, y = queue.popleft()
            for d in DIRECTIONS:
                new_x = x + d[0]
                new_y = y + d[1]
                if self._is_valid(grid, new_x,
                                  new_y) and grid[new_x][new_y] == '1':
                    queue.append((new_x, new_y))

    def numIslands(self, grid: List[List[str]]) -> int:

        if not grid or not grid[0]:
            return 0

        r = len(grid)
        c = len(grid[0])
        count = 0

        for x in range(r):
            for y in range(c):
                if grid[x][y] == '1':
                    count += 1
                    self.dfs(grid, x, y)
        return count


from SimilarityMeasure import SimilarityMeasure
from model import Model
from enumeration import enum
from datetime import *
import numpy

class SearchAlgorithm:
	"""
	An abstract class/interface defining the expected behaviour of an optimization algorithm
	"""
	#optimizer = Optimizer();					# the optimizer with this this instance is associated
	#similarityMeasure = SimilarityMeasure();	# a SimilarityMeasure object used to calculate the fitness of models
	models = {};								# Map<hyperparams, models>
	maxIterations = 1024;						# hard cap on number of iterations before termination

	elapsedTime = datetime.now();	# how long the search took

	# TODO: determine if we need a macrostate enum as a safety measure

	# boundaries on search
	# TODO: resolve hard-coded things
	ensembleSizes = numpy.array([]);					# set of possible ensemble sizes
	backrubTemps = numpy.array([]);						# set of possible backrub temperatures
	boltzmannTemps = numpy.array([]);					# possible boltzmann temperatures or range of possible temps
	continuousBoltzmann = False;						# are we searching a range, or do we only have discrete macrostate data?
	steepnessRange = numpy.array([]);					# {minSteepness, maxSteepness}
	weightMins = numpy.array([]);						# minimum weights
	weightMaxs = numpy.array([]);						# maximum weights

	# which parameters to search, default is to search all of them.
	searchEnsemble = True;
	searchBackrub = True;
	searchBoltzmann = True;
	searchSteepness = True;
	searchWeights = numpy.array([]);

	# record of best parameters found so far
	bestEnsembleSize = 1;
	bestBackrubTemp = 1.0;
	bestBoltzmannTemp = 1.0;
	bestSteepness = 1.0;
	bestWeights = [];
	bestMatchVal = 0;
	bestFrequencies = numpy.array([]);

	# print things to console?
	suppressOutputs = False;

	def __init__(self, models = None, similarityMeasure:SimilarityMeasure = None, continuousBoltzmann:bool = False):
		"""
		Default constructor

		@param models				Map<string, model> of a set of Model objects holding the data read in
		@param similarityMeasure	the similarity measure to use
		@param continuousBoltzmann	whether to the boltzmann averagin search is contiuous or discrete
		"""
		self.similarityMeasure = similarityMeasure;
		self.models = models;
		self.continuousBoltzmann = continuousBoltzmann;
		self.searchEnsemble = True;
		self.searchBackrub = True;
		self.searchBoltzmann = True;
		self.searchSteepness = True;
		self.suppressOutputs = False;

		#self.optimizer = optimizer;

	def setSimilarityMeasure(self, similarityMeasure:SimilarityMeasure) -> None:
		"""
		Sets the similarity measure to be used

		@param similarityMeasure	new similarity measure to be used
		@return void
		"""
		self.similarityMeasure = similarityMeasure;

	def setMaxIterations(self, iter:int) -> None:
		"""
		Sets the max number of iterations of the algorithm. Typically the algorithm *will*
		run for this number of iterations....

		@param iter		int, max limit
		@return void
		"""
		self.maxIterations = iter;

	def setContinuousBoltzmann(self, yes:bool) -> None:
		"""
		Sets the algorithm whether to search through a continuous range of boltzmann averaging
		temperatures or not

		@param yes		bool, true for yes
		@return void
		"""
		if yes is not bool:
			raise TypeError("Input is not a bool");
		self.continuousBoltzmann = yes;
		return None;

	def getContinuousBoltzmann(self) -> bool:
		return self.continuousBoltzmann;

	def setParamBounds(self, ensembleSizes:"int[]", backrubTemps:"float[]", boltzmannTemps:"float[]", steepnessRange:"float[]", weightMins:"float[]", weightMaxs:"float[]") -> None:
		"""
		Sets the bounds on the parameter space to search through

		@param ensembleSizes		int[] of the ensemble sizes
		@param backrubTemps			float[] of discrete backrub temperatures
		@param boltzmannTemps		float[] of discrete Boltzmann temperatures
		@param steepnessRange		float[] of length 2: {lower bound, upper bound}
		@param weightMins			float[] of minimum weights
		@param weightMaxs			float[] of maximum weights
		@return void
		"""
		self.ensembleSizes = ensembleSizes;
		self.backrubTemps = backrubTemps;
		self.boltzmannTemps = boltzmannTemps;
		self.steepnessRange = steepnessRange;
		self.weightMins = weightMins;
		self.weightMaxs = weightMaxs;
		#self.searchWeights = numpy.array([True] * self.weightMins.shape[0]); # why was this line here anyways?

	def setSearchParameters(self, ensemble:bool, backrub:bool, boltzmann:bool, steepness:bool, weights:"bool[]") -> None:
		"""
		Sets which parameters to search through

		@param ensemble		bool
		@param backrub		bool
		@param boltzmann	bool
		@param steepness	bool
		@param weights		bool[]
		@return void
		"""
		self.searchEnsemble = ensemble;
		self.searchBackrub = backrub;
		self.searchBoltzmann = boltzmann;
		self.searchSteepness = steepness;
		self.searchWeights = weights;

	def setAllSearchToTrue(self) -> None:
		"""
		All parameters are to be searched

		@params void
		@return void
		"""
		self.searchEnsemble = True;
		self.searchBackrub = True;
		self.searchBoltzmann = True;
		self.searchSteepness = True;
		self.searchWeights = numpy.array([True] * self.weightMins.shape[0]);

	def setAllSearchToFalse(self) -> None:
		"""
		No parameters are to be searched, this is verifying some found parameters

		@params void
		@return void
		"""
		self.searchEnsemble = False;
		self.searchBackrub = False;
		self.searchBoltzmann = False;
		self.searchSteepness = False;
		self.searchWeights = numpy.array([False] * self.weightMins.shape[0]);

	# VIRTUAL ABSTRACT
	def iterate(self) -> None:
		"""
		Start searching

		@param void
		@return void
		"""
		raise NotImplementedError;

	def boundCheckBoltzmann(self, newBoltzmann:float) -> float:
		"""
		Check whether a new Boltzmann averaging temp is in the bounds
		If it is out of bounds, trim to the bound
		Should only be used when searching on a continuous range

		@param newBoltzmann		value to check against the bounds
		@return float	the input value if it's within bounds or a corrected value if not
		"""
		# validate that this function should be used
		if not self.continuousBoltzmann:
			raise AssertionError("This search is not on a continuous range of Boltzmann averaging temps");
			return None; # in the unlikely event that the call was in a try block, return a null to make sure that things still screw up down the line

		elif not self.searchBoltzmann:
			return self.boltzmannTemps[0];
		elif newBoltzmann < -1:						# less than inf
			return self.boltzmannTemps[1];
		elif newBoltzmann == 0:						# force move away from 0
			return self.boltzmannTemps[0];
		elif newBoltzmann < self.boltzmannTemps[0]: # lower than lower bound set to 0 - min boltzmann
			return 0;
		elif newBoltzmann > self.boltzmannTemps[1]:	# higher than higher bound set to -1 - inf
			return -1;
		else:
			return newBoltzmann;

	def boundCheckSteepness(self, newSteep:float) -> float:
		"""
		Check whether a new steepness value is in the bounds
		If it is out of bounds, trim to the bound

		@param newSteep		float, the value to check
		@return float		the corrected steepness value
		"""
		if not self.searchSteepness:	# check if we're searching steepness
			return self.steepnessRange[0];
		elif newSteep < self.steepnessRange[0]:
			return self.steepnessRange[0];
		elif newSteep > self.steepnessRange[1]:
			return self.steepnessRange[1];
		else:
			return newSteep;

	def boundCheckWeights(self, newWeights:numpy.array) -> numpy.array:
		"""
		Check whether a new set of weights in the specified bounds.
		Values out of bounds will be trimmed to the bound

		@param newWeights:		float[], new weights to check
		@return float[]			the checked set of weights
		"""
		newWeights = numpy.array(newWeights);	# useful for intellisense, otherwise the computer doesn't know type >.<
		for i in range(newWeights.size):
			if not self.searchWeights[i]:	# first check if this weight is to be searched
				newWeights[i] = self.weightMins[i];
			elif newWeights[i] < self.weightMins[i]:
				newWeights[i] = self.weightMins[i];
			elif newWeights[i] > self.weightMaxs[i]:
				newWeights[i] = self.weightMaxs[i];
			else:
				pass;
		return newWeights;

	def getModelByParams(self, param1, param2, param3) -> Model:
		"""
		Gets a model by the specified pre-determined parameters.
		Return object is a not deep copy that should not be modified

		@param ensembleS		int, ensemble size
		@param backrubT			float, backrub temperature
		@param boltzmannT		float, boltzmann averaging temp
		@return Model with specified params
		"""
		# convert to string
		param1 = str(param1);
		param2 = str(param2);
		param3 = str(param3);

		return self.models[param1 + " " + param2 + " " + param3];

	def getBestParameters(self) -> {}:
		"""
		Returns the currently best parameters.
		Keys:
			'ensembleSize'
			'backrubTemp'
			'boltzmannTemp'
			'steepness'
			'weights'
			'match'

		@param void
		@return		a dictionary with keys ensembleSize, backrubTemp, boltzmannTemp, steepness, weights
		"""
		bestParams = {};
		bestParams['ensembleSize'] = self.bestEnsembleSize;
		bestParams['backrubTemp'] = self.bestBackrubTemp;
		bestParams['boltzmannTemp'] = self.bestBoltzmannTemp;
		bestParams['steepness'] = self.bestSteepness;
		bestParams['weights'] = self.bestWeights;
		bestParams['match'] = self.bestMatchVal;
		return bestParams;

	def getBestFrequencies(self) -> numpy.array:
		"""
		Returns the set of frequencies corresponding to the best match

		@param void
		@return float[][] of frequencies
		"""
		return numpy.array(self.bestFrequencies);


# 问题描述
# 给定word1和word2两个单词，找到使word1和word2相同所需的最少步骤，每个步骤可以删除任一字符串中的一个
# 字符。

# 示例
# 输入："sea"和"eat"
# 输出：2
# 解释：第1步将"sea"变成"ea"，第2步"eat"变成"ea"

# 源码实现
class Solution:
    def minDistance(self, word1, word2):
        m, n = len(word1), len(word2)
        dp = [[0] * (n + 1) for i in range(m + 1)]
        for i in range(m):
            for j in range(n):
                dp[i + 1][j + 1] = max(dp[i][j + 1], dp[i + 1][j], dp[i][j] + (word1[i] == word2[j]))
        return m + n - 2 * dp[m][n]

if __name__ == '__main__':
    s = Solution()
    word1 = "sea"
    word2 = "eat"
    print("输入1：", word1)
    print("输入2：", word2)
    print("输出：", s.minDistance(word1, word2))

# 运行结果
# 输入1： sea
# 输入2： eat
# 输出： 2

进阶/例217.两个字符串的删除操作.py

例217.两个字符串的删除操作.py


#import statements
import tkinter as tk

try:
    #fixes window resolution blur in Windows OS
    from ctypes import windll
    windll.shcore.SetProcessDpiAwareness(1)
finally:

    #creates root window
    root = tk.Tk()

    #adds title text
    root.title('Tkinter Window Demo')

    ''' this section below covers the elements which make up a  window'''

    #default window width and height
    window_width =300
    window_height =200

    #get computer screen dimensions
    screen_width= root.winfo_screenwidth()
    screen_height= root.winfo_screenheight()

    #get center points of the screen compared to window size
    center_x= int(screen_width/2 -window_width/2)
    center_y = int(screen_height/2 - window_height/2)

    #root window with dimensions and position (WidthxHeight+-LR+-TB) + == from the Left or Top, - == from the Right or Bottom
    #sets position of window in the center of the screen
    root.geometry(f'{window_width}x{window_height}+{center_x}+{center_y}')

    #controls whether or not window is resisable (default is true) (width, height)
    root.resizable(True, False)

    #controls minimum size of window (width, height)
    root.minsize(window_width-100, window_height)

    #controls maximum size of window (width, height)
    root.maxsize(1000, window_height)

    #control the opacity/transparency of the window
    root.attributes('-alpha',0.7)

    root.wm_iconbitmap('//boxofholding/Data/Brennen/GitHub/Personal-Projects/relearn1/Python/TKinter/assets/pythontutorial.ico')

    #places a label on the root window
    message = tk.Label(root, text="Hello, World!")
    message.pack()

    #forces window to topsmost layer
    root.attributes('-topmost', 1)

    '''Note could also use .lift or .lower to make '''
    #keeps the window displayed
    root.mainloop()


#01

'''Ao final do processamento, o valor da variável SOMA será 91.

Isso ocorre porque o laço "enquanto" adiciona o valor de K à SOMA a cada iteração, começando com K=1 e
incrementando K a cada iteração até que K seja igual a INDICE (13, neste caso). Portanto, a soma será 1+2+3+...+13, que é igual a 91.'''

#02
'''2) Dado a sequência de Fibonacci, onde se inicia por 0 e 1 e o próximo valor sempre será a soma dos 2 valores anteriores
(exemplo: 0, 1, 1, 2, 3, 5, 8, 13, 21, 34...), escreva um programa na linguagem que desejar onde, informado um número,
ele calcule a sequência de Fibonacci e retorne uma mensagem avisando se o número informado pertence ou não a sequência.'''

n =  int(input('Informe um número: '))
def Fibonacci(n):
    seque_fib = [0]*n
    a = 0
    b = 1
    seque_fib[0]=0
    for i in range (1,n-1):
        prox = a + b
        a = b
        seque_fib[i] = a
        b = prox
        seque_fib[i+1] = b
        if n == prox:
            print('O número está presente na sequência')
        else:
            pass
    return seque_fib
print(Fibonacci(n))

#Também poderia ser assim:
num = int(input('Informe um número:'))

fibonacci = [0, 1]
while fibonacci[-1] < num:
    proximo = fibonacci[-1] + fibonacci[-2]
    fibonacci.append(proximo)

if num in fibonacci:
    print(f"{num} está na sequência de Fibonacci!")
else:
    print(f"{num} não está na sequência de Fibonacci.")

print("A sequência de Fibonacci até", num, "é:",fibonacci)

#3
'''3) Descubra a lógica e complete o próximo elemento:

a) 1, 3, 5, 7, _9__

b) 2, 4, 8, 16, 32, 64, __128__

c) 0, 1, 4, 9, 16, 25, 36, __49__

d) 4, 16, 36, 64, __100__

e) 1, 1, 2, 3, 5, 8, __13__

f) 2,10, 12, 16, 17, 18, 19, __200__
'''

'''
a) O próximo elemento é 9, pois a lógica é somar 2 a cada número.

b) O próximo elemento é 128, pois a lógica é multiplicar o número anterior pelo número por 2.

c) O próximo elemento é 49, Cada número é igual ao anterior acrescido de um número ímpar que segue a sequência (+1, +3, +5, +7, +9, +11, +13).
Assim, devemos acrescentar 13, obtendo 36 + 13 = 49.

d) Cada número é igual ao quadrado dos números pares. Com isso, temos que 64 = 8^2. Então, o próximo número par é 10, e o seu quadrado é 10^2 = 100.

e) O próximo elemento é 13, pois a lógica é somar os dois números anteriores para obter o
próximo número na sequência (exemplo: 1+1=2, 1+2=3, 2+3=5, 3+5=8, etc).

f) O próximo elemento é 200, pois a lógica é a sequencia de números que se iniciam com a letra D.
'''

#4
'''4 - Dois veículos (um carro e um caminhão) saem respectivamente de cidades opostas pela mesma rodovia. O carro de Ribeirão Preto em direção a Franca,
a uma velocidade constante de 110 km/h e o caminhão de Franca em direção a Ribeirão Preto a uma velocidade constante de 80 km/h.
Quando eles se cruzarem na rodovia, qual estará mais próximo a cidade de Ribeirão Preto?

IMPORTANTE:

a) Considerar a distância de 100km entre a cidade de Ribeirão Preto <-> Franca.

b) Considerar 2 pedágios como obstáculo e que o caminhão leva 5 minutos a mais para passar em cada um deles e o carro possui tag de pedágio (Sem Parar)

c) Explique como chegou no resultado.
'''

'''
O carro e o caminhão estão se aproximando um do outro em velocidades opostas, então sua velocidade relativa é a soma de suas velocidades. Portanto, a velocidade relativa é:

110 km/h + 80 km/h = 190 km/h

A distância entre as duas cidades é de 100 km, e quando os dois veículos se encontram na estrada, eles terão percorrido juntos uma distância igual a 100 km.

Para encontrar o tempo que eles levam para se encontrarem, podemos usar a fórmula:

tempo = distância / velocidade

Substituindo os valores, obtemos:

tempo = 100 km / 190 km/h ≈ 0,5263 h ≈ 31,58 minutos

Portanto, os dois veículos se encontrarão na rodovia após cerca de 31,58 minutos.

O carro está viajando em direção a Franca, então ele está mais próximo de Franca quando eles se encontram. Ele terá percorrido uma distância igual a:

distância percorrida pelo carro = 110 km/h x 0,5263 h ≈ 57,89 km

Portanto, quando os dois veículos se encontrarem na estrada, o carro estará a uma distância de aproximadamente 42,11 km da cidade de Ribeirão Preto.

O caminhão, por outro lado, está viajando em direção a Ribeirão Preto e estará mais próximo dessa cidade quando eles se encontrarem. No entanto,
o caminhão levará mais tempo para percorrer a mesma distância, devido aos pedágios.
Supondo que cada pedágio leve 5 minutos para passar, o caminhão levará 10 minutos adicionais (ou 0,1667 horas) para chegar ao ponto de encontro.

Assim, a distância percorrida pelo caminhão até o ponto de encontro será:

distância percorrida pelo caminhão = 80 km/h x (0,5263 h + 0,1667 h) ≈ 60 km

Portanto, quando os dois veículos se encontrarem na estrada, o caminhão estará a uma distância de aproximadamente 40 km da cidade de Ribeirão Preto.

Concluindo, quando os dois veículos se encontrarem na rodovia, o caminhão estará mais próximo da cidade de Ribeirão Preto, a cerca de 40 km,
enquanto o carro estará mais próximo de Franca, a cerca de 57,89 km.'''

5#
'''5) Escreva um programa que inverta os caracteres de um string.

IMPORTANTE:

a) Essa string pode ser informada através de qualquer entrada de sua preferência ou pode ser previamente definida no código;

b) Evite usar funções prontas, como, por exemplo, reverse;

'''

palavra = "Estágio dos sonhos"
palavra_invertida = ""

for letra in palavra:
    palavra_invertida = letra + palavra_invertida

print(palavra_invertida)


import numpy as np
import pickle
import sys

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def df_sigmoid(x):
    return sigmoid(x) * (1 - sigmoid(x))

class MLP:  # {
    def __init__(
            self,
            input_layer_neurons,
            hidden_layer_neurons,
            output_layer_neurons,
            f=sigmoid,
            df=df_sigmoid
            ):  # {
        # assign parameters to the object
        self.input_layer_neurons = input_layer_neurons
        self.hidden_layer_neurons = hidden_layer_neurons
        self.output_layer_neurons = output_layer_neurons
        self.f = f
        self.df = df
        self.squared_err = -1

        """
        As  stated in  "Y. Bengio, X. Glorot, Understanding  the  difficulty of
        training  deep feedforward  neuralnetworks, AISTATS 2010"  this is  the
        optimal  way to initialize the weights of a neuralnetwork because it is
        the closest to a linear function  in a `tanh` or `sigmoid`. If we use a
        `sigmoid`  function, the min/max  values should  be multiplied  by 4 to
        better accommodate the curve
        """
        max = np.sqrt(6. / (input_layer_neurons + hidden_layer_neurons))
        # max = 0.5  # use to test later
        min = - max

        if (self.f == sigmoid):
            print("Using sigmoid")
            sys.stdout.flush()
            max *= 4
            min *= 4

        # h_neuron[neuron] => array of weights (W_h)
        self.hidden_layer_weights = (max-min) * np.random.rand(
                hidden_layer_neurons,
                input_layer_neurons
                ) + min
        # bias[neuron] => bias (b_h)
        self.hidden_layer_bias = (max-min) * np.random.rand(
                hidden_layer_neurons,
                1
                ) + min

        # o_neuron[n] => array of weights (W_o)
        self.output_layer_weights = (max-min) * np.random.rand(
                output_layer_neurons,
                hidden_layer_neurons
                ) + min

        # bias[neuron] => bias (b_o)
        self.output_layer_bias = (max-min) * np.random.rand(
                output_layer_neurons,
                1
                ) + min
    # ==end __init__ }

    """ feed forward the input to get the output{
    feed forward to propagate the input through the network,
    producing activation functions and their derivatives
    --- arguments ---
    X: one test case, must be the same size as input_neurons
    } """
    def feed_forward(self, X):  # {
        # from the input layer to hidden:
        f_h = np.zeros(self.hidden_layer_neurons)
        df_h = np.zeros(self.hidden_layer_neurons)
        # for each hidden neuron, calculate net, f(net) and df(net)/dnet
        for neuron in range(self.hidden_layer_neurons):
            W = self.hidden_layer_weights[neuron]
            b = self.hidden_layer_bias[neuron]
            # net = (input . W) + b
            net = X.dot(W) + b
            f_h[neuron] = self.f(net)
            df_h[neuron] = self.df(net)
        # ==end for

        # from the hidden layer to output:
        f_o = np.zeros(self.output_layer_neurons)
        df_o = np.zeros(self.output_layer_neurons)
        # for each output neuron, calculate net, f(net) and df(net)/dnet
        for neuron in range(self.output_layer_neurons):
            W = self.output_layer_weights[neuron]
            b = self.output_layer_bias[neuron]
            # net = (f_h . W) + b
            net = f_h.dot(W) + b
            f_o[neuron] = self.f(net)
            df_o[neuron] = self.df(net)
        # ==end for
        return (f_h, df_h, f_o, df_o)
    # ==end feed_forward }

    """ calculate deltas {
    Calculate the  delta for  each layer, based  on the  local error  using the
    generalized delta rule to solve the gradient descendent
    --- arguments ---
    expected - the expected output. Must  be of length  equal to the  number of
               output_layer_neurons
    out - the actual output  layer values. Must be the  same length as expected
    df_h - hidden layer  activation derivatives. Must be the same length as the
           number of hidden_layer_neurons
    df_o - output layer  activation derivatives. Must be the same length as the
           number of output_layer_neurons
    } """
    def get_deltas(self, expected, out, df_h, df_o):  # {
        # delta = expected_out - actual_out
        delta = expected - out

        # Apply generalized delta rule to the output layer
        delta_o = np.multiply(delta, df_o)
        # Apply generalized delta rule to the hidden layer
        delta_h = np.multiply(
                (delta_o @ self.output_layer_weights),
                df_h
                )
        return (delta, delta_o, delta_h)
    # ==end get_deltas }

    """ learn using the backpropagation algorithm {
    Run feed_forward for  the whole  dataset, updating the error every round to
    get an average  error. This total  will be used to determine how  good  the
    network is at solving  the problem. When  it is good (error < threshold) we
    can stop training.
    Every feed forward round, the weights of each  neuron get updated using the
    backpropagation  algorithm, witch uses  the gradient descendant to find the
    direction of the minimum  for the error function. In the  case of a average
    square error, this gradient is calculated by multiplying each error by  its
    correspondent activation function derivative
    --- arguments ---
    X - An array of test  cases. Each X[i] must  be a test case of length equal
        to the number of input_neurons
    expected_output - An array of  expected results, each  corresponding to one
                      test  case in X. One test  case must have length equal to
                      the number of output_neurons
    eta - Learning  rate. Is  a factor to  how much  the neuron's  weights  are
          going to be changed. Must be in range (0,1]
    threshold - Error threshold.  Indicates  how low  the error  must be  to be
                for the network to be considered 'good'
    } """
    def learn(self, X, expected_output, eta=0.1, threshold=1e-2, file=''):  # {
        if self.squared_err == -1:
            self.squared_err = threshold * 2
        else:
            print('recovered with avg err: ', self.squared_err)
            sys.stdout.flush()
        # while we are not good enough
        while self.squared_err >= threshold:
            self.squared_err = 0
            for test in range(len(X)):
                x = X[test]
                y = expected_output[test]

                # run feed_forward for the test case to get activation
                # value and derivatives for all layers
                (f_h, df_h, f_o, df_o) = self.feed_forward(x)

                # calculate gradient descendent using generalized delta rule
                delta, delta_o, delta_h = self.get_deltas(y, f_o, df_h, df_o)

                # Cost function squared error = sum(||delta||^2)
                self.squared_err += np.sum(delta ** 2)

                # Update weights in the output layer
                # w'[i] = w[i] + (eta * (sum_j(delta_o[i][j] * f_h[i][j])))
                self.output_layer_weights = np.asarray(
                        self.output_layer_weights + (
                            eta * (np.asmatrix(delta_o).T @ np.asmatrix(f_h))
                            )
                        )
                # Update bias in the output layer
                # b'[i] = b[i] + (eta * delta_o[i])
                self.output_layer_bias = np.asarray(self.output_layer_bias + (
                        eta * np.asmatrix(delta_o).T  # * [1,...]
                        ))

                # Update weights in the hidden layer
                # w'[i] = w[i] + (eta * (sum_j(delta_h[i][j] * x[i][j])))
                self.hidden_layer_weights = np.asarray(
                        self.hidden_layer_weights + (
                            eta * (np.asmatrix(delta_h).T @ np.asmatrix(x))
                            )
                        )
                # Update bias in the hidden layer
                # b'[i] = b[i] + (eta * delta_h[i])
                self.hidden_layer_bias = np.asarray(self.hidden_layer_bias + (
                        eta * np.asmatrix(delta_h).T  # * [1,...]
                        ))
            # ==end for test
            self.squared_err = self.squared_err / len(X)
            if file != '':
                pickle.dump(self, open(file, 'wb'))
            print('Avg Err: ', self.squared_err)
            sys.stdout.flush()
        # ==end while
    # ==end learn }
# ==end MLP }


from __future__ import print_function
from .logger import logger
from .kinto2yaml import introspect_server

async def initialize_server(async_client, config, bucket=None, collection=None,
                            force=False, delete_missing_records=False):
    logger.debug("Converting YAML config into a server batch.")
    bid = bucket
    cid = collection
    # 1. Introspect current server state.
    if not force or delete_missing_records:
        current_server_status = await introspect_server(
            async_client,
            bucket=bucket,
            collection=collection,
            records=True
        )
        current_server_buckets = current_server_status["buckets"]
    else:
        # We don't need to load it because we will override it nevertheless.
        current_server_buckets = {}
    # 2. For each bucket
    if 'buckets' in config:
        buckets = config.get("buckets", {})
    else:  # pragma: no cover
        # Legacy for file before kinto-wizard 4.0
        logger.warning("Your file seems to be in legacy format. "
                       "Please add a `buckets:` root level.")
        buckets = config
    with async_client.batch() as batch:
        for bucket_id, bucket in buckets.items():
            # Skip buckets that we don't want to import.
            if bid and bucket_id != bid:
                logger.debug("Skip bucket {}".format(bucket_id))
                continue
            bucket_exists = bucket_id in current_server_buckets
            bucket_data = bucket.get('data', {})
            bucket_permissions = bucket.get('permissions', {})
            bucket_groups = bucket.get('groups', {})
            bucket_collections = bucket.get('collections', {})

            # Skip bucket if we don't have a collection in them
            if cid and cid not in bucket_collections:
                logger.debug("Skip bucket {}".format(bucket_id))
                continue

            if not bucket_exists:
                bucket_current_groups = {}
                bucket_current_collections = {}

                # Create the bucket if not present in the introspection
                batch.create_bucket(id=bucket_id,
                                    data=bucket_data,
                                    permissions=bucket_permissions,
                                    safe=(not force))
            else:
                current_bucket = current_server_buckets[bucket_id]
                bucket_current_groups = {}
                bucket_current_collections = {}
                current_bucket_data = {}
                current_bucket_permissions = {}

                if current_bucket:
                    bucket_current_groups = current_bucket.get('groups', {})
                    bucket_current_collections = current_bucket.get('collections', {})

                    # Patch the bucket if mandatory
                    current_bucket_data = current_bucket.get('data', {})
                    current_bucket_permissions = current_bucket.get('permissions', {})

                if (current_bucket_data != bucket_data or
                        current_bucket_permissions != bucket_permissions):
                    batch.patch_bucket(id=bucket_id,
                                       data=bucket_data,
                                       permissions=bucket_permissions)

            # 2.1 For each group, patch it if needed
            for group_id, group_info in bucket_groups.items():
                group_exists = bucket_exists and group_id in bucket_current_groups
                group_data = group_info.get('data', {})
                group_permissions = group_info.get('permissions', {})

                if not group_exists:
                    batch.create_group(id=group_id,
                                       bucket=bucket_id,
                                       data=group_data,
                                       permissions=group_permissions,
                                       safe=(not force))
                else:
                    current_group = bucket_current_groups[group_id]
                    current_group_data = current_group.get('data', {})
                    current_group_permissions = current_group.get('permissions', {})

                    if (current_group_data != group_data or
                            current_group_permissions != group_permissions):
                        batch.patch_group(id=group_id,
                                          bucket=bucket_id,
                                          data=group_data,
                                          permissions=group_permissions)

            # 2.2 For each collection patch it if mandatory
            for collection_id, collection in bucket_collections.items():
                # Skip collections that we don't want to import.
                if cid and collection_id != cid:
                    logger.debug("Skip collection {}/{}".format(bucket_id, collection_id))
                    continue
                collection_exists = bucket_exists and collection_id in bucket_current_collections
                collection_data = collection.get('data', {})
                collection_permissions = collection.get('permissions', {})

                if not collection_exists:
                    batch.create_collection(id=collection_id,
                                            bucket=bucket_id,
                                            data=collection_data,
                                            permissions=collection_permissions,
                                            safe=(not force))
                else:
                    current_collection = bucket_current_collections[collection_id]
                    current_collection_data = current_collection.get('data', {})
                    current_collection_permissions = current_collection.get('permissions', {})

                    if (current_collection_data != collection_data or
                            current_collection_permissions != collection_permissions):
                        batch.patch_collection(id=collection_id,
                                               bucket=bucket_id,
                                               data=collection_data,
                                               permissions=collection_permissions)

                # 2.2.1 For each collection, create its records.
                collection_records = collection.get('records', {})
                for record_id, record in collection_records.items():
                    record_exists = (collection_exists and
                                     record_id in current_collection.get('records', {}))
                    record_data = record.get('data', {})
                    record_permissions = record.get('permissions', None)

                    if not record_exists:
                        batch.create_record(id=record_id,
                                            bucket=bucket_id,
                                            collection=collection_id,
                                            data=record_data,
                                            permissions=record_permissions,
                                            safe=(not force))
                    else:
                        current_record = current_collection['records'][record_id]
                        current_record_data = current_record.get('data', {})
                        current_record_permissions = current_record.get('permissions', {})
                        if (current_record_data != record_data or
                                current_record_permissions != record_permissions):
                            batch.update_record(id=record_id,
                                                bucket=bucket_id,
                                                collection=collection_id,
                                                data=record_data,
                                                permissions=record_permissions)

                if delete_missing_records and collection_exists and collection_records:
                    # Fetch all records IDs
                    file_records_ids = set(collection_records.keys())
                    server_records_ids = set(current_collection['records'].keys())

                    to_delete = server_records_ids - file_records_ids
                    if not force:
                        message = ("Are you sure that you want to delete the "
                                   "following {} records?".format(len(list(to_delete))))
                        value = input(message)
                        if value.lower() not in ['y', 'yes']:
                            print("Exiting")
                            exit(1)
                    for record_id in to_delete:
                        batch.delete_record(id=record_id,
                                            bucket=bucket_id,
                                            collection=collection_id)

        logger.debug('Sending batch:\n\n%s' % batch.session.requests)
    logger.info("Batch uploaded")


from __future__ import annotations
import numpy as np
import tvm
from torch import fx
from tvm import te, relax, IRModule
from tvm import meta_schedule as ms
from typing import List, Optional, Tuple, Union
from tvm import topi
from tvm.script import relax as R
from tvm.script import tir as T
import torch.nn as nn
import torch
import sys
import builtins
mlc = sys.modules[__name__]

# ---------------------- custom te ---------------------------------------#
def mean_te(x, axis):
    shape = x.shape
    reduce_axis = []
    mean_ = 1
    for i in axis:
        reduce_axis.append(te.reduce_axis((0, shape[int(i)])))
        mean_ *= shape[int(i)]
    mean_ = te.const(int(mean_))
    sum_ = te.compute((int(shape[1]),), lambda i: te.sum(x[reduce_axis[0],i,reduce_axis[1],reduce_axis[2]], axis=reduce_axis), name='sum_out')
    out = te.compute(sum_.shape, lambda *i: sum_(*i) / mean_, name='mean_out')
    return out

def var_te(x, axis):
    shape = x.shape
    reduce_axis = []
    n_ = 1
    for i in axis:
        reduce_axis.append(te.reduce_axis((0, shape[int(i)])))
        n_ *= shape[int(i)]
    n_ = te.const(int(n_))
    mean_ = mean_te(x, axis)
    sub_ = te.compute(x.shape, lambda n, c, h, w: te.power(x[n, c, h, w] - mean_[c], 2), name='sub_out')
    sum_ = te.compute(mean_.shape, lambda i: te.sum(sub_[reduce_axis[0],i,reduce_axis[1],reduce_axis[2]], axis=reduce_axis), name='sum_out')
    var_out = te.compute(sum_.shape, lambda i: sum_[i] / n_, name='var_out')
    return var_out

# ---------------------- 构造映射函数 ---------------------------------------#
# torch graph/nn.Module --> IR high level relax.function
def map_params(param: nn.Parameter):
    return relax.const(param.data.cpu().numpy(), dtype='float32')

def fetch_attr(fx_mod, target: str):
    # 获取mod属性
    target_atoms = target.split('.')
    attr_itr = fx_mod
    for i, atom in enumerate(target_atoms):
        if not hasattr(attr_itr, atom):
            raise RuntimeError(f"Node referenced nonexistant target {'.'.join(target_atoms[:i])}")
        attr_itr = getattr(attr_itr, atom)
    return attr_itr

# call_function
def map_matmul(bb: relax.BlockBuilder, node_map, node):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    w = node_map[node.args[1]]
    return bb.emit(relax.op.matmul(x, w))

def map_relu(bb: relax.BlockBuilder, node_map, node):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    return bb.emit(relax.op.nn.relu(x))

def map_add(bb: relax.BlockBuilder, node_map, node):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    b = node_map[node.args[1]]
    if isinstance(b.struct_info, relax.TupleStructInfo):
       b = b[0]
    return bb.emit(relax.op.add(x, b))

def map_flatten(bb: relax.BlockBuilder, node_map, node):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    dims = node.args[1]
    shape = np.array([int(i) for i in x.struct_info.shape])
    new_shape = shape[:dims]
    new_shape = np.append(new_shape, shape[dims:].prod())

    return bb.emit(relax.op.reshape(x, tuple(new_shape)))

def map_other_op(bb: relax.BlockBuilder, node_map, node):
    pass

# call_module
def map_nn_relu(bb: relax.BlockBuilder, node_map, node, nn_module):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    return bb.emit(relax.op.nn.relu(x))

def map_nn_linear(bb: relax.BlockBuilder, node_map, node, nn_module):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    w = map_params(nn_module.weight)
    bias = map_params(nn_module.bias)
    return bb.emit(relax.op.linear(x, w, bias))

def map_nn_conv2d(bb: relax.BlockBuilder, node_map, node, nn_module):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    kernel = map_params(nn_module.weight)
    bias = None
    if nn_module.bias is not None:
        bias = map_params(nn_module.bias)
    strides = nn_module.stride
    padding = nn_module.padding
    dilation = nn_module.dilation
    conv_out = bb.emit(relax.op.nn.conv2d(x, kernel, strides, padding, dilation))
    if bias:
        return bb.emit(relax.op.add(conv_out, bias))
    return conv_out

def map_nn_BatchNorm2d(bb: relax.BlockBuilder, node_map, node, nn_module):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    affine = nn_module.affine
    eps = nn_module.eps

    # train stage
    # moving_mean = moving_mean * momentum + data_mean * (1 - momentum)
    # moving_var = moving_var * momentum + data_var * (1 - momentum)
    # momentum = relax.const(nn_module.momentum)
    # data_mean = bb.emit(relax.op.mean(x, [0, 2, 3]))
    # data_var = bb.emit(relax.op.variance(x, [0, 2,3]))

    # inference stage
    moving_mean = map_params(nn_module.running_mean)
    moving_var = map_params(nn_module.running_var)

    if affine:
        gamme = map_params(nn_module.weight)
        beta = map_params(nn_module.bias)

    return bb.emit(relax.op.nn.batch_norm(x, gamme, beta, moving_mean, moving_var, epsilon=eps, axis=1))

def map_nn_maxpool(bb: relax.BlockBuilder, node_map, node, nn_module):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    pool_size = nn_module.kernel_size
    strides = nn_module.stride
    padding = nn_module.padding
    return bb.emit(relax.op.nn.max_pool2d(x, pool_size, strides, padding))

def map_nn_avgpool(bb: relax.BlockBuilder, node_map, node, nn_module):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    output_size = nn_module.output_size
    return bb.emit(relax.op.nn.adaptive_avg_pool2d(x, output_size))

# call_method
def map_view(bb: relax.BlockBuilder, node_map, node):
    x = node_map[node.args[0]]
    if isinstance(x.struct_info, relax.TupleStructInfo):
        x = x[0]
    shape = node.args[1]
    return bb.emit(relax.op.reshape(x, shape))

call_map_function = {
    torch.matmul: map_matmul,
    torch.add: map_add,
    torch.relu: map_relu,
    torch.flatten: map_flatten,
    # torch.Tensor.__add__: map_add
}

call_map_module = {
    torch.nn.Linear: map_nn_linear,
    torch.nn.ReLU: map_nn_relu,
    torch.nn.Conv2d: map_nn_conv2d,
    torch.nn.BatchNorm2d: map_nn_BatchNorm2d,
    torch.nn.MaxPool2d: map_nn_maxpool,
    torch.nn.AdaptiveAvgPool2d: map_nn_avgpool,
}

call_map_method = {
    'view': map_view
}

def from_fx(fx_module: Union[torch.fx.GrapgModule, nn.Module], input_shapes,
            call_map_function=call_map_function,
            call_map_module=call_map_module,
            call_map_method=call_map_method):
    '''
        torch.fx.graph ---> high level relax function
    '''
    if isinstance(fx_module, nn.Module):
        fx_module = fx.symbolic_trace(fx_module)
    input_index = 0
    node_map = {}
    node_bn_map = {}
    named_modules = dict(fx_module.named_modules())

    bb = relax.BlockBuilder()
    fn_inputs = []
    fn_output = None
    with bb.function('main'):
        with bb.dataflow():
            for node in fx_module.graph.nodes:
                if node.op == 'placeholder':
                    input_shape = input_shapes[input_index]
                    input_index = input_index + 1
                    fn_input = relax.Var(node.target, R.Tensor(input_shape, 'float32'))
                    fn_inputs.append(fn_input)
                    node_map[node] = fn_input
                elif node.op == 'get_attr':
                    node_map[node] = map_params(fetch_attr(fx_module, node.target))
                elif node.op == 'call_function':
                    if node.target in call_map_function.keys():
                        node_map[node] = call_map_function[node.target](bb, node_map, node)
                    else:  # 内置 "+" 运算符定义错误
                        node_map[node] = call_map_function[torch.add](bb, node_map, node)
                # --------------------- add call_method ------------------------#
                elif node.op == 'call_method':
                    node_map[node] = call_map_method[node.target](bb, node_map, node)
                # --------------------------------------------------------------#
                elif node.op == 'call_module':
                    nn_module = named_modules[node.target]
                    # node_map[node] = call_map_module[nn_module](bb, node_map, node, nn_module)
                    node_map[node] = call_map_module[type(nn_module)](bb, node_map, node, nn_module)
                elif node.op == 'output':
                    output = node_map[node.args[0]]
                    if isinstance(output.struct_info, relax.TupleStructInfo):
                        output = output[0]
                    if fn_output is not None:
                        raise Warning("error")
                    fn_output = bb.emit_output(output)
        bb.emit_func_output(fn_output, fn_inputs)
    return bb.get()
# ------------------------------------------------------------- #

# ---------------------- 图优化 ---------------------------------------#
# 算子融合 high level relax function ---> high level relax function
# eg: matmul & add
def create_fuse_dense_add(call: relax.Call, value: relax.Call, func=None, fn_name=None):
    b = call.args[1]
    x = value.args[0]
    w = value.args[1]
    T_ = False

    if not isinstance(w, relax.Constant):
        value = func(w)
        # if value.op == tvm.ir.Op.get('relax.permute_dims')
        w = func(w).args[0]
        T_ = True

    params_x = relax.Var('x', x.struct_info)
    params_w = relax.Var('w', w.struct_info)
    params_b = relax.Var('b', b.struct_info)

    bb = relax.BlockBuilder()
    with bb.function(fn_name, [params_x, params_w, params_b]):
        with bb.dataflow():
            if T_:
                lv0 = bb.emit(relax.op.linear(params_x, params_w))
            else:
                lv0 = bb.emit(relax.op.matmul(params_x, params_w))

            gv = bb.emit_output(bb.emit(relax.op.add(lv0, params_b)))
        bb.emit_func_output(gv)

    fused_fn = bb.get()[fn_name].with_attr('Primitive', 1)
    return fused_fn, x, w, b

@relax.expr_functor.mutator
class DenseAddFusor(relax.PyExprMutator):
    def __init__(self, mod: Optional[IRModule] = None):
        super().__init__()
        self.mod_ = mod
        self.add_op = tvm.ir.Op.get('relax.add')
        self.permute_op = tvm.ir.Op.get('relax.permute_dims')
        self.matmul_op = tvm.ir.Op.get('relax.matmul')
        self.counter = 0

    def transform(self):
        # relax function
        for global_var, func in self.mod_.functions.items():
            if not isinstance(func, relax.Function):
                continue
            if func.attrs is not None and "Primitive" in func.attrs and func.attrs['Primitive'] != 0:
                continue

            updated_func = self.visit_expr(func)
            updated_func = relax.analysis.remove_all_unused(updated_func)
            self.builder_.update_func(global_var, updated_func)
        return self.builder_.get()

    def visit_call_(self, call):
        call = self.visit_expr_post_order(call)

        def match_call(node, op):
            if not isinstance(node, relax.Call):
                return False
            return node.op == op

        if not match_call(call, self.add_op):
            return call

        value = self.lookup_binding(call.args[0])
        if value is None or not match_call(value, self.matmul_op):
            return call

        fn_name = "fused_dense_add%d" % (self.counter)
        self.counter += 1
        fused_fn, x, w, b = create_fuse_dense_add(call, value, self.lookup_binding, fn_name)
        global_var = self.builder_.add_func(fused_fn, fn_name)

        return relax.Call(global_var, [x, w, b], None, None)

@tvm.ir.transform.module_pass(opt_level=2, name="DeseAddFuse")
class FuseDenseAddPass:
    """The wrapper for the LowerTensorIR pass."""
    def transform_module(self, mod, ctx):
        return DenseAddFusor(mod).transform()
# ------------------------------------------------------------- #

# ---------------------- 映射至TensorIR Call ---------------------------------------#
# high level relax function ---> Tensor IR function
def map_mean_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    attrs = call.attrs
    x = call.args[0]
    return bb.call_te(mean_te, x, attrs.axis)

def map_var_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    attrs = call.attrs
    x = call.args[0]
    return bb.call_te(var_te, x, attrs.axis)

def map_add_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x, b = call.args
    return bb.call_te(topi.add, x, b)

def map_matmul_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x, w = call.args
    if isinstance(w, relax.expr.DataflowVar) and (func(w).op == tvm.ir.Op.get('relax.permute_dims')):
        w = func(w).args[0]
        return bb.call_te(topi.nn.dense, x, w)
    return bb.call_te(topi.nn.matmul, x, w)

def map_relu_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x = call.args[0]
    return bb.call_te(topi.nn.relu, x)

def map_conv_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x, k = call.args
    attrs = call.attrs
    return bb.call_te(topi.nn.conv2d, x, k, attrs.strides, attrs.padding, attrs.dilation)

def map_reshape_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x, shape = call.args
    return bb.call_te(topi.reshape, x, shape)

def map_BatchNorm2d_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x, gamma, beta, moving_mean, moving_var = call.args[:5]
    attrs = call.attrs
    return bb.call_te(topi.nn.batch_norm, x, gamma, beta, moving_mean, moving_var, attrs.axis, attrs.epsilon, attrs.center, attrs.scale)

def map_maxpool2d_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x = call.args[0]
    attrs = call.attrs
    return bb.call_te(topi.nn.pool2d, x, attrs.pool_size, attrs.strides, attrs.dilation, attrs.padding, 'max', attrs.ceil_mode, attrs.layout)

def map_adaptiveAvgPool2d_te(bb: relax.BlockBuilder, call: relax.Call, func=None):
    x = call.args[0]
    attrs = call.attrs
    return bb.call_te(topi.nn.adaptive_pool, x, attrs.output_size, 'avg', attrs.layout)

op_map = {
    'relax.variance': map_var_te,
    'relax.mean': map_mean_te,
    'relax.matmul': map_matmul_te,
    'relax.add': map_add_te,
    'relax.nn.relu': map_relu_te,
    'relax.nn.conv2d': map_conv_te,
    'relax.reshape': map_reshape_te,
    'relax.nn.batch_norm': map_BatchNorm2d_te,
    'relax.nn.max_pool2d': map_maxpool2d_te,
    'relax.nn.adaptive_avg_pool2d': map_adaptiveAvgPool2d_te,
}

@relax.expr_functor.mutator
class LowerToTensorIR(relax.PyExprMutator):
    def __init__(self, mod: Optional[IRModule], op_map=op_map):
        super().__init__()
        self.mod_ = mod
        self.op_map = {
            tvm.ir.Op.get(k): v for k, v in op_map.items()
        }

    def visit_call_(self, call: relax.Call):
        call = self.visit_expr_post_order(call)

        if call.op in self.op_map:
            return self.op_map[call.op](self.builder_, call, self.lookup_binding)
        return call

    def transform(self):
        for global_val, func in self.mod_.functions.items():
            if not isinstance(func, relax.Function):
                continue
            updated_fn = self.visit_expr(func)
            updated_fn = relax.analysis.remove_all_unused(updated_fn)
            self.builder_.update_func(global_val, updated_fn)

        return self.builder_.get()

@tvm.ir.transform.module_pass(opt_level=0, name="LowerToTensorIR")
class LowerToTensorIRPass:
    """The wrapper for the LowerTensorIR pass."""
    def transform_module(self, mod, ctx):
        return LowerToTensorIR(mod, op_map).transform()

# ------------------------------------------------------------- #

# ---------------------- 算子优化 ---------------------------------------#
# 只支持一个main的IRModule优化
def mlc_tune_tir(Model: IRModule, target='cuda --max_threads_per_block=1024 --max_shared_memory_per_block=49152',
                work_dir="./tune_tmp/",
                task_name='main',
                max_trials_global=64,
                num_trials_per_iter=64,
                compile_tir_target='cuda'):
    print("target: %s; compile_tie_target: %s" % (target, compile_tir_target))
    fn_names = [x.name_hint for x in Model.functions]
    fn_names.remove('main')
    print(len(fn_names))
    for i, fn_name in enumerate(fn_names):
        print("op-%d : %s" %(i, fn_name))
        mod_ = tvm.IRModule.from_expr(Model[fn_name].with_attr("global_symbol", 'main'))

        tuned_record = ms.tune_tir(mod_, target=target,
                            work_dir=work_dir,
                            task_name=task_name,
                            max_trials_global=max_trials_global,
                            num_trials_per_iter=num_trials_per_iter)
        tuned_sch = ms.tir_integration.compile_tir(tuned_record, mod_, target=compile_tir_target)
        new_func = tuned_sch.mod['main'].with_attr("global_symbol", fn_name)
        gv = Model.get_global_var(fn_name)
        Model.update_func(gv, new_func)

    return Model