#! /bin/env python
# encoding=utf-8
# gusimiu@baidu.com
#

import sys
import copy

import numpy
import sklearn
from sklearn import cross_validation

import pydev
import mlreader

if __name__=='__main__':
    arg = pydev.Arg('A simple tool for machine learning based ont scikit-learn')
    arg.str_opt('filename', 'f', 'input file of data')
    arg.str_opt('test', 't', 'data file for test')
    arg.str_opt('reader_config', 'r', 'config of reader, format: filename#section')
    arg.bool_opt('auc', 'a', 'output auc score of features in training data.')
    opt = arg.init_arg()

    train_reader = mlreader.DataReader()
    if opt.reader_config:
        train_reader.config(opt.reader_config)
    train_reader.read(opt.filename)

    train_X = train_reader.data
    train_Y = train_reader.label
    test_X = None
    test_Y = None

    test_reader = None
    if opt.test:
        # copy the config from train_reader.
        # make two reader the same config.
        test_reader = copy.copy( train_reader )
        test_reader.clear_data()

        test_reader.read(opt.test)
        test_X = test_reader.data
        test_Y = test_reader.label
    else:
        train_X, test_X, train_Y, test_Y = cross_validation.train_test_split(
                train_X, train_Y, test_size=0.3, random_state=0)

    if opt.auc:
        print >> sys.stderr, 'Calculate AUC score'
        train_reader.auc(train_X, train_Y, file('auc.txt', 'w'))

    from sklearn import svm
    from sklearn import linear_model
    from sklearn import neighbors
    from sklearn import naive_bayes
    from sklearn import tree
    from sklearn import ensemble
    from simple_nn import SimpleNetwork
    import nnet_tf

    models = [
        #('lr', linear_model.LogisticRegression() ),
        #('knn', neighbors.KNeighborsClassifier() ),
        #('gnb', naive_bayes.GaussianNB() ),
        #('tree', tree.DecisionTreeClassifier() ),
        #('simple_gbdt', ensemble.GradientBoostingClassifier(n_estimators=50, learning_rate=0.1, max_depth=5, random_state=0) ),
        #('best_gbdt', ensemble.GradientBoostingClassifier(n_estimators=500, learning_rate=0.05, max_depth=6, random_state=0) ),
        #('ada', ensemble.AdaBoostClassifier(n_estimators=100) ),
        #('rf', ensemble.RandomForestClassifier(n_estimators=100) ),
        #('svm', svm.SVC() ),
        #('simp_nn', SimpleNetwork(len(train_X[0]), 1, [256, 256, 128], output_01=True)),
        #('simp_nn', SimpleNetwork( len(train_X[0]), 1, [12, 12], output_01=True)),
        #('simp_nn_lr', SimpleNetwork(len(train_X[0]), [], output_01=True)),
        #('fc3_nnet', nnet_tf.ConfigNetwork('conf/net.conf', 'fc3_net', output_01=True)),
        #('mnist_fc', nnet_tf.ConfigNetwork('conf/net.conf', 'mnist_fc', output_01=True)),
        #('mnist_conv2d', nnet_tf.ConfigNetwork('conf/net.conf', 'mnist_conv2d', output_01=True)),
        ('mnist_conv2d_big', nnet_tf.ConfigNetwork('conf/net.conf', 'mnist_conv2d_big', output_01=True)),
        ]

    def report(pred, label, X, reader, error_writer, out_stream):

        if len(pred.shape) > len(label.shape):
            # try to flatten.
            print >> sys.stderr, 'Force flatten! [%s] => [%s]' % (pred.shape, label.shape)
            pred.shape = label.shape

        if isinstance(label.shape, int) or len(label.shape) == 1 or label.shape[1]==1:
            true_negative = len(filter(lambda x:x==0, pred + label))
            true_positive = len(filter(lambda x:x==2, pred + label))
            false_positive = len(filter(lambda x:x==1, pred - label))
            false_negative = len(filter(lambda x:x==-1, pred - label))

            for i in range(len(pred)):
                if pred[i] != label[i]:
                    if reader is None:
                        print >> error_writer, '%d\tP=%.1f\tT=%.1f\t%s' % (
                                i, pred[i], label[i], X[i])
                    else:
                        print >> error_writer, '%d\tP=%.1f\tT=%.1f\t%s\t%s' % (
                                i, pred[i], label[i], reader.info[i], reader.text_format(X[i]))

            diff_count = len(filter(lambda x:x!=0, pred - label))
            accuracy = (len(label) - diff_count) * 100. / len(label)

            print >> out_stream, 'Accuracy : ### %.2f%% (%d/%d) ###' % (accuracy, len(label)-diff_count, len(label))
            print >> out_stream, '     P: %.3f%%   R: %.3f%%' % (
                                    100. * true_positive / (true_positive + false_positive),
                                    100. * true_positive / (true_positive + false_negative)
                                    )

            print >> out_stream, ' Positive: true:%d false:%d' % (true_positive, false_positive)
            print >> out_stream, ' Negative: true:%d false:%d' % (true_negative, false_negative)

        else:
            same_count = len(filter(lambda x:(x[0]-x[1]).dot(x[0]-x[1])<1e-4, zip(pred, label)))
            accuracy = same_count * 100. / len(label)
            print >> out_stream, 'Accuracy : ### %.2f%% (%d/%d) ###' % (accuracy, same_count, len(label))

    for name, model in models:
        print >> sys.stderr, '====> model [%s] <====' % name

        model.fit(train_X, train_Y)
        print >> sys.stderr, 'Training over.'

        # TEST: training-set
        # test for best performance(ignore overfitting.).
        pred = model.predict(train_X)
        print >> sys.stderr, 'Predict [Training-set] over.'
        report(pred, train_Y, train_X, train_reader, file('error_of_train.txt', 'w'), sys.stderr)

        # TEST: test-set.
        pred = model.predict(test_X)
        print >> sys.stderr, 'Predict [Test-set] over.'
        report(pred, test_Y, test_X, test_reader, file('error_of_test.txt', 'w'), sys.stderr)


import re

with open("cdpNeighborsDetails.txt", "r") as file:
    for line in file:

        device_id = re.search(r'(Device ID: )(.*)', line)
        if device_id:
            print(device_id.group())

        ip_address = re.search(r'(IP address: )(.*)', line)
        if ip_address:
            print(ip_address.group())

        interface_and_port = re.search(r'(Interface: )([a-zA-Z0-9]*/[0-9]*, )( Port ID \(outgoing port\):)( [a-zA-Z0-9]*/[0-9]*)', line)
        if interface_and_port:
            print(interface_and_port.group())

####################
## Device ID:
## IP address:
## Interface:
## Port ID (outgoing port):


# Выполнить эти комманды в консоли питона перед запуском, загрузка модулей nltk
# import nltk
# nltk.download('punkt')
# nltk.download('stopwords')

import csv
import string
import pymorphy2
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.stem import SnowballStemmer
import pandas as pd

# Предварительные настройки
stop_words = stopwords.words("russian")  # nltk загрузка стоп-слов
snowball = SnowballStemmer(language="russian")  # nltk стемматизатор
morph = pymorphy2.MorphAnalyzer(lang='ru')  # Морфический анализатор

#  text = 'В МЭР рассказали о проблемах в реализации совместных проектов с Кубой'

def tokenizer(text):
    result = []
    for element in sent_tokenize(text, language="russian"):  # Токенизация предложения
        tokens = word_tokenize(element, language="russian")  # Токенизация слов
        result_tokens = []
        tokens = [''.join(c for c in s if c not in string.punctuation) for s in tokens]  # удаляем знаки препинания
        tokens = [s for s in tokens if s]  # удаляем пустые элементы
        for token in tokens:
            if token not in stop_words and token != "—" and token != "–":  # удаление стоп-слов и дефисов
                token = morph.parse(token)[0].normal_form  # Нормализация
                token = snowball.stem(token)  # Стемматизация
                result_tokens.append(token)
        result.append(result_tokens)
    #  if len(result) > 1:  # проверка на мультитокеновость
    print(result)
    return result

"""
# Тестовые данные
with open('input.csv', newline='', encoding="UTF-8") as inputfile:
    spamreader = csv.reader(inputfile, delimiter='|')
    for row in spamreader:
        if row:
            linkrow = row[0]
            textrow = row[1]
            daterow = row[2]
            tokenizer(textrow)
            print(linkrow, textrow, daterow)
"""

df = pd.read_csv("input2.csv", error_bad_lines=False, names=["link", "text", "date"])
products_list = df.values.tolist()
for row in products_list:
    if row:
        linkrow = row[0]
        textrow = row[1]
        daterow = row[2]
        tokenized = tokenizer(textrow)
        print(linkrow, textrow, daterow)


from django.urls import path

from .views.categories import CategoryCreateAndListView, CategoryDetailView, FilterCategoryView
from .views.products import FilterProductView, ProductCreateAndListView, ProductDetailView

urlpatterns = [
    path('products', ProductCreateAndListView.as_view(), name='product_list'),
    path('products/filter', FilterProductView.as_view()),
    path('products/<int:pk>', ProductDetailView.as_view()),
    path('categories', CategoryCreateAndListView.as_view(), name='category_list'),
    path('categories/filter', FilterCategoryView.as_view()),
    path('categories/<int:pk>', CategoryDetailView.as_view()),
]


from . import Setting

from PIL import Image

class CameraSetting(Setting) :

    default_settings = {
        'saturation' : 0,
        'contrast' : 0,
        'brightness' : 0,
        'sharpness' : 0,
        'iso' : 0,
        'historize' : True
    }

    def post_setting(self,photo) :
        #return photo.resize(self.size, Image.ANTIALIAS)
        pass

    def set_options(self,options) :
        for option, value in options.items() :
            if option in self.default_settings :
                print ('set {} to {}'.format(option,value))
                self.camera.setSetting(option,value)
            else :
                print ('Invalid options',option)

    def reset(self) :
        self.set_options(self.default_settings)


from scrape.image_scrape import ImageScraping
from utils.cmd_args import CommandLineArgs
import os

'''
setup global variable parameters
'''
CMD_ARGS = CommandLineArgs.scrape_args(logtype='.log')

ENGINE = CMD_ARGS.engine
SAVEDIR = os.path.abspath(CMD_ARGS.savedir)
KEYWORD = CMD_ARGS.keyword
N_IMAGES = CMD_ARGS.num_images
SEED_FLAG = CMD_ARGS.seed

if SEED_FLAG:
    SEED = 100 # ご自由に変更してください
else:
    SEED = None

if __name__ == '__main__':
    img_scrape = ImageScraping(site=ENGINE, seed=SEED, savedir=SAVEDIR)
    img_scrape.run_scrape(keyword=KEYWORD, n_img=N_IMAGES)


import requests
from requests_html import HTMLSession

import json
import time

def get_yahoo_finance_USD_SGD_rate():
    """ Get USDSGD exchange rate from yahoo finance by scapping the website """
    url = "https://sg.finance.yahoo.com/quote/USDSGD=X/"
    try:
        session = HTMLSession()
        response = session.get(url)
    except requests.exceptions.RequestException as e:
        print(e)
    USDSGD = float(
        response.html.xpath(
            "//fin-streamer[@data-field='regularMarketPrice'][@data-symbol='SGD=X']")[0].text)
    print(USDSGD)
    return (USDSGD)

def get_mas_USD_SGD_rate():
    """ Get USDSGD exchange rate from MAS API """
    api_link = "https://eservices.mas.gov.sg/api/action/datastore/search.json?resource_id=95932927-c8bc-4e7a-b484-68a66a24edfe&fields=usd_sgd,end_of_day&limit=1&sort=end_of_day%20desc"
    r = requests.get(api_link).json()
    while r["success"] != True:
        print("Failed to get USDSGD rate. Trying again...")
        r = requests.get(api_link).json()
    USDSGD = float(r["result"]["records"][0]["usd_sgd"])

    return USDSGD

def get_price(coin_id_param):
    """ Gets the current price of active coins from CoinGecko API """
    currencyParam = ','.join(['usd', 'sgd'])
    payload = {'ids': coin_id_param, 'vs_currencies': currencyParam}
    r = requests.get('https://api.coingecko.com/api/v3/simple/price',
                     params=payload)
    return r.json()

def get_price_all(coin_id_param):
    """ Gets the current price of all coins (including inactive ones) from CoinGecko API """
    currencyParam = ','.join(['usd', 'sgd'])
    payload = {'ids': coin_id_param, 'vs_currencies': currencyParam}
    r = requests.get('https://api.coingecko.com/api/v3/simple/price',
                     params=payload)
    return r.json()

def getHistoricalPrice(coinID, date, currency='usd'):
    #coingecko format: dd-mm-yyyy
    query_date = date[:2] + '-' + date[3:5] + '-' + date[6:]
    r = requests.get(
        'https://api.coingecko.com/api/v3/coins/{}/history?date={}&localization=false'
        .format(coinID, query_date))
    success = False
    price = 0
    if r.status_code == 404:
        return 0
    while not success:
        try:
            price = r.json()['market_data']['current_price'][currency]
            success = True
        except json.decoder.JSONDecodeError:
            print('Json decoder error! Trying again...')
            time.sleep(5)
            r = requests.get(
                'https://api.coingecko.com/api/v3/coins/{}/history?date={}&localization=false'
                .format(coinID, query_date))
    return price


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

pd.set_option('display.max_columns', None)
pd.set_option("display.width", 500)

titanic = sns.load_dataset("titanic")
df = titanic.copy()
df.head()
df.tail()
df.shape
df.info()
df.index
df.columns
df.describe().T
df.isnull().values.any()
df.isnull().sum()

def check_df(data_frame, head=5):
    print("##################### Shape #####################")
    print(data_frame.shape)
    print("##################### Types #####################")
    print(data_frame.dtypes)
    print("##################### Head #####################")
    print(data_frame.head(head))
    print("##################### Tail #####################")
    print(data_frame.tail(head))
    print("##################### NA #####################")
    print(data_frame.isnull().sum())
    print("##################### Describe #####################")
    print(data_frame.describe().T)

check_df(df)

tips = sns.load_dataset("tips")
check_df(tips)


# flake8: noqa: F841
import collections
import tempfile
from pathlib import Path
from typing import List, Mapping, Any, Hashable, DefaultDict

from pandas._typing import Scalar, ArrayLike

import pandas as pd
import numpy as np

from pandas.core.window import ExponentialMovingWindow

def test_types_init() -> None:
    pd.Series(1)
    pd.Series((1, 2, 3))
    pd.Series(np.array([1, 2, 3]))
    pd.Series(data=[1, 2, 3, 4], name="series")
    pd.Series(data=[1, 2, 3, 4], dtype=np.int8)
    pd.Series(data={'row1': [1, 2], 'row2': [3, 4]})
    pd.Series(data=[1, 2, 3, 4], index=[4, 3, 2, 1], copy=True)

def test_types_any() -> None:
    res1: bool = pd.Series([False, False]).any()
    res2: bool = pd.Series([False, False]).any(bool_only=False)
    res3: bool = pd.Series([np.nan]).any(skipna=False)

def test_types_all() -> None:
    res1: bool = pd.Series([False, False]).all()
    res2: bool = pd.Series([False, False]).all(bool_only=False)
    res3: bool = pd.Series([np.nan]).all(skipna=False)

def test_types_csv() -> None:
    s = pd.Series(data=[1, 2, 3])
    csv_df: str = s.to_csv()

    with tempfile.NamedTemporaryFile() as file:
        s.to_csv(file.name)
        s2: pd.DataFrame = pd.read_csv(file.name)

    with tempfile.NamedTemporaryFile() as file:
        s.to_csv(Path(file.name))
        s3: pd.DataFrame = pd.read_csv(Path(file.name))

    # This keyword was added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
    with tempfile.NamedTemporaryFile() as file:
        s.to_csv(file.name, errors='replace')
        s4: pd.DataFrame = pd.read_csv(file.name)

def test_types_copy() -> None:
    s = pd.Series(data=[1, 2, 3, 4])
    s2: pd.Series = s.copy()

def test_types_select() -> None:
    s = pd.Series(data={'row1': 1, 'row2': 2})
    s[0]
    s[1:]

def test_types_iloc_iat() -> None:
    s = pd.Series(data={'row1': 1, 'row2': 2})
    s2 = pd.Series(data=[1, 2])
    s.loc['row1']
    s.iat[0]
    s2.loc[0]
    s2.iat[0]

def test_types_loc_at() -> None:
    s = pd.Series(data={'row1': 1, 'row2': 2})
    s2 = pd.Series(data=[1, 2])
    s.loc['row1']
    s.at['row1']
    s2.loc[1]
    s2.at[1]

def test_types_boolean_indexing() -> None:
    s = pd.Series([0, 1, 2])
    s[s > 1]
    s[s]

def test_types_df_to_df_comparison() -> None:
    s = pd.Series(data={'col1': [1, 2]})
    s2 = pd.Series(data={'col1': [3, 2]})
    res_gt: pd.Series = s > s2
    res_ge: pd.Series = s >= s2
    res_lt: pd.Series = s < s2
    res_le: pd.Series = s <= s2
    res_e: pd.Series = s == s2

def test_types_head_tail() -> None:
    s = pd.Series([0, 1, 2])
    s.head(1)
    s.tail(1)

def test_types_sample() -> None:
    s = pd.Series([0, 1, 2])
    s.sample(frac=0.5)
    s.sample(n=1)

def test_types_nlargest_nsmallest() -> None:
    s = pd.Series([0, 1, 2])
    s.nlargest(1)
    s.nlargest(1, 'first')
    s.nsmallest(1, 'last')
    s.nsmallest(1, 'all')

def test_types_filter() -> None:
    s = pd.Series(data=[1, 2, 3, 4], index=['cow', 'coal', 'coalesce', ''])
    s.filter(items=['cow'])
    s.filter(regex='co.*')
    s.filter(like='al')

def test_types_setting() -> None:
    s = pd.Series([0, 1, 2])
    s[3] = 4
    s[s == 1] = 5
    s[:] = 3

def test_types_drop() -> None:
    s = pd.Series([0, 1, 2])
    res: pd.Series = s.drop(0)
    res2: pd.Series = s.drop([0, 1])
    res3: pd.Series = s.drop(0, axis=0)
    res4: None = s.drop([0, 1], inplace=True, errors='raise')
    res5: None = s.drop([0, 1], inplace=True, errors='ignore')

def test_types_drop_multilevel() -> None:
    index = pd.MultiIndex(levels=[['top', 'bottom'], ['first', 'second', 'third']],
                          codes=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]])
    s = pd.Series(data=[1, 2, 3, 4, 5, 6], index=index)
    res: pd.Series = s.drop(labels='first', level=1)

def test_types_dropna() -> None:
    s = pd.Series([1, np.nan, np.nan])
    res: pd.Series = s.dropna()
    res2: None = s.dropna(axis=0, inplace=True)

def test_types_fillna() -> None:
    s = pd.Series([1, np.nan, np.nan, 3])
    res: pd.Series = s.fillna(0)
    res2: pd.Series = s.fillna(0, axis='index')
    res3: pd.Series = s.fillna(method='backfill', axis=0)
    res4: None = s.fillna(method='bfill', inplace=True)
    res5: pd.Series = s.fillna(method='pad')
    res6: pd.Series = s.fillna(method='ffill', limit=1)

def test_types_sort_index() -> None:
    s = pd.Series([1, 2, 3], index=[2, 3, 1])
    res: pd.Series = s.sort_index()
    res2: None = s.sort_index(ascending=False, inplace=True)
    res3: pd.Series = s.sort_index(kind="mergesort")

# This was added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
def test_types_sort_index_with_key() -> None:
    s = pd.Series([1, 2, 3], index=['a', 'B', 'c'])
    res: pd.Series = s.sort_index(key=lambda k: k.str.lower())

def test_types_sort_values() -> None:
    s = pd.Series([4, 2, 1, 3])
    res: pd.Series = s.sort_values(0)
    res2: pd.Series = s.sort_values(ascending=False)
    res3: None = s.sort_values(inplace=True, kind='quicksort')
    res4: pd.Series = s.sort_values(na_position='last')
    res5: pd.Series = s.sort_values(ignore_index=True)

# This was added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
def test_types_sort_values_with_key() -> None:
    s = pd.Series([1, 2, 3], index=[2, 3, 1])
    res: pd.Series = s.sort_values(key=lambda k: -k)

def test_types_shift() -> None:
    s = pd.Series([1, 2, 3])
    s.shift()
    s.shift(axis=0, periods=1)
    s.shift(-1, fill_value=0)

def test_types_rank() -> None:
    s = pd.Series([1, 1, 2, 5, 6, np.nan, 'milion'])
    s.rank()
    s.rank(axis=0, na_option='bottom')
    s.rank(method="min", pct=True)
    s.rank(method="dense", ascending=True)
    s.rank(method="first", numeric_only=True)

def test_types_mean() -> None:
    s = pd.Series([1, 2, 3, np.nan])
    f1: float = s.mean()
    s1: pd.Series = s.mean(axis=0, level=0)
    f2: float = s.mean(skipna=False)
    f3: float = s.mean(numeric_only=False)

def test_types_median() -> None:
    s = pd.Series([1, 2, 3, np.nan])
    f1: float = s.median()
    s1: pd.Series = s.median(axis=0, level=0)
    f2: float = s.median(skipna=False)
    f3: float = s.median(numeric_only=False)

def test_types_sum() -> None:
    s = pd.Series([1, 2, 3, np.nan])
    s.sum()
    s.sum(axis=0, level=0)
    s.sum(skipna=False)
    s.sum(numeric_only=False)
    s.sum(min_count=4)

def test_types_cumsum() -> None:
    s = pd.Series([1, 2, 3, np.nan])
    s.cumsum()
    s.cumsum(axis=0)
    s.cumsum(skipna=False)

def test_types_min() -> None:
    s = pd.Series([1, 2, 3, np.nan])
    s.min()
    s.min(axis=0)
    s.min(level=0)
    s.min(skipna=False)

def test_types_max() -> None:
    s = pd.Series([1, 2, 3, np.nan])
    s.max()
    s.max(axis=0)
    s.max(level=0)
    s.max(skipna=False)

def test_types_quantile() -> None:
    s = pd.Series([1, 2, 3, 10])
    s.quantile([0.25, 0.5])
    s.quantile(0.75)
    s.quantile()
    s.quantile(interpolation='nearest')

def test_types_clip() -> None:
    s = pd.Series([-10, 2, 3, 10])
    s.clip(lower=0, upper=5)
    s.clip(lower=0, upper=5, inplace=True)

def test_types_abs() -> None:
    s = pd.Series([-10, 2, 3, 10])
    s.abs()

def test_types_var() -> None:
    s = pd.Series([-10, 2, 3, 10])
    s.var()
    s.var(axis=0, ddof=1)
    s.var(skipna=True, numeric_only=False)

def test_types_std() -> None:
    s = pd.Series([-10, 2, 3, 10])
    s.std()
    s.std(axis=0, ddof=1)
    s.std(skipna=True, numeric_only=False)

def test_types_idxmin() -> None:
    s = pd.Series([-10, 2, 3, 10])
    s.idxmin()
    s.idxmin(axis=0)

def test_types_idxmax() -> None:
    s = pd.Series([-10, 2, 3, 10])
    s.idxmax()
    s.idxmax(axis=0)

def test_types_value_counts() -> None:
    s = pd.Series([1, 2])
    s.value_counts()

def test_types_unique() -> None:
    s = pd.Series([-10, 2, 2, 3, 10, 10])
    s.unique()

def test_types_apply() -> None:
    s = pd.Series([-10, 2, 2, 3, 10, 10])
    s.apply(lambda x: x ** 2)
    s.apply(np.exp)
    s.apply(str)

def test_types_element_wise_arithmetic() -> None:
    s = pd.Series([0, 1, -10])
    s2 = pd.Series([7, -5, 10])

    res_add1: pd.Series = s + s2
    res_add2: pd.Series = s.add(s2, fill_value=0)

    res_sub: pd.Series = s - s2
    res_sub2: pd.Series = s.sub(s2, fill_value=0)

    res_mul: pd.Series = s * s2
    res_mul2: pd.Series = s.mul(s2, fill_value=0)

    res_div: pd. Series = s / s2
    res_div2: pd. Series = s.div(s2, fill_value=0)

    res_floordiv: pd.Series = s // s2
    res_floordiv2: pd.Series = s.floordiv(s2, fill_value=0)

    res_mod: pd.Series = s % s2
    res_mod2: pd.Series = s.mod(s2, fill_value=0)

    res_pow: pd.Series = s ** abs(s2)
    res_pow2: pd.Series = s.pow(abs(s2), fill_value=0)

def test_types_scalar_arithmetic() -> None:
    s = pd.Series([0,1,-10])

    res_add1: pd.Series = s + 1
    res_add2: pd.Series = s.add(1, fill_value=0)

    res_sub: pd.Series = s - 1
    res_sub2: pd.Series = s.sub(1, fill_value=0)

    res_mul: pd.Series = s * 2
    res_mul2: pd.Series = s.mul(2, fill_value=0)

    res_div: pd. Series = s / 2
    res_div2: pd. Series = s.div(2, fill_value=0)

    res_floordiv: pd.Series = s // 2
    res_floordiv2: pd.Series = s.floordiv(2, fill_value=0)

    res_mod: pd.Series = s % 2
    res_mod2: pd.Series = s.mod(2, fill_value=0)

    res_pow: pd.Series = s ** 2
    res_pow1: pd.Series = s ** 0
    res_pow2: pd.Series = s ** 0.213
    res_pow3: pd.Series = s.pow(0.5)

def test_types_groupby() -> None:
    s = pd.Series([4, 2, 1, 8], index=['a', 'b', 'a', 'b'])
    s.groupby(['a', 'b', 'a', 'b'])
    s.groupby(level=0)
    s.groupby(s > 2)

# This added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
def test_types_group_by_with_dropna_keyword() -> None:
    s = pd.Series([1, 2, 3, 3], index=['col1', 'col2', 'col3', np.nan])
    s.groupby(level=0, dropna=True).sum()
    s.groupby(level=0, dropna=False).sum()
    s.groupby(level=0).sum()

def test_types_plot() -> None:
    s = pd.Series([0, 1, 1, 0, -10])
    s.plot.hist()

def test_types_window() -> None:
    s = pd.Series([0, 1, 1, 0, 5, 1, -10])
    s.expanding()
    s.expanding(axis=0, center=True)

    s.rolling(2)
    s.rolling(2, axis=0, center=True)

def test_types_cov() -> None:
    s1 = pd.Series([0, 1, 1, 0, 5, 1, -10])
    s2 = pd.Series([0, 2, 12, -4, 7, 9, 2])
    s1.cov(s2)
    s1.cov(s2, min_periods=1)
    # ddof param was added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
    s1.cov(s2, ddof=2)

def test_update() -> None:
    s1 = pd.Series([0, 1, 1, 0, 5, 1, -10])
    s1.update(pd.Series([0, 2, 12]))
    # Series.update() accepting objects that can be coerced to a Series was added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
    s1.update([1, 2, -4, 3])
    s1.update([1, "b", "c", "d"])
    s1.update({1: 9, 3: 4})

def test_to_markdown() -> None:
    s = pd.Series([0, 1, 1, 0, 5, 1, -10])
    s.to_markdown()
    s.to_markdown(buf=None, mode="wt")
    # index param was added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
    s.to_markdown(index=False)

# compare() method added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
def test_types_compare() -> None:
    s1 = pd.Series([0, 1, 1, 0, 5, 1, -10])
    s2 = pd.Series([0, 2, 12, -4, 7, 9, 2])
    s1.compare(s2)
    s2.compare(s1, align_axis='columns', keep_shape=True, keep_equal=True)

def test_types_agg() -> None:
    s = pd.Series([1, 2, 3], index=['col1', 'col2', 'col3'])
    s.agg("min")
    s.agg(x=max, y='min', z=np.mean)
    s.agg("mean", axis=0)

def test_types_describe() -> None:
    s = pd.Series([1, 2, 3, np.datetime64("2000-01-01")])
    s.describe()
    s.describe(percentiles=[0.5], include='all')
    s.describe(exclude=np.number)
    # datetime_is_numeric param added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
    s.describe(datetime_is_numeric=True)

def test_types_resample() -> None:
    s = pd.Series(range(9), index=pd.date_range('1/1/2000', periods=9, freq='T'))
    s.resample('3T').sum()
    # origin and offset params added in 1.1.0 https://pandas.pydata.org/docs/whatsnew/v1.1.0.html
    s.resample('20min', origin='epoch', offset=pd.Timedelta(value=2, unit='minutes'))

# set_flags() method added in 1.2.0 https://pandas.pydata.org/docs/whatsnew/v1.2.0.html
def test_types_set_flags() -> None:
    pd.Series([1, 2], index=['a', 'b']).set_flags(allows_duplicate_labels=False)
    pd.Series([3, 4], index=['a', 'a']).set_flags(allows_duplicate_labels=True)
    pd.Series([5, 2], index=['a', 'a'])

def test_types_getitem() -> None:
    s = pd.Series({'key': [0, 1, 2, 3]})
    key: List[int] = s['key']
    s2 = pd.Series([0, 1, 2, 3])
    value: int = s2[0]
    s3: pd.Series = s[:2]

def test_types_eq() -> None:
    s1 = pd.Series([1, 2, 3])
    res1: pd.Series = s1 == 1
    s2 = pd.Series([1, 2, 4])
    res2: pd.Series = s1 == s2

def test_types_rename_axis() -> None:
    s: pd.Series = pd.Series([1, 2, 3]).rename_axis("A")

def test_types_values() -> None:
    n1: ArrayLike = pd.Series([1, 2, 3]).values
    n2: ArrayLike = pd.Series(list('aabc')).values
    n3: ArrayLike = pd.Series(list('aabc')).astype('category').values
    n4: ArrayLike = pd.Series(pd.date_range('20130101', periods=3, tz='US/Eastern')).values

def test_types_rename() -> None:
    # Scalar
    s1 = pd.Series([1, 2, 3]).rename("A")
    # Hashable Sequence
    s2 = pd.Series([1, 2, 3]).rename(("A", "B"))
    # Optional
    s3 = pd.Series([1, 2, 3]).rename(None)

    # Functions
    def add1(x: int) -> int:
        return x + 1

    s4 = pd.Series([1, 2, 3]).rename(add1)

    # Dictionary
    s5 = pd.Series([1, 2, 3]).rename({1: 10})
    # inplace
    s6: None = pd.Series([1, 2, 3]).rename("A", inplace=True)

def test_types_ne() -> None:
    s1 = pd.Series([1, 2, 3])
    s2 = pd.Series([1, 2, 4])
    s3: pd.Series = s1 != s2

def test_types_bfill() -> None:
    s1 = pd.Series([1, 2, 3])
    s2: pd.Series = s1.bfill(inplace=False)
    s3: None = s1.bfill(inplace=True)

def test_types_ewm() -> None:
    s1 = pd.Series([1, 2, 3])
    w1: ExponentialMovingWindow = s1.ewm(com=0.3, min_periods=0, adjust=False, ignore_na=True, axis=0)
    w2: ExponentialMovingWindow = s1.ewm(alpha=0.4)
    w3: ExponentialMovingWindow = s1.ewm(span=1.6)
    w4: ExponentialMovingWindow = s1.ewm(halflife=0.7)

def test_types_ffill() -> None:
    s1 = pd.Series([1, 2, 3])
    s2: pd.Series = s1.ffill(inplace=False)
    s3: None = s1.ffill(inplace=True)

def test_types_as_type() -> None:
    s1 = pd.Series([1, 2, 8, 9])
    s2: pd.Series = s1.astype('int32')

def test_types_dot() -> None:
    s1 = pd.Series([0, 1, 2, 3])
    s2 = pd.Series([-1, 2, -3, 4])
    df1 = pd.DataFrame([[0, 1], [-2, 3], [4, -5], [6, 7]])
    n1 = np.array([[0, 1], [1, 2], [-1, -1], [2, 0]])
    sc1: Scalar = s1.dot(s2)
    sc2: Scalar = s1 @ s2
    s3: pd.Series = s1.dot(df1)
    s4: pd.Series = s1 @ df1
    n2: np.ndarray = s1.dot(n1)
    n3: np.ndarray = s1 @ n1

def test_to_dict() -> None:
    s = pd.Series([[0, 1]])

    # Complete default
    no_params: Mapping[Hashable, Any] = s.to_dict()

    # Other types
    ordered_dict: collections.OrderedDict[Hashable, Any] = s.to_dict(into=collections.OrderedDict)

    # collections.defaultdict is a special case
    default_dict: DefaultDict[Hashable, Any] = s.to_dict(into=collections.defaultdict(list))

def test_index_assignment() -> None:
    s = pd.Series([[0, 1]])
    s.index = pd.Index(['a'])


import pygame
class Button:
    def __init__(self,pos,image,action):
        self.action=action
        self.pos=pos
        self.image=image
        self.rect=self.image.get_rect()
        self.rect.center=pos
        self.clicked=False
    def draw(self,surface):
        surface.blit(self.image,self.rect)
        pos = pygame.mouse.get_pos()
        if self.rect.collidepoint(pos):
            if pygame.mouse.get_pressed()[0] == 1 and self.clicked==False:
                self.clicked=True
                return True
        if pygame.mouse.get_pressed()[0] == 0:
            self.clicked=False
        return False


from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import re

# dynamic web scrapping using selenium
def createDriver():
    # define route
    route = ChromeDriverManager(path="./chromedriver").install()

    # define service
    serv = Service(route)

    # define options
    chr_options = Options()
    chr_options.add_experimental_option("detach", True)
    user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36"
    chr_options.add_argument(f"user-agent={user_agent}")
    chr_options.add_argument("--start-maximized")

    # disable same-origin-policy
    chr_options.add_argument("--disable-web-security")
    chr_options.add_argument("--disable-extensions")
    chr_options.add_argument("--disable-notifications")

    # ignore secure connection modal
    chr_options.add_argument("--ignore-certificate-errors")
    chr_options.add_argument("--no-sandbox")
    chr_options.add_argument("--log-level=3")
    chr_options.add_argument("--allow-running-insecure-content")
    chr_options.add_argument("--no-default-browser-check")

    # avoid unnecessary first time tasks
    chr_options.add_argument("--no-first-run")

    # direct connection
    chr_options.add_argument("--no-proxy-server")
    chr_options.add_argument("--disable-blink-features=AutomationControlled")

    except_opt = [
        'enable_automation',
        'ignore-certificate-errors',
        'enable-logging'
    ]

    pref = {
        "profile.default_content_setting_values.notifications": 2,
        "intl.accept_languages": ["es-ES", "es"],
        "credentials_enable_service": False
    }

    chr_options.add_experimental_option("excludeSwitches", except_opt)
    chr_options.add_experimental_option("prefs", pref)

    # instantiate driver
    driver = webdriver.Chrome(service=serv, options=chr_options)

    return driver

if __name__ == '__main__':
    url = "https://www.amazon.es/Elgato-Facecam-videoconferencia-enfoque-fijo-Zoom/dp/B0973DV11T/ref=psdc_937753031_t1_B08LVNYYZH"

    # initiate driver in url
    mydriver = createDriver()
    mydriver.get(url)
    mydriver.implicitly_wait(7)

    # get relevant data
    productName = mydriver.find_element(By.CSS_SELECTOR, "span#productTitle").text
    realPrice = mydriver.find_element(By.CSS_SELECTOR, "div#corePrice_feature_div").text.replace(",", ".")
    discount = mydriver.find_element(By.XPATH, "//div[@class='a-section a-spacing-none aok-align-center']/span[1]").text
    nReviews = mydriver.find_element(By.CSS_SELECTOR, "div#averageCustomerReviews").text

    print(nReviews)


import abc
from copy import deepcopy
from inspect import isclass
from typing import Any, Dict, List, Optional, Set, Type, Union

FlatDict = Dict[str, Any]
Config = Dict[str, Any]

class ConfigManager:

    def __init__(
        self,
        init_cfg: Config,
        components: Optional[List[Union[Type["ConfigComponent"], "ConfigComponent"]]] = None
    ):
        self.init_cfg = init_cfg
        self._components: List[Union[Type["ConfigComponent"], "ConfigComponent"]] = []

        if components is not None:
            for component in components:
                self.add_component(component)

    def add_component(self, component: Union[Type["ConfigComponent"], "ConfigComponent"]) -> None:
        if any(c.name() == component.name() for c in self._components):
            raise ValueError(f"Duplicate '{component.name}' ConfigComponent")

        self._components.append(component)
        component.manager = self

    @property
    def components(self) -> List[Union[Type["ConfigComponent"], "ConfigComponent"]]:
        return self._components

    def configure_config(self) -> Config:
        cfg = deepcopy(self.init_cfg)

        for component in self.components:
            if isclass(component):
                component_obj = component()
            else:
                component_obj = component

            if component_obj.is_enabled(cfg):
                cfg.update(component_obj.configure(cfg))

        return cfg

class ConfigComponent(abc.ABC):

    def __init__(self):
        self._manager: Optional["ConfigManager"] = None

    @property
    def manager(self) -> "ConfigManager":
        if self._manager is None:
            raise RuntimeError("")

        return self._manager

    @manager.setter
    def manager(self, manager: "ConfigManager") -> None:
        self._manager = manager

    def is_enabled(self, cfg: Config) -> bool:
        return True

    @classmethod
    def name(cls) -> str:
        return cls.__name__

    @abc.abstractmethod
    def configure(self, cfg: Config) -> FlatDict:
        pass

class Component1(ConfigComponent):

    def configure(self, cfg) -> FlatDict:
        return {"foo": "bar"}

class Component2(ConfigComponent):

    def configure(self, cfg) -> FlatDict:
        return {cfg["foo"]: "baz"}

if __name__ == "__main__":
    c1 = Component1()
    cm = ConfigManager(init_cfg={}, components=[c1, Component2])
    cfg = cm.configure_config()
    print(cfg)


#!/usr/bin/python

import wx

class Host:
    def __init__(self, ctrl, name):
        self._ctrl = ctrl
        self._name = name
        self._client = None

    def setName(self, name):
        self._name = name

    def getName(self):
        return self._name

    def removeMe(self, event):
        self._ctrl.removeNode()
        self._parent.Dismiss()

    def addTraffic(self, event):
        self._ctrl.addTraffic()
        self._parent.Dismiss()

    def getPanel(self, parent):
        panel = wx.Panel(parent, -1)
        a = wx.Button(panel, -1, "Add traffic", (1, 1))
        self._parent = parent
        wx.EVT_BUTTON(panel, a.GetId(), self.addTraffic)
        sz = a.GetBestSize()
        b = wx.Button(panel, -1, "Remove me", (1, sz.height))
        wx.EVT_BUTTON(panel, b.GetId(), self.removeMe)
        bsz = b.GetBestSize()
        panel.SetSize( (max(sz.width, bsz.width), sz.height + bsz.height) )
        return panel

    def assign(self, event):
        self._client = event.GetClientData()

    def getInfoPanel(self, parent):
        wx.StaticText(parent, -1, "Host ", wx.Point(5,5))
        wx.StaticText(parent, -1, "   Name: " + self.getName(), wx.Point(5,20))
        wx.StaticText(parent, -1, "   Assigned to: ", wx.Point(5,39))
        tID = wx.NewId()
        choice = wx.Choice(parent, tID, wx.Point(100, 35), size = wx.Size(105, 22), choices = [])
        wx.EVT_CHOICE(parent, tID, self.assign)
        id = choice.Append(" <None> ")
        choice.SetClientData(id, None)
        choice.SetSelection(id)
        for item in self._ctrl.getClients():
            id = choice.Append(str(item[1][1]))
            choice.SetClientData(id, item[0])
            if self._client == item[0]:
                choice.SetSelection(id)

    def getAssignedClient(self):
        return self._client

    def isValidClientAssigned(self):
        for item in self._ctrl.getClients():
            if self._client == item[0]:
                return True
        return False


import tkinter as tk

import subprocess

from tkinter import messagebox

from tkinter import filedialog

class Application(tk.Frame):

    def __init__(self, master=None):

        super().__init__(master)

        self.master = master

        self.master.title("KAK- Ravaan*(Private Tool- MADE BY RAVAAN:))")

        self.pack()

        self.create_widgets()

        self.generated_urls = ''

        self.original_urls = ''

        # Set window properties

        self.master.resizable(width=False, height=False)

        self.master.attributes('-fullscreen', False)

    def create_widgets(self):

        # Create a frame for the URL input

        url_frame = tk.Frame(self)

        url_frame.pack(side=tk.TOP, fill=tk.X, padx=10, pady=10)

        url_label = tk.Label(url_frame, text="Enter URL:")

        url_label.pack(side=tk.LEFT, padx=5)

        self.url_entry = tk.Entry(url_frame)

        self.url_entry.pack(side=tk.LEFT, fill=tk.X, expand=True, padx=5)

        # Create a button to show instructions

        instr_button = tk.Button(self, text="How to use", bg="orange", command=self.show_instructions)

        instr_button.pack(side=tk.TOP, padx=5, pady=5)

        # Create a frame for the output text

        output_frame = tk.Frame(self)

        output_frame.pack(side=tk.TOP, fill=tk.BOTH, expand=True, padx=10, pady=10)

        self.output_text = tk.Text(output_frame)

        self.output_text.pack(side=tk.TOP, fill=tk.BOTH, expand=True)

        # Create a button to run the tool

        run_button = tk.Button(self, text="Run Waybackurls", bg="red", command=self.run_waybackurls)

        self.url_entry.bind("<Return>", lambda event: self.run_waybackurls())

        run_button.pack(side=tk.TOP, padx=5, pady=5)

        # Create buttons to filter by file extension

        self.extensions = ['.js', '.py', '.mysql', '.env', '.json', '.db', '.pem', '.bak', '.php', '.xml', '.yml', '.yaml', '.txt', '.png', '.md', '.mp4', '.gz', '.log', '.env', '.html', '.cfg', '.php', '.sh', '.pdf', '.xls', '.config', '.csv', '.sqlite', '.sql', '.tmp']

        self.filter_buttons = []

        for extension in self.extensions:

            button = tk.Button(self, text=f"{extension}", command=lambda ext=extension: self.filter_output(ext))

            button.pack(side=tk.LEFT, padx=2, pady=2)

            self.filter_buttons.append(button)

        # Create an Export button

        export_button = tk.Button(self, text="Export", command=self.export_output, bg="green")

        export_button.pack(side=tk.TOP, padx=5, pady=5)

    def show_instructions(self):

        instructions = "This Tool uses waybackurls by Tomnomnom as a engine and Filters Ravaan uses for WAPT. Simply Enter the Url in the Enter URL: https://foo.com, Next you can apply filter and Export the URLs"

        messagebox.showinfo("Instructions", instructions)

    def run_waybackurls(self):

        self.generated_urls = subprocess.check_output(f"waybackurls {self.url_entry.get()}", shell=True).decode("utf-8")

        self.original_urls = self.generated_urls

        self.output_text.delete("1.0", tk.END)

        self.output_text.insert(tk.END, self.generated_urls)

    def filter_output(self, extension):

        if self.generated_urls == '':

            tk.messagebox.showerror("Error", "Please run the tool first to generate URLs.")

            return

        # Check if the same filter has been applied previously

        if hasattr(self, 'last_filter') and self.last_filter == extension:

            self.generated_urls = self.original_urls

            delattr(self, 'last_filter')

        else:

            # Filter URLs by extension

            filtered_output = ""

            for line in self.original_urls.split("\n"):

                if line.endswith(extension):

                    filtered_output += line + "\n"

            # Check if the filtered output is empty

            if filtered_output == "":

                tk.messagebox.showinfo("Info", f"No URLs found with extension {extension}.")

                return

            self.generated_urls = filtered_output

            self.last_filter = extension

        # Update output text

        self.output_text.delete("1.0", tk.END)

        self.output_text.insert(tk.END, self.generated_urls)

    def export_output(self):

        if self.generated_urls == '':

            messagebox.showerror("Error", "Please run the tool first to generate URLs.")

            return

        filename = filedialog.asksaveasfilename(defaultextension=".txt",

                                                filetypes=(("Text files", "*.txt"), ("All files", "*.*")))

        if filename:

            with open(filename, 'w') as f:

                f.write(self.generated_urls)

            messagebox.showinfo("Success", f"URLs successfully exported to {filename}.")

root = tk.Tk()

app = Application(master=root)

app.mainloop()


## 基于mindspore实现语义分割案例

# 导入依赖库
# os库
import os
# 引入numpy
import numpy as np
# 引入读写不同数据文件格式的函数
import scipy.io
# 引入数据序列化和反序列化
import pickle
# 引入操作图像方法
from PIL import Image
# 引入高级的文件,文件夹,压缩包处理模块
import shutil
# 引入计算机视觉库
import cv2
# 引入归一化提供训练测试所用的数据集
from mindspore.mindrecord import FileWriter
# 引入数据读取
import mindspore.dataset as de
# 引入MindSpore
import mindspore as ms
# 引入神经网络模块
import mindspore.nn as nn
#导入mindspore中的ops模块
import mindspore.ops  as P
# 引入张量模块
from mindspore import Tensor, ParallelMode
from mindspore import load_checkpoint
from mindspore import load_param_into_net
from mindspore import dtype as mstype
# 引入python解释器和它的环境有关的函数
import sys
# 将字典转为easydict
from easydict import EasyDict as edict
# 引入模型训练或推理的高阶接口。
# 引入用于构建Callback函数基类。
from mindspore.train import ModelCheckpoint, CheckpointConfig,LossMonitor, TimeMonitor,Model
# 引入集合通信接口
from mindspore.communication import init, get_rank, get_group_size
# 引入用于构建Callback函数的基类。
from mindspore import amp
from mindspore import set_seed
import PIL
# 引入绘图模块
import matplotlib.pyplot as plt
# 引入可视化库
import matplotlib as mpl
# 引入可视化库色彩模块
import matplotlib.colors as colors

batch_SZ = 2

## 数据预处理
# 设置Opencv的线程数量为0
cv2.setNumThreads(0)

# 数据集对象，用于载入语义分割数据集
class SegDataset:
    def __init__(self,
                 image_mean,            # 图像像素值平均值
                 image_std,             # 图像像素值标准差
                 data_file='',          # 数据集文件路径
                 batch_size=batch_SZ,   # 单次训练所使用样本的数量
                 crop_size=512,         # 随机裁剪后的图片大小
                 max_scale=2.0,         # 最大缩放比例
                 min_scale=0.5,         # 最小缩放比例
                 ignore_label=255,      # 忽略标签值
                 num_classes=21,        # 图像和标签中的类别数量
                 num_readers=2,         # 读取数据的IO线程数量
                 num_parallel_calls=1,  # 数据集batch的并行度
                 shard_id=None,         # 数据集分片ID，None表示无分片
                 shard_num=None         # 数据集分片数量，None表示无分片
                               ):
        # 定义数据集文件路径
        self.data_file = data_file
        # 定义单次训练所使用样本的数量
        self.batch_size = batch_size
        # 定义随机裁剪后的图片大小
        self.crop_size = crop_size
        # 定义图像像素值平均值
        self.image_mean = np.array(image_mean, dtype=np.float32)
        # 定义图像像素值标准差
        self.image_std = np.array(image_std, dtype=np.float32)
        # 定义最大缩放比例
        self.max_scale = max_scale
        # 定义最小缩放比例
        self.min_scale = min_scale
        # 定义忽略标签值
        self.ignore_label = ignore_label
        # 定义图像和标签中的类别数量
        self.num_classes = num_classes
        # 定义读取数据的IO线程数量
        self.num_readers = num_readers
        # 定义数据集batch的并行度
        self.num_parallel_calls = num_parallel_calls
        # 定义数据集分片ID
        self.shard_id = shard_id
        # 定义数据集分片数量
        self.shard_num = shard_num
        # VOC数据集原始图片文件夹路径
        self.voc_img_dir = os.path.join(self.data_file,'JPEG')
        # VOC数据集语义标注图片文件夹路径
        self.voc_anno_dir = os.path.join(self.data_file,'MASK1')
        # VOC数据集训练集文件列表路径
        self.voc_train_lst = os.path.join(self.data_file,'train.txt')
        # VOC数据集验证集文件列表路径
        self.voc_val_lst = os.path.join(self.data_file,'val.txt')
        #  VOC数据集使用的灰度标注图片文件夹路径
        self.voc_anno_gray_dir = os.path.join(self.data_file,'SegmentationClassGray')
        # 生成的MindRecord文件保存路径
        self.mindrecord_save =  os.path.join(self.data_file,'VOC_mindrecord')
        # 最大缩放比例必须大于最小缩放比例
        assert max_scale > min_scale

    #数据预处理，包括图像的解码，尺度缩放，随机裁剪等操作
    def preprocess_(self, image, label):
        #bgr图像解码
        image_out = cv2.imdecode(np.frombuffer(image, dtype=np.uint8), cv2.IMREAD_COLOR)
        #灰度图像解码
        label_out = cv2.imdecode(np.frombuffer(label, dtype=np.uint8), cv2.IMREAD_GRAYSCALE)
        #尺度缩放
        sc = np.random.uniform(self.min_scale, self.max_scale)
        new_h, new_w = int(sc * image_out.shape[0]), int(sc * image_out.shape[1])
        image_out = cv2.resize(image_out, (new_w, new_h), interpolation=cv2.INTER_CUBIC)
        label_out = cv2.resize(label_out, (new_w, new_h), interpolation=cv2.INTER_NEAREST)
        #图像标准化
        image_out = (image_out - self.image_mean) / self.image_std
        #随机裁剪
        h_, w_ = max(new_h, self.crop_size), max(new_w, self.crop_size)
        pad_h, pad_w = h_ - new_h, w_ - new_w
        if pad_h > 0 or pad_w > 0:
            image_out = cv2.copyMakeBorder(image_out, 0, pad_h, 0, pad_w, cv2.BORDER_CONSTANT, value=0)
            label_out = cv2.copyMakeBorder(label_out, 0, pad_h, 0, pad_w, cv2.BORDER_CONSTANT, value=self.ignore_label)
        offset_h = np.random.randint(0, h_ - self.crop_size + 1)
        offset_w = np.random.randint(0, w_ - self.crop_size + 1)
        image_out = image_out[offset_h: offset_h + self.crop_size, offset_w: offset_w + self.crop_size, :]
        label_out = label_out[offset_h: offset_h + self.crop_size, offset_w: offset_w+self.crop_size]

        #随机水平翻转
        if np.random.uniform(0.0, 1.0) > 0.5:
            image_out = image_out[:, ::-1, :]
            label_out = label_out[:, ::-1]

        #图像转置以符合模型输入要求
        image_out = image_out.transpose((2, 0, 1))
        image_out = image_out.copy()
        label_out = label_out.copy()
        return image_out, label_out

    #得到灰度数据集的方法，若文件已存在，则直接返回
    def get_gray_dataset(self):
        if os.path.exists(self.voc_anno_gray_dir):
            print('the gray file is already exists！')
            return
        os.makedirs(self.voc_anno_gray_dir)

        #转换彩色图像为灰度图像，并保存到指定路径
        print('converting voc color png to gray png ...')
        for ann in os.listdir(self.voc_anno_dir):
            ann_im = Image.open(os.path.join(self.voc_anno_dir, ann))
            #将该图像转换为灰度图像
            ann_im = Image.fromarray(np.array(ann_im))
            ann_im.save(os.path.join(self.voc_anno_gray_dir, ann))
        print('converting done')

    #获取MindRecord格式的数据集，num_shards为生成MindRecord的分片数，shuffle为是否对数据做洗牌处理
    def get_mindrecord_dataset(self, is_training, num_shards=1, shuffle=True):

        print('self.voc_train_lst: ', self.voc_train_lst)
        print(self.mindrecord_save) # VOC_mindrecord
        datas = []
        if is_training:
            data_lst = self.voc_train_lst
            self.mindrecord_save = os.path.join(self.mindrecord_save,'train')
        else:
            data_lst = self.voc_val_lst
            self.mindrecord_save = os.path.join(self.mindrecord_save,'eval')

        if os.path.exists(self.mindrecord_save):
            #shutil.rmtree(self.mindrecord_save)
            print('mindrecord file is already exists！')
            self.mindrecord_save = os.path.join(self.mindrecord_save,'VOC_mindrecord')
            return

        with open(data_lst) as f:
            lines = f.readlines()
        if shuffle:
            np.random.shuffle(lines)

        print('creating mindrecord dataset...')
        os.makedirs(self.mindrecord_save)
        self.mindrecord_save = os.path.join(self.mindrecord_save,'VOC_mindrecord')
        print('number of samples:', len(lines))
        #定义MindRecord的schema
        seg_schema = {"file_name": {"type": "string"}, "label": {"type": "bytes"}, "data": {"type": "bytes"}}
        writer = FileWriter(file_name=self.mindrecord_save, shard_num=num_shards)
        writer.add_schema(seg_schema, "seg_schema")

        #将schema写入MindRecord
        cnt = 0
        for l in lines:
            id_ = l.strip()
            img_path = os.path.join(self.voc_img_dir, id_ + '.jpg')
            label_path = os.path.join(self.voc_anno_gray_dir, id_ + '.png')

            sample_ = {"file_name": img_path.split('/')[-1]}
            with open(img_path, 'rb') as f:
                sample_['data'] = f.read()
            with open(label_path, 'rb') as f:
                sample_['label'] = f.read()
            datas.append(sample_)
            cnt += 1
            if cnt % 1000 == 0:
                writer.write_raw_data(datas)
                print('number of samples written:', cnt)
                datas = []

        if datas:
            writer.write_raw_data(datas)
        writer.commit()
        print('number of samples written:', cnt)
        print('Create Mindrecord Done')

    #生成文件
    def get_dataset(self, repeat=1):
        data_set = de.MindDataset(dataset_files=self.mindrecord_save, columns_list=["data", "label"],
                                  shuffle=True, num_parallel_workers=self.num_readers,
                                  num_shards=self.shard_num, shard_id=self.shard_id)
        transforms_list = self.preprocess_
        data_set = data_set.map(operations=transforms_list, input_columns=["data", "label"],
                                output_columns=["data", "label"],
                                num_parallel_workers=self.num_parallel_calls)
        data_set = data_set.shuffle(buffer_size=self.batch_size * 10)
        data_set = data_set.batch(self.batch_size, drop_remainder=True)
        data_set = data_set.repeat(repeat)
        return data_set

# 实验过程
## 模型构建

# 定义1x1卷积层
def conv1x1(in_planes, out_planes, stride=1):
    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, weight_init='xavier_uniform')

# 定义3x3卷积层
def conv3x3(in_planes, out_planes, stride=1, dilation=1, padding=1):
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, pad_mode='pad', padding=padding,
                     dilation=dilation, weight_init='xavier_uniform')

#定义Resnet主体网络
class Resnet(nn.Cell):
    def __init__(self, block, block_num, output_stride, use_batch_statistics=True):
        super(Resnet, self).__init__()
        self.inplanes = 64# 输入通道数
        # 第一层卷积层
        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, pad_mode='pad', padding=3,
                               weight_init='xavier_uniform')
        self.bn1 = nn.BatchNorm2d(self.inplanes, use_batch_statistics=use_batch_statistics)
        self.relu = nn.ReLU()
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='same')
        # 第一层残差块
        self.layer1 = self._make_layer(block, 64, block_num[0], use_batch_statistics=use_batch_statistics)
        # 第二层残差块
        self.layer2 = self._make_layer(block, 128, block_num[1], stride=2, use_batch_statistics=use_batch_statistics)
        # 根据输出步长选择第三、第四层残差块
        if output_stride == 16:
            self.layer3 = self._make_layer(block, 256, block_num[2], stride=2,
                                           use_batch_statistics=use_batch_statistics)
            self.layer4 = self._make_layer(block, 512, block_num[3], stride=1, base_dilation=2, grids=[1, 2, 4],
                                           use_batch_statistics=use_batch_statistics)
        elif output_stride == 8:
            self.layer3 = self._make_layer(block, 256, block_num[2], stride=1, base_dilation=2,
                                           use_batch_statistics=use_batch_statistics)
            self.layer4 = self._make_layer(block, 512, block_num[3], stride=1, base_dilation=4, grids=[1, 2, 4],
                                           use_batch_statistics=use_batch_statistics)
    # 构建残差块
    def _make_layer(self, block, planes, blocks, stride=1, base_dilation=1, grids=None, use_batch_statistics=True):
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.SequentialCell([
                conv1x1(self.inplanes, planes * block.expansion, stride),
                nn.BatchNorm2d(planes * block.expansion, use_batch_statistics=use_batch_statistics)
            ])

        if grids is None:
            grids = [1] * blocks

        layers = [
            block(self.inplanes, planes, stride, downsample, dilation=base_dilation * grids[0],
                  use_batch_statistics=use_batch_statistics)
        ]
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(
                block(self.inplanes, planes, dilation=base_dilation * grids[i],
                      use_batch_statistics=use_batch_statistics))

        return nn.SequentialCell(layers)
    # 前向推理
    def construct(self, x):
        out = self.conv1(x)     # 第一层卷积
        out = self.bn1(out)
        out = self.relu(out)
        out = self.maxpool(out) # 第一层池化
        out = self.layer1(out)  # 第一层残差块
        out = self.layer2(out)  # 第二层残差块
        out = self.layer3(out)  # 第三层残差块
        out = self.layer4(out)  # 第四层残差块

        return out

#构建Bottleneck，用于ResNeXt中构建残差块
class Bottleneck(nn.Cell):
    # 扩充率为4
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None, dilation=1, use_batch_statistics=True):
        super(Bottleneck, self).__init__()
        # 第一个1x1卷积层
        self.conv1 = conv1x1(inplanes, planes)
        # 第一个BatchNorm层
        self.bn1 = nn.BatchNorm2d(planes, use_batch_statistics=use_batch_statistics)
        # 第二个3x3卷积层
        self.conv2 = conv3x3(planes, planes, stride, dilation, dilation)
        # 第二个BatchNorm层
        self.bn2 = nn.BatchNorm2d(planes, use_batch_statistics=use_batch_statistics)
        # 第三个1x1卷积层，维度扩充
        self.conv3 = conv1x1(planes, planes * self.expansion)
        # 第三个BatchNorm层
        self.bn3 = nn.BatchNorm2d(planes * self.expansion, use_batch_statistics=use_batch_statistics)
        # Relu激活函数
        self.relu = nn.ReLU()
        # 下采样层，使维度匹配
        self.downsample = downsample
        # 张量相加操作
        self.add = P.Add()

    def construct(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            identity = self.downsample(x)
        # 将identity和out相加
        out = self.add(out, identity)
        out = self.relu(out)
        return out

#构建ASPP，用于DeepLabV3+中的ASPP模块
class ASPP(nn.Cell):
    def __init__(self, atrous_rates, phase='train', in_channels=2048, num_classes=21,
                 use_batch_statistics=True):
        super(ASPP, self).__init__()
        # 训练或者测试阶段
        self.phase = phase
        # 输出通道数
        out_channels = 256
        # ASPP卷积层1
        self.aspp1 = ASPPConv(in_channels, out_channels, atrous_rates[0], use_batch_statistics=use_batch_statistics)
         # ASPP卷积层2
        self.aspp2 = ASPPConv(in_channels, out_channels, atrous_rates[1], use_batch_statistics=use_batch_statistics)
         # ASPP卷积层3
        self.aspp3 = ASPPConv(in_channels, out_channels, atrous_rates[2], use_batch_statistics=use_batch_statistics)
         # ASPP卷积层4
        self.aspp4 = ASPPConv(in_channels, out_channels, atrous_rates[3], use_batch_statistics=use_batch_statistics)
        # ASPP池化层
        self.aspp_pooling = ASPPPooling(in_channels, out_channels)
        # 输出通道数为(out_channels * (len(atrous_rates) + 1))
        self.conv1 = nn.Conv2d(out_channels * (len(atrous_rates) + 1), out_channels, kernel_size=1,
                               weight_init='xavier_uniform')
        # BatchNorm层
        self.bn1 = nn.BatchNorm2d(out_channels, use_batch_statistics=use_batch_statistics)
        # Relu激活函数
        self.relu = nn.ReLU()
        # 输出通道数为num_classes
        self.conv2 = nn.Conv2d(out_channels, num_classes, kernel_size=1, weight_init='xavier_uniform', has_bias=True)
        # 沿着通道维度拼接
        self.concat = P.Concat(axis=1)
        # 随机失活，防止过拟合
        self.drop = nn.Dropout(p=0.3)

    def construct(self, x):
        x1 = self.aspp1(x)
        x2 = self.aspp2(x)
        x3 = self.aspp3(x)
        x4 = self.aspp4(x)
        x5 = self.aspp_pooling(x)

        x = self.concat((x1, x2))
        x = self.concat((x, x3))
        x = self.concat((x, x4))
        x = self.concat((x, x5))

        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        # 如果是训练阶段，则进行随机失活
        if self.phase == 'train':
            x = self.drop(x)
        x = self.conv2(x)
        return x

##定义ASPPPooling类，用于定义ASPP池操作
class ASPPPooling(nn.Cell):
    #定义参数
    def __init__(self, in_channels, out_channels, use_batch_statistics=True):
        super(ASPPPooling, self).__init__()
        #定义卷积层操作
        self.conv = nn.SequentialCell([
            #1x1卷积操作
            nn.Conv2d(in_channels, out_channels, kernel_size=1, weight_init='xavier_uniform'),
            #批量归一化操作
            nn.BatchNorm2d(out_channels, use_batch_statistics=use_batch_statistics),
            #激活函数操作
            nn.ReLU()
        ])
        #定义shape操作
        self.shape = P.Shape()

    def construct(self, x):
        #获取输入x的大小
        size = self.shape(x)
        #先对输入x进行全局平均池化操作
        out = nn.AvgPool2d(size[2])(x)
        #再进行卷积、批量归一化、激活操作
        out = self.conv(out)
        #大小变化为输入x的大小
        out = P.ResizeNearestNeighbor((size[2], size[3]), True)(out)
        return out

#定义ASPPConv类，用于定义ASPP卷积操作
class ASPPConv(nn.Cell):
    #定义参数
    def __init__(self, in_channels, out_channels, atrous_rate=1, use_batch_statistics=True):
        super(ASPPConv, self).__init__()
        #根据不同的空洞卷积率定义不同的卷积操作
        if atrous_rate == 1:
            conv = nn.Conv2d(in_channels, out_channels, kernel_size=1, has_bias=False, weight_init='xavier_uniform')
        else:
            conv = nn.Conv2d(in_channels, out_channels, kernel_size=3, pad_mode='pad', padding=atrous_rate,
                             dilation=atrous_rate, weight_init='xavier_uniform')
        #批量归一化操作
        bn = nn.BatchNorm2d(out_channels, use_batch_statistics=use_batch_statistics)
        #激活函数操作
        relu = nn.ReLU()
        #定义卷积层操作
        self.aspp_conv = nn.SequentialCell([conv, bn, relu])

    def construct(self, x):
        #进行卷积、批量归一化、激活操作
        out = self.aspp_conv(x)
        return out

#定义DeepLabV3类，用于定义整个DeepLabV3网络
class DeepLabV3(nn.Cell):
    def __init__(self, phase='train', num_classes=21, output_stride=16, freeze_bn=False):
        super(DeepLabV3, self).__init__()
        #根据输入的参数freeze_bn来判断是否使用批量归一化操作
        use_batch_statistics = not freeze_bn
        #调用Resnet类来构建ResNet网络
        self.resnet = Resnet(Bottleneck, [3, 4, 23, 3], output_stride=output_stride,
                             use_batch_statistics=use_batch_statistics)
        #调用ASPP类来构建ASPP网络
        self.aspp = ASPP([1, 6, 12, 18], phase, 2048, num_classes,
                         use_batch_statistics=use_batch_statistics)
        #定义shape操作
        self.shape = P.Shape()

    def construct(self, x):
        #获取输入x的大小
        size = self.shape(x)
        #将输入x输入到ResNet网络中，得到输出
        out = self.resnet(x)
        #将ResNet的输出输入到ASPP网络中，得到ASPP的输出
        out = self.aspp(out)
        #将ASPP的输出进行大小变换，变成与输入x相同的大小
        out = P.ResizeBilinear((size[2], size[3]), True)(out)
        return out

#定义不同的学习率
#生成cosine学习率下降序列
def cosine_lr(base_lr, decay_steps, total_steps):
    for i in range(int(total_steps)):
        step_ = min(i, decay_steps)
        yield base_lr * 0.5 * (1 + np.cos(np.pi * step_ / decay_steps))

#生成polynomial学习率下降序列
def poly_lr(base_lr, decay_steps, total_steps, end_lr=0.0001, power=0.9):
    for i in range(int(total_steps)):
        step_ = min(i, decay_steps)
        yield (base_lr - end_lr) * ((1.0 - step_ / decay_steps) ** power) + end_lr

#生成exponential学习率下降序列
def exponential_lr(base_lr, decay_steps, decay_rate, total_steps, staircase=False):
    for i in range(total_steps):
        if staircase:
            power_ = i // decay_steps
        else:
            power_ = float(i) / decay_steps
        yield base_lr * (decay_rate ** power_)

#定义损失函数
class SoftmaxCrossEntropyLoss(nn.Cell):
    def __init__(self, num_cls=21, ignore_label=255):
        super(SoftmaxCrossEntropyLoss, self).__init__()
        #one-hot编码相关操作
        self.one_hot = P.OneHot(axis=-1)
        self.on_value = Tensor(1.0, mstype.float32)
        self.off_value = Tensor(0.0, mstype.float32)
        #类型转换相关操作
        self.cast = P.Cast()
        #softmax交叉熵损失函数及相关操作
        self.ce = nn.SoftmaxCrossEntropyWithLogits()
        #类别数及忽略标签
        self.not_equal = P.NotEqual()
        self.num_cls = num_cls
        self.ignore_label = ignore_label
        #矩阵乘法及求和相关操作
        self.mul = P.Mul()
        self.sum = P.ReduceSum(False)
        self.div = P.RealDiv()
        #转置及形状变换相关操作
        self.transpose = P.Transpose()
        self.reshape = P.Reshape()
    #将标签转换为整形
    def construct(self, logits, labels):
        #将标签拉成一维并转换形状
        labels_int = self.cast(labels, mstype.int32)
        labels_int = self.reshape(labels_int, (-1,))
         #将logits转置并转换形状
        logits_ = self.transpose(logits, (0, 2, 3, 1))
        logits_ = self.reshape(logits_, (-1, self.num_cls))
         #生成权重
        weights = self.not_equal(labels_int, self.ignore_label)
        weights = self.cast(weights, mstype.float32)
        #生成one-hot标签
        one_hot_labels = self.one_hot(labels_int, self.num_cls, self.on_value, self.off_value)
        #计算softmax交叉熵损失
        loss = self.ce(logits_, one_hot_labels)
         #加权
        loss = self.mul(weights, loss)
        #求平均损失
        loss = self.div(self.sum(loss), self.sum(weights))
        return loss

# 模型训练
## 构建训练函数

# 设置随机种子
set_seed(1)
# 设置上下文
ms.set_context(mode=ms.GRAPH_MODE, save_graphs=False,
                    device_target="CPU")
# 建立训练网络
class BuildTrainNetwork(nn.Cell):
    def __init__(self, network, criterion):
        super(BuildTrainNetwork, self).__init__()
        self.network = network
        self.criterion = criterion

    def construct(self, input_data, label):
        output = self.network(input_data)
        net_loss = self.criterion(output, label)
        return net_loss

# 训练函数
def train(args):
    # 如果使用分布式训练，则初始化
    if args.is_distributed:
        init()
        args.rank = get_rank()
        args.group_size = get_group_size()

        parallel_mode = ParallelMode.DATA_PARALLEL
        ms.set_auto_parallel_context(parallel_mode=parallel_mode, gradients_mean=True, device_num=args.group_size)

    # 构建数据集
    dataset = SegDataset(image_mean=args.image_mean,
                                        image_std=args.image_std,
                                        data_file=args.data_file,
                                        batch_size=args.batch_size,
                                        crop_size=args.crop_size,
                                        max_scale=args.max_scale,
                                        min_scale=args.min_scale,
                                        ignore_label=args.ignore_label,
                                        num_classes=args.num_classes,
                                        num_readers=2,
                                        num_parallel_calls=4,
                                        shard_id=args.rank,
                                        shard_num=args.group_size)
    dataset.get_gray_dataset()
    dataset.get_mindrecord_dataset(is_training=True)
    dataset = dataset.get_dataset(repeat=1)

    # 构建相关网络
    if args.model == 'deeplab_v3_s16':
        network = DeepLabV3('train', args.num_classes, 16, args.freeze_bn)
    elif args.model == 'deeplab_v3_s8':
        network = DeepLabV3('train', args.num_classes, 8, args.freeze_bn)
    else:
        raise NotImplementedError('model [{:s}] not recognized'.format(args.model))

    # 构建损失函数
    loss_ = SoftmaxCrossEntropyLoss(args.num_classes, args.ignore_label)
    loss_.add_flags_recursive(fp32=True)
    train_net = BuildTrainNetwork(network, loss_)

    # l加载预训练模型
    if os.path.exists(args.ckpt_file):
        param_dict = ms.load_checkpoint(args.ckpt_file)
        ms.load_param_into_net(train_net, param_dict)

    # 优化器
    iters_per_epoch = dataset.get_dataset_size()
    total_train_steps = iters_per_epoch * args.train_epochs
    if args.lr_type == 'cos':
        lr_iter = cosine_lr(args.base_lr, total_train_steps, total_train_steps)
    elif args.lr_type == 'poly':
        lr_iter = poly_lr(args.base_lr, total_train_steps, total_train_steps, end_lr=0.0, power=0.9)
    elif args.lr_type == 'exp':
        lr_iter = exponential_lr(args.base_lr, args.lr_decay_step, args.lr_decay_rate,
                                                total_train_steps, staircase=True)
    else:
        raise ValueError('unknown learning rate type')
    opt = nn.Momentum(params=train_net.trainable_params(), learning_rate=lr_iter, momentum=0.9, weight_decay=0.0001,
                      loss_scale=args.loss_scale)

    # 损失梯度缩放
    manager_loss_scale = amp.FixedLossScaleManager(args.loss_scale, drop_overflow_update=False)
    model = Model(train_net, optimizer=opt, amp_level="O3", loss_scale_manager=manager_loss_scale)

    # 回调函数，用于保存 ckpts
    time_cb = TimeMonitor(data_size=iters_per_epoch)
    loss_cb = LossMonitor()
    cbs = [time_cb, loss_cb]

    if args.rank == 0:
        config_ck = CheckpointConfig(save_checkpoint_steps=iters_per_epoch,
                                     keep_checkpoint_max=args.keep_checkpoint_max)
        ckpoint_cb = ModelCheckpoint(prefix=args.model, directory=args.train_dir, config=config_ck)
        cbs.append(ckpoint_cb)
    # 开始训练
    model.train(args.train_epochs, dataset, callbacks=cbs,dataset_sink_mode=True)

## 模型训练

##设定相关参数并转为edict对象
cfg = edict({
    "batch_size": batch_SZ,
    "crop_size": 513,
    "image_mean": [103.53, 116.28, 123.675],    #图片均值
    "image_std": [57.375, 57.120, 58.395],      #图片标准差
    "min_scale": 0.5,                           #最小缩放比例
    "max_scale": 2.0,                           #最大缩放比例
    "ignore_label": 255,                        #忽略标签
    "num_classes": 21,                          #分类数
    "train_epochs" : 50,                         #训练轮数
    "lr_type": 'cos',                           #学习率变化方式
    "base_lr": 0.0,                             #基础学习率
    "lr_decay_step": 3*91,                      #学习率递减步数
    "lr_decay_rate" :0.1,                       #学习率递减率
    "loss_scale": 2048,                         #损失函数缩放比例
    "model": 'deeplab_v3_s8',                   #模型类型
    'rank': 0,                                  #排名
    'group_size':1,                             #组大小
    'keep_checkpoint_max':1,                    #最大保存点数
    'train_dir': 'model',                       #训练目录
    'is_distributed':False,                     #是否分布式训练
    'freeze_bn':True                            #是否冻结BN层
})

if __name__ == '__main__':
    device_target = ms.get_context("device_target")
    mode = ms.GRAPH_MODE
    print("device:", device_target)
    ms.context.set_context(mode=mode, device_target="CPU")

    #如果训练目录存在，则删除
    if os.path.exists(cfg.train_dir):
        shutil.rmtree(cfg.train_dir)

    #数据路径和checkpoint路径
    data_path = './seg2'

    cfg.data_file = data_path
    ckpt_path = './ckpt/deeplab_v3_s8-300_11.ckpt'
    cfg.ckpt_file = ckpt_path

    #开始训练模型
    train(cfg)
    exit(0)


from __future__ import print_function

import os
import sys
import logging

class AIDbg(object):

    def __init__(self, kernal, user_uri, realm, verbose):

        self.kernal   = kernal
        self.user_uri = user_uri
        self.realm    = realm
        self.verbose  = verbose

        if self.verbose:
            logging.getLogger().setLevel(logging.DEBUG)
        else:
            logging.getLogger().setLevel(logging.INFO)

    def print_help(self):
        print (":h          help")
        print (":c <skills> compile <skills>")
        print (":m          show memory / context")
        print (":t          %s prolog tracing" % ('disable' if self.run_trace else 'enable'))
        print (":v          verbose logging %s" % ('off' if self.verbose else 'on'))
        print (":q          quit")

    def process_command(self, line):

        if line == ":h":
            self.print_help()

        # FIXME: we'd have to force-reload the python module
        elif line[:2] == ":c":

            parts = line.split(' ')
            if len(parts) < 2:
                logging.error('?usage')
                self.print_help()
                return

            self.kernal.compile_skill_multi (parts[1:])

        elif line == ":m":

            print ("ctx.user   = %s" % self.ctx.user)
            print ("ctx.realm  = %s" % self.ctx.realm)
            print ("ctx.lang   = %s" % self.ctx.lang)
            memd = self.kernal.mem_dump(self.ctx.realm)
            for k, v, score in memd:
                print(u'MEM(%-8s):    %-20s: %s (%f)' % (self.ctx.realm, k, v, score))
            memd = self.kernal.mem_dump(self.ctx.user)
            for k, v, score in memd:
                print(u'MEM(%-8s):    %-20s: %s (%f)' % (self.ctx.user, k, v, score))

        elif line == ":t":
            self.run_trace = not self.run_trace

        elif line == ":v":
            self.verbose = not self.verbose
            if self.verbose:
                logging.getLogger().setLevel(logging.DEBUG)
            else:
                logging.getLogger().setLevel(logging.INFO)

        else:
            logging.error("? command error ('%s')" % line)
            self.print_help()

    def run(self):

        self.ctx       = self.kernal.create_context(user=self.user_uri, realm=self.realm)
        self.run_trace = False

        while True:

            line = raw_input ('dbg (:h for help)> ')

            if not line:
                continue
            if len(line)<1:
                continue

            if line == ':q':
                break

            if line[0] == ':':
                self.process_command(line)
                continue

            out, score, action = self.kernal.process_input(self.ctx, line, run_trace=self.run_trace)

            if action:
                logging.info(u'RESP: [%6.1f] %s | action: %s' % (score, out, unicode(action)))
            else:
                logging.info(u'RESP: [%6.1f] %s ' % (score, out))


import numpy as np
import ase.io
from nice.blocks import *
from nice.utilities import *
from nice.rascal_coefficients import process_structures
import copy
from rascal.representations import SphericalInvariants

def get_nice_powerspectrum():
    return StandardSequence([
        StandardBlock(None, None, None,
                      ThresholdExpansioner(num_expand=None, mode='invariants'),
                      None, None)
    ],
                            initial_scaler=None)

def get_nice_ps_kernel(structures, hypers):

    all_species = get_all_species(structures)
    coefficients = get_spherical_expansion(structures, hypers, all_species, split_by_central_specie=False,
                                           show_progress = False)
    nice = get_nice_powerspectrum()
    nice.fit(coefficients)
    nice_ps = nice.transform(coefficients, return_only_invariants = True)[2]
    return nice_ps.dot(nice_ps.T)

def get_rascal_ps_kernel(structures, hypers):
    structures = process_structures(structures)
    soap = SphericalInvariants(**hypers)
    librascal_ps = soap.transform(structures).get_features(soap)
    return librascal_ps.dot(librascal_ps.T)

def test_powerspectrum_kernels(epsilon = 1e-10):
    structures = ase.io.read('../reference_configurations/methane_100.extxyz', index = ':')
    HYPERS = {
        'interaction_cutoff': 6.3,
        'max_radial': 5,
        'max_angular': 5,
        'gaussian_sigma_type': 'Constant',
        'gaussian_sigma_constant': 0.3,
        'cutoff_smooth_width': 0.3,
        'radial_basis': 'GTO',

    }

    HYPERS_PS = copy.deepcopy(HYPERS)
    HYPERS_PS['normalize'] = False
    HYPERS_PS['soap_type'] = 'PowerSpectrum'

    nice_kernel = get_nice_ps_kernel(structures, HYPERS)
    rascal_kernel = get_rascal_ps_kernel(structures, HYPERS_PS)

    nice_kernel = np.reshape(nice_kernel, [-1])
    rascal_kernel = np.reshape(rascal_kernel, [-1])

    mask = rascal_kernel > epsilon
    nice_kernel = nice_kernel[mask]
    rascal_kernel = rascal_kernel[mask]

    ratios = nice_kernel / rascal_kernel
    discrepancy = (np.max(ratios) - np.min(ratios)) / np.mean(ratios)
    assert discrepancy < epsilon


import logging
import re
from collections import namedtuple
from urllib.parse import urljoin

from scrapy import Request, Spider, signals
from scrapy.utils.request import request_fingerprint
from warcio.warcwriter import WARCWriter

from .utils import resource_matches_base_url, write_warc_request_response

Resource = namedtuple("Resource", ["name", "type", "link_type", "content"])
REGEXP_CSS_URL = re.compile(r"""url\(['"]?(.*?)['"]?\)""")

Extractor = namedtuple("Extractor", ["name", "type", "link_type", "xpath"])
EXTRACTORS = [
    # Media (images, video etc.)
    Extractor(name="media", type="link", link_type="dependency", xpath="//img/@src"),
    Extractor(name="media", type="link", link_type="dependency", xpath="//audio/@src"),
    Extractor(name="media", type="link", link_type="dependency", xpath="//video/@src"),
    Extractor(name="media", type="link", link_type="dependency", xpath="//source/@src"),
    Extractor(name="media", type="link", link_type="dependency", xpath="//embed/@src"),
    Extractor(
        name="media", type="link", link_type="dependency", xpath="//object/@data"
    ),
    # CSS
    Extractor(
        name="css",
        type="link",
        link_type="dependency",
        xpath="//link[@rel = 'stylesheet']/@href",
    ),
    Extractor(name="css", type="code", link_type="dependency", xpath="//style/text()"),
    Extractor(name="css", type="code", link_type="dependency", xpath="//*/@style"),
    # JavaScript
    Extractor(name="js", type="link", link_type="dependency", xpath="//script/@src"),
    Extractor(name="js", type="code", link_type="dependency", xpath="//script/text()"),
    # TODO: add "javascript:XXX" on //a/@href etc.
    # TODO: add inline JS (onload, onchange, onclick etc.)
    # Internal/external links and iframes
    # TODO: iframe sources must be considered as if they were the same as the
    # current page being archived (same depth, get all dependencies etc.).
    Extractor(name="other", type="link", link_type="anchor", xpath="//iframe/@src"),
    Extractor(name="other", type="link", link_type="anchor", xpath="//a/@href"),
    Extractor(name="other", type="link", link_type="anchor", xpath="//area/@href"),
    Extractor(
        name="other",
        type="link",
        link_type="anchor",
        xpath="//link[not(@rel = 'stylesheet')]/@href",
    ),
    # TODO: link rel=icon should be considered a dependency (what about other
    # link rel=xxx?)
    # TODO: add all other "//link/@href"
]

def extract_resources(response):
    for extractor in EXTRACTORS:
        for content in response.xpath(extractor.xpath).extract():
            yield Resource(
                name=extractor.name,
                type=extractor.type,
                link_type=extractor.link_type,
                content=content,
            )

class CrauSpider(Spider):

    name = "crawler-spider"
    custom_settings = {
        "CONCURRENT_REQUESTS": 256,
        "CONCURRENT_REQUESTS_PER_DOMAIN": 16,
        "DNSCACHE_ENABLED": True,
        "DNSCACHE_SIZE": 500000,
        "DNS_TIMEOUT": 5,
        "DOWNLOAD_MAXSIZE": 5 * 1024 * 1024,
        "DOWNLOAD_TIMEOUT": 15,
        "REACTOR_THREADPOOL_MAXSIZE": 40,
        "REDIRECT_ENABLED": False,
        "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue",
        "SPIDER_MIDDLEWARES_BASE": {
            "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
            "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": 500,
            "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
            "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
        },
    }

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super().from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def __init__(self, warc_filename, urls, max_depth=1, allowed_uris=None):
        super().__init__()
        self.max_depth = int(max_depth)
        self.warc_filename = warc_filename
        self.urls = urls
        self._request_history = set()
        self.warc_fobj = None
        self.warc_writer = None
        self.allowed_uris = allowed_uris if allowed_uris else []

    def spider_closed(self, spider):
        if self.warc_fobj is not None:
            self.warc_fobj.close()

    def make_request(self, request_class=Request, *args, **kwargs):
        """Method to create requests and implements a custom dedup filter"""

        kwargs["dont_filter"] = kwargs.get("dont_filter", True)
        kwargs["errback"] = kwargs.get("errback", self.parse_request_error)

        meta = kwargs.get("meta", {})
        meta["handle_httpstatus_all"] = meta.get("handle_httpstatus_all", True)
        meta["dont_redirect"] = meta.get("dont_redirect", True)
        kwargs["meta"] = meta

        request = request_class(*args, **kwargs)
        if "#" in request.url:
            request = request.replace(url=request.url[: request.url.find("#")])

        # This `if` filters duplicated requests - we don't use scrapy's dedup
        # filter because it has a bug, which filters out requests in undesired
        # cases <https://github.com/scrapy/scrapy/issues/1225>.
        # TODO: check if this dedup filter does not have the same problem
        # scrapy have (the problem is related to canonicalize request url).
        request_hash = request_fingerprint(request)
        # TODO: may move this in-memory set to a temp file since the number of
        # requests can be pretty large.
        if request_hash in self._request_history:
            return None
        else:
            self._request_history.add(request_hash)
            return request

    def write_warc(self, response):
        # TODO: transform this method into `write_response` so we can have
        # other response writers than WARC (CSV, for example - would be great
        # if we can add specific parsers to save HTML's title and text into
        # CSV, for example).
        write_warc_request_response(self.warc_writer, response)

    def start_requests(self):
        """Start requests with depth = 0

        depth will be 0 for all primary URLs and all requisites (CSS, Images
        and JS) of these URLs. For links found on these URLs, depth will be
        incremented, and so on.
        """
        self.warc_fobj = open(self.warc_filename, mode="wb")
        self.warc_writer = WARCWriter(self.warc_fobj, gzip=True)

        for url in self.urls:
            yield self.make_request(
                url=url, meta={"depth": 0, "main_url": url}, callback=self.parse
            )

    def parse(self, response):
        main_url = response.request.url
        # TODO: what if response.request.url != response.url?
        current_depth = response.request.meta["depth"]
        next_depth = current_depth + 1

        content_type = response.headers.get("Content-Type", b"").decode(
            "ascii"
        )  # TODO: decode properly
        if content_type and content_type.split(";")[0].lower() != "text/html":
            logging.debug(
                f"[{current_depth}] Content-Type not found for {main_url}, parsing as media"
            )
            yield self.parse_media(response)
            return

        logging.debug(f"[{current_depth}] Saving HTML {response.request.url}")
        self.write_warc(response)

        redirect_url = None
        if 300 <= response.status <= 399 and "Location" in response.headers:
            redirect_url = urljoin(
                response.request.url,
                response.headers["Location"].decode("ascii"),  # TODO: decode properly
            )

        for resource in extract_resources(response):
            if resource.type == "link":
                # TODO: handle "//" URLs correctly
                absolute_url = urljoin(main_url, resource.content)
                depth = None
                if resource.link_type == "dependency":
                    depth = current_depth
                elif resource.link_type == "anchor":
                    depth = next_depth
                for request in self.collect_link(
                    main_url, resource.name, absolute_url, depth
                ):
                    if request is None or (
                        redirect_url is not None and redirect_url == request.url
                    ):
                        continue
                    elif (
                        self.allowed_uris
                        and resource.link_type == "anchor"
                        and not resource_matches_base_url(
                            absolute_url, self.allowed_uris
                        )
                    ):
                        logging.info(f"Different domain. Skipping {absolute_url}.")
                        continue
                    yield request

            elif resource.type == "code":
                for request in self.collect_code(
                    main_url, resource.name, resource.content, current_depth
                ):
                    if request is None:
                        continue
                    yield request

        if redirect_url is not None:
            # TODO: how to deal with redirect loops?
            logging.debug(f"[{current_depth}] Redirecting to {redirect_url}")
            yield self.make_request(
                url=redirect_url,
                meta={"depth": current_depth, "main_url": main_url},
                callback=self.parse,
            )

    def parse_request_error(self, failure):
        pass
        # TODO: should we do something with this failure?

    def parse_css(self, response):
        meta = response.request.meta

        for request in self.collect_code(
            response.request.url, "css", response.body, meta["depth"]
        ):
            if request is None:
                continue
            yield request

        logging.debug(f"Saving CSS {response.request.url}")
        self.write_warc(response)

    def parse_js(self, response):
        meta = response.request.meta

        for request in self.collect_code(
            response.request.url, "js", response.body, meta["depth"]
        ):
            if request is None:
                continue
            yield request

        logging.debug(f"Saving JS {response.request.url}")
        self.write_warc(response)

    def parse_media(self, response):
        logging.debug(f"Saving MEDIA {response.request.url}")
        self.write_warc(response)

    def collect_link(self, main_url, link_type, url, depth):
        if depth > self.max_depth:
            logging.debug(
                f"[{depth}] IGNORING (depth exceeded) get link {link_type} {url}"
            )
            return []
        elif not url.startswith("http"):
            logging.debug(f"[{depth}] IGNORING (not HTTP) get link {link_type} {url}")
            return []

        if link_type == "media":
            return [
                self.make_request(
                    url=url,
                    callback=self.parse_media,
                    meta={"depth": depth, "main_url": main_url},
                )
            ]
        elif link_type == "css":
            return [
                self.make_request(
                    url=url,
                    callback=self.parse_css,
                    meta={"depth": depth, "main_url": main_url},
                )
            ]
        elif link_type == "js":
            return [
                self.make_request(
                    url=url,
                    callback=self.parse_js,
                    meta={"depth": depth, "main_url": main_url},
                )
            ]
        elif link_type == "other":
            return [
                self.make_request(
                    url=url,
                    callback=self.parse,
                    meta={"depth": depth, "main_url": main_url},
                )
            ]
        else:
            return [
                self.make_request(
                    url=url,
                    callback=self.parse,
                    meta={"depth": depth, "main_url": main_url},
                )
            ]

    def collect_code(self, main_url, code_type, code, depth):
        if depth > self.max_depth:
            logging.debug(
                f"[{depth}] IGNORING (depth exceeded) getting dependencies for {code_type}"
            )
            return []
        elif code_type == "css":
            if isinstance(code, bytes):
                code = code.decode("utf-8")  # TODO: decode properly
            requests = []
            for result in REGEXP_CSS_URL.findall(code):
                url = urljoin(main_url, result)
                if url.startswith("data:"):
                    continue
                requests.append(
                    self.make_request(
                        url=url,
                        callback=self.parse_media,
                        meta={"depth": depth, "main_url": main_url},
                    )
                )
            return requests
        elif code_type == "js":
            # TODO: extract other references from JS code
            return []
        else:
            logging.info(f"[{depth}] [TODO] PARSE CODE {code_type} {code}")
            return []
            # TODO: change


"""Module for options and configuration."""

from typing import List, Tuple

DEFAULT_FORMAT = '%Y%m%d'

DEFAULT_FORMAT_CANDIDATES = [
    ('%Y%m', 6),
    ('%Y%m%d', 8),
    ('%Y%m%d%H%M%S', 14),
    ('%Y%m%d %H%M%S', 15),
]

def get_date_format() -> str:
    """Return the date format according to configuration value.

    Returns:
        str: date format
    """
    return DEFAULT_FORMAT

def get_format_candidates() -> List[Tuple[str, int]]:
    """Return a sequence of date format candidates to try parsing the input value.

    Returns:
        List[Tuple[str, int]]: sequence of date format candidates, begin the first
            element of the tuple the format string, and the second the expected length
            of the input value.
    """
    return DEFAULT_FORMAT_CANDIDATES


from sklearn.svm import OneClassSVM
from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler

# ## Load data

data_url = "C:/Users/Mahee/Documents/Group proj/Data/1128_1419_fx_tenor_curves_data.csv"
data = pd.read_csv(data_url)

# #### Make column for log returns of spot rate

data['Log Returns'] = np.log(data['Spot Rate'].shift(1)/data['Spot Rate'])

currency = 'GBP'
filteredData = data[data['Currency'] == currency]
dates = filteredData['Date']
spot_rates = filteredData['Spot Rate']
spot_log_ret = filteredData['Log Returns']

# resets index no. so starts from 0 (when picking a currency index does not start from 0)
dates=dates.reset_index(drop=True)
spot_rates=spot_rates.reset_index(drop=True)
spot_log_ret=spot_log_ret.reset_index(drop=True)

# ## One class SVM algorithm

def ocsvm( spot_rate_df, nu, kernel, gamma):
    spot_rates_flat = spot_rate_df.values.reshape(-1,1) # flatten data
    scaler = StandardScaler()
    spot_rates_scaled = scaler.fit_transform(spot_rates_flat) # scale data
    scaled_df = pd.DataFrame(spot_rates_scaled)

    model = OneClassSVM(nu=nu, kernel=kernel, gamma=gamma)
    model.fit(scaled_df) # train data

    # Create a column for the anomaly
    spot_rate_df['anomaly'] = pd.Series(model.predict(scaled_df)).map({1: False, -1: True})
    return spot_rate_df

def plotAnomaly(df_anomaly, dates, spot_rates):
    plt.plot(dates, spot_rates)
    plt.xlabel("Dates")
    plt.ylabel("Spot Rate")
    plt.title( "Spot rate for GBP")
    anomalyDates=dates.loc[df_anomaly['anomaly']] # locates all dates where anomaly is true
    anomalySpots = spot_rates.loc[df_anomaly['anomaly']] # locates all spot rates where anomaly is true
    plt.scatter(anomalyDates, anomalySpots, color='red')
    plt.show()

# ## Results

df_anomaly_2 = ocsvm(spot_rates.copy(), 0.01, 'rbf', 0.01)
plotAnomaly(df_anomaly_2, dates, spot_rates)

# #### Log return results

df_anomaly_3 = ocsvm(spot_log_ret.copy(), 0.001, 'rbf', 0.01)
plotAnomaly(df_anomaly_3, dates, spot_rates)
plt.plot(dates[1:], spot_log_ret[1:])
plt.scatter(dates[1:].loc[df_anomaly_3['anomaly'][1:]], spot_log_ret[1:].loc[df_anomaly_3['anomaly'][1:]], color='red')
plt.show()


"""
References:
    blueprint/resources/widgets_template
"""
import shutil
from collections import namedtuple
from os import mkdir
from os import path as xpath
from textwrap import dedent

from lk_logger import lk
from lk_utils import dumps
from lk_utils import loads

from blueprint.src import io
from blueprint.src.common import sort_formatted_list
from blueprint.src.typehint import *

def main(widgets_dir=io.widgets_dir):
    """
    Input:
        io.json_3 (this is generated by `..qml_modules_indexing.no3_all_qml
        _widgets`.)
        io.module_1, io.module_2, io.module_3.

    Output:
        widgets_dir (the default is `declare_qtquick.widgets.api.*`)
    """
    data_r = loads(io.json_3)  # type: TJson3Data

    # data_r 的数据结构比较复杂. 下面的代码逻辑主要基于 `io.json_4.<data>.<key
    # :qtquick>` 进行观察和编写.

    lk.logd('creating dirs')
    _create_dirs(widgets_dir, data_r.keys())

    BILTuple = namedtuple('BIL', ('base', 'init', 'list'))
    tmpl_files = BILTuple(
        f'{io.resources_dir}/widgets_template/__base__.txt',
        f'{io.resources_dir}/widgets_template/__init__.txt',
        f'{io.resources_dir}/widgets_template/__list__.txt',
    )
    tmpl_data = BILTuple(*map(loads, tmpl_files))

    with lk.counting(len(data_r)):
        for package, v0 in data_r.items():
            if package == '': continue
            lk.logdx(package)

            target_dir = f'{widgets_dir}/{package.replace(".", "/").lower()}'
            target_files = BILTuple(
                f'{target_dir}/__base__.py',
                f'{target_dir}/__init__.py',
                f'{target_dir}/__list__.py',
            )
            assert not xpath.exists(target_files.init)

            # generate target files (__base__, __init__, __list__).
            _generate_base(tmpl_files.base, target_files.base)
            _generate_init(tmpl_data.init, target_files.init, package=package)
            _generate_list(tmpl_data.list, target_files.list, data=v0)

    # put readme and __init__ files in api root dir.
    shutil.copyfile(io.readme, f'{widgets_dir}/readme.md')
    shutil.copyfile(io.api_init, f'{widgets_dir}/__init__.py')

# ------------------------------------------------------------------------------

def _create_dirs(widgets_dir, packages):
    dirs_ = set()

    for pkg in packages:
        if pkg == '':
            continue
        else:
            pkg = pkg.lower()

        tmp = widgets_dir
        for node in pkg.split('.'):
            tmp += '/' + node
            dirs_.add(tmp)

    with lk.counting(len(dirs_)):
        for d in sorted(dirs_):
            lk.logx(xpath.relpath(d, widgets_dir))
            if not xpath.exists(d):
                mkdir(d)

def _generate_base(file_i: TPath, file_o: TPath):
    # there is no placeholders in file_i. we just copy file_i to file_o.
    shutil.copyfile(file_i, file_o)

def _generate_init(tmpl_i: str, file_o: TPath, package):
    # tmpl: 'template'
    # kwargs: {'package': <str>}
    dumps(
        tmpl_i.format(QMLTYPE=package).strip(),
        file_o
    )

def _generate_list(tmpl_i: str, file_o: TPath, data: TWidgetData):
    # tmpl: 'template'
    # kwargs: {
    #   'data': {  # type: TWidgetData
    #       <str widget_name>: {
    #           'parent': (<str parent_package>,
    #                      <str parent_name>),
    #           'props': {
    #               <str prop_name>: <str prop_type>,
    #               ...
    #           }
    #       }
    #   }
    # }

    base_component = 'C'
    # # base_component = 'Component'

    widgets_dict = {}  # type: TWidgetSheetData2
    widget_tmpl = dedent('''
        class {WIDGET}({PARENT}, {PROP_SHEET}):
            pass
    ''').strip()

    with lk.counting(len(data)):
        for widget_name, v0 in data.items():
            parent_name = v0['parent'] or base_component
            lk.logax(widget_name, parent_name)

            widgets_dict[widget_name] = (
                parent_name,
                widget_tmpl.format(
                    WIDGET=widget_name,
                    PARENT=parent_name,
                    PROP_SHEET=f'W.Ps{widget_name}'
                )
            )

    dumps(tmpl_i.format(WIDGETS='\n\n\n'.join(
        sort_formatted_list(widgets_dict, (base_component,))
    )).strip(), file_o)

if __name__ == '__main__':
    # mkdir('../../../tests/api')
    # main('../../../tests/api')

    mkdir(io.widgets_dir)
    main(io.widgets_dir)