"""
Given a string text, you want to use the characters of text to form as many instances of the word "balloon" as possible.

You can use each character in text at most once. Return the maximum number of instances that can be formed.

Example 1:
Input: text = "nlaebolko"
Output: 1

Example 2:
Input: text = "loonbalxballpoon"
Output: 2

Example 3:
Input: text = "leetcode"
Output: 0
"""
import collections

def maxNumberOfBalloons(text: str) -> int:

    #count instances of how char
    count = Counter(text)
    print(count)
    #var for instances of balon
    instances = float('inf')

    for char in "balon":
        if char in "ban":
            instances = min(instances, count[char])
        else:
            instances = min(instances, count[char]//2)
    return instances
#         arr = list(text)
#         print(arr)

    #count how many times balloon can be spelled from a string
    # need only balon because cant have the same key in a dict
    #use floor div for l and o
    # each letter can only be used once

#         #make counter
#         counter = 0
#         #make dict
#         letters = {}
#         # populate dict with letters and number of times they occur
#         for i in range(arr):
#             letters[i] = arr[i]
#         print(letters)
#         #if it can make ballon update counter
#         #pop letters off dict
#         #if another instance of ballon can be spelled, update counter
#         #else end
#         # return counter

print(maxNumberOfBalloons("nlaebolko"))


name = input("Type your name: ")
print("Welcome", name, "to this adventure!")

answer = input("You are on a dirt road. It has come to an end and you can go left or right. Which way would you like to go? (left/right) ").lower()

if answer == "left":
    answer = input("You come to a river. You can walk around it or swim across? (walk/swim) ").lower()
    if answer == "swim":
        print("You swam across and were eaten by a snake.")
    elif answer == "walk":
        print("You walked for many miles, ran out of water and you lost the game.")
    else:
        print("Not a valid option. You lose.")
elif answer == "right":
    answer = input("You come to a bridge. It looks wobbly. Do you want to cross it or head back? (cross/back) ").lower()
    if answer == "back":
        print("You went back the way you came and drove home.")
    elif answer == "cross":
        answer = input("You cross the bridge and meet a stranger. Do you talk to them? (yes/no) ").lower()
        if answer == "no":
            print("The stranger became angry and stole your money.")
        elif answer == "yes":
            print("The stranger gave you a a magical map. You win!")
        else:
            print("Not a valid option. You lose.")
    else:
        print("Not a valid option. You lose.")
else:
    print("Not a valid option. You lose.")


import sys
import pandas as pd
from sqlalchemy import create_engine
import re
import nltk
import numpy as np
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
import warnings

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer,TfidfTransformer
from sklearn.multioutput import MultiOutputClassifier
from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier
from sklearn.metrics import classification_report,accuracy_score,precision_score,recall_score,f1_score,confusion_matrix
from sklearn.model_selection import train_test_split,GridSearchCV
from sklearn.externals import joblib

warnings.filterwarnings('ignore')
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')

# load data from database
def load_data(database_filepath):
    '''
    Load data from database.
    Args:
        database_filepath(str): Filepath of database.
    Returns:
        X(ndarray): Messages.
        Y(ndarray): Categories.
        category_names(list): Names of categories
    '''
    engine = create_engine(f'sqlite:///{database_filepath}')
    t_name = database_filepath.split('/')[-1].split('.')[0]
    df = pd.read_sql_table(table_name=t_name,con=engine)
    category_names = list(df.columns[4:])
    X = df['message'].values
    Y = df.drop(['id','message','original','genre'],axis=1).values

    return (X,Y,category_names)

def tokenize(text):
    '''
    Tokenize the text.
    Args:
        text(str): File for tokenize.
    Returns:
        tokens(list): Tokens.
    '''
    stop_words = stopwords.words("english")
    lemmatizer = WordNetLemmatizer()
    # normalize case and remove punctuation
    text = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())

    # tokenize text
    tokens = word_tokenize(text)

    # lemmatize and remove stop words
    tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words]

    return tokens

def build_model():

    pipeline = Pipeline([
        ('vect',CountVectorizer(tokenizer=tokenize)),
        ('tfidf',TfidfTransformer()),
        ('clf',MultiOutputClassifier(RandomForestClassifier(random_state=7)))
        ])

    parameters = {'tfidf__use_idf': (True, False),
    'clf__estimator__min_samples_leaf':[1,3,5],
    'clf__estimator__n_estimators':[100,200]}

    model = GridSearchCV(pipeline,param_grid=parameters,cv=5)

    return model

def evaluate_model(model, X_test, Y_test, category_names):
    Y_pred = model.predict(X_test)
    accuracy_list = []
    precision_list = []
    recall_list = []
    f1_list = []
    # bad_categories = []

    for x in range(len(category_names)):
        accuracy =  accuracy_score(Y_test[:, x], Y_pred[:, x])
        precision = precision_score(Y_test[:, x], Y_pred[:, x],average='weighted')
        recall = recall_score(Y_test[:,x], Y_pred[:, x],average='weighted')
        f1 = f1_score(Y_test[:, x], Y_pred[:, x],average='weighted')
        accuracy_list.append(accuracy)
        precision_list.append(precision)
        recall_list.append(recall)
        f1_list.append(f1)
        # if accuracy < 0.90:
        #     bad_categories.append(x)
        print(category_names[x])
        print("\tAccuracy: {:.3f}\t\t Precision: {:.3f}\t\t Recall: {:.3f}\t\t F1_score: {:.3f}".\
            format(accuracy,precision,recall,f1))
    print('Overall mean')
    print("\tAccuracy: {:.3f}\t\t Precision: {:.3f}\t\t Recall: {:.3f}\t\t F1_score: {:.3f}".\
        format(np.mean(accuracy_list),np.mean(precision_list),np.mean(recall_list),np.mean(f1_list)))

def save_model(model, model_filepath):
    joblib.dump(model, model_filepath)

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


from _collections import deque

MAP = [list(map(str, input().strip())) for _ in range(8)]
visited = [[[False for _ in range(9)] for _ in range(8)] for _ in range(8)]
dy = [1, 1, 0, -1, 0,  -1, -1, 0, 1]
dx = [0, 1, 1, 1, 0, 0, -1, -1, -1]

flag = False
Q = deque()
Q.append([7, 0, 0])
visited[7][0][0] = True

while Q:
    y, x, cnt = Q.popleft()
    if y == 0 and x == 7:
        flag = True
        break

    for dir in range(9):
        ty, tx = y + dy[dir], x + dx[dir]
        tcnt = min(cnt + 1, 8)
        if ty < 0 or ty >= 8 or tx < 0 or tx >= 8: continue

        if ty - cnt >= 0 and MAP[ty - cnt][tx] == '#':
            continue
        if ty - cnt - 1 >= 0 and MAP[ty - cnt - 1][tx] == '#':
            continue
        if not visited[ty][tx][tcnt]:
            visited[ty][tx][tcnt] = True
            Q.append([ty, tx, tcnt])

if not flag:
    print(0)
else:
    print(1)

BOJ/BOJ_16954_움직이는미로탈출.py

BOJ_16954_움직이는미로탈출.py


from datetime import datetime
FIRST_ACCEPTED_YEAR = 2013
LAST_ACCEPTED_YEAR = 2017

def convert_scoreboard_directory_to_date(base_scoreboard_directory):
    directory = base_scoreboard_directory[:-1]
    day = directory[-2:]

    directory = directory[:-3]
    month_slash_index = directory.rfind('/')
    month = directory[month_slash_index + 1: month_slash_index + 3]

    directory = directory[:month_slash_index]
    year = directory[-4:]
    return {'day': int(day), 'month': int(month), 'year': int(year)}

def extract_date_directory(base_scoreboard_directory):
    date = convert_scoreboard_directory_to_date(base_scoreboard_directory)
    day = date['day']
    month = date['month']
    year = date['year']

    directory = str(year) + '/' + convert_month_int_to_string(month)
    directory += '/' + convert_number_to_string(day) + '/'
    return directory

def get_earliest_possible_date_directory():
    year = FIRST_ACCEPTED_YEAR
    year_game_data = get_first_and_last_game_of_year(year)
    day = year_game_data['start_day']
    month = year_game_data['start_month']

    directory = str(year) + '/' + convert_month_int_to_string(month)
    directory += '/' + convert_number_to_string(day) + '/'
    return directory

def convert_month_int_to_string(month):
    if month == 3:
        return '03_March'
    elif month == 4:
        return '04_April'
    elif month == 5:
        return '05_May'
    elif month == 6:
        return '06_June'
    elif month == 7:
        return '07_July'
    elif month == 8:
        return '08_August'
    elif month == 9:
        return '09_September'
    elif month == 10:
        return '10_October'
    else:
        raise Exception('Invalid month integer: ' + str(month))

def get_default_start_and_end_of_month(month):
    if month == 3:
        return {'start': 1, 'end': 31}
    elif month == 4:
        return {'start': 1, 'end': 30}
    elif month == 5:
        return {'start': 1, 'end': 31}
    elif month == 6:
        return {'start': 1, 'end': 30}
    elif month == 7:
        return {'start': 1, 'end': 31}
    elif month == 8:
        return {'start': 1, 'end': 31}
    elif month == 9:
        return {'start': 1, 'end': 30}
    elif month == 10:
        return {'start': 1, 'end': 31}
    else:
        raise Exception('Invalid month integer: ' + str(month))

def convert_number_to_string(number):
    if 0 <= number < 10:
        string = '0' + str(number)
        return string
    return str(number)

def is_valid_year(year):
    assert FIRST_ACCEPTED_YEAR <= year <= LAST_ACCEPTED_YEAR, \
        'This given year is not currently accepted: ' + str(year)

def is_valid_day_in_month(month, day):
    assert 3 <= month <= 10, 'This given month is not currently accepted: ' + str(month)
    # The day must exist in the month
    if month in [3, 5, 7, 8, 10]:
        assert 1 <= day <= 31, 'Day ' + str(day) + 'does not exist in this month: ' + str(month)
    if month in [4, 6, 9]:
        assert 1 <= day <= 30, 'Day ' + str(day) + 'does not exist in this month: ' + str(month)

def get_first_and_last_game_of_year(year):
    if year == 2013:
        start_day = 31
        start_month = 3
        end_day = 30
        end_month = 9
        all_star_day = 16

    elif year == 2014:
        start_day = 31
        start_month = 3
        end_day = 28
        end_month = 9
        all_star_day = 15

    elif year == 2015:
        start_day = 5
        start_month = 4
        end_day = 4
        end_month = 10
        all_star_day = 14

    elif year == 2016:
        start_day = 4
        start_month = 4
        end_day = 2
        end_month = 10
        all_star_day = 12

    elif year == 2017:
        start_day = 2
        start_month = 4
        end_day = 1
        end_month = 10
        all_star_day = 11

    else:
        raise Exception('Invalid year: ' + str(year))

    return dict(start_day=start_day, start_month=start_month, end_day=end_day,
                end_month=end_month, all_star_day=all_star_day)

def date_directories_are_the_same(first_date_directory, second_date_directory):
    first_date_data = convert_scoreboard_directory_to_date(first_date_directory)
    second_date_data = convert_scoreboard_directory_to_date(second_date_directory)

    if these_dates_are_the_same(first_date_data, second_date_data):
        return True
    return False

def first_directory_is_earlier_or_the_same(first_date_directory, second_date_directory):
    first_date_data = convert_scoreboard_directory_to_date(first_date_directory)
    second_date_data = convert_scoreboard_directory_to_date(second_date_directory)

    if these_dates_are_the_same(first_date_data, second_date_data):
        return True
    return the_first_date_is_earlier(first_date_data, second_date_data)

def first_date_directory_is_earlier(first_date_directory, second_date_directory):
    first_date_data = convert_scoreboard_directory_to_date(first_date_directory)
    second_date_data = convert_scoreboard_directory_to_date(second_date_directory)

    if these_dates_are_the_same(first_date_data, second_date_data):
        return False
    return the_first_date_is_earlier(first_date_data, second_date_data)

def the_first_date_is_earlier(first_given_date, second_given_date):
    # Return True if the first comes before the second; otherwise return False
    first_given_day = first_given_date['day']
    first_given_month = first_given_date['month']
    first_given_year = first_given_date['year']

    second_given_day = second_given_date['day']
    second_given_month = second_given_date['month']
    second_given_year = second_given_date['year']

    # Continue if the years are the same
    if first_given_year < second_given_year:
        return True
    if first_given_year > second_given_year:
        return False

    # Continue if the months are the same
    if first_given_month < second_given_month:
        return True
    if first_given_month > second_given_month:
        return False

    # Continue if the days are the same
    if first_given_day < second_given_day:
        return True
    if first_given_day > second_given_day:
        return False
    # Dates are the same should be called first to prevent this case
    return False

def these_dates_are_the_same(first_given_date, second_given_date):
    # Return True if and only if the two dates are the same
    first_given_day = first_given_date['day']
    first_given_month = first_given_date['month']
    first_given_year = first_given_date['year']

    second_given_day = second_given_date['day']
    second_given_month = second_given_date['month']
    second_given_year = second_given_date['year']

    if first_given_year != second_given_year:
        return False
    if first_given_month != second_given_month:
        return False
    if first_given_day != second_given_day:
        return False
    return True

def is_summer_month(base_scoreboard_directory):
    date = convert_scoreboard_directory_to_date(base_scoreboard_directory)
    month = date['month']

    if 5 < month < 9:
        return True
    return False


import os
import slack
from get_firestore_tracked_cases import get_firestore_tracked_cases

def list_tracked_cases_all(channel_id, user_id):
    """
    Display all the Google Cloud support cases being tracked in the Slack
    worskpace to the user that submitted the command.

    Parameters
    ----------
    channel_id : str
      unique string used to idenify a Slack channel. Used to send messages to
      the channel
    user_id : str
      the Slack user_id of the user who submitted the request. Used to send
      ephemeral messages to the user
    """
    client = slack.WebClient(token=os.environ.get("SLACK_TOKEN"))
    tracked_cases = get_firestore_tracked_cases()

    all_tracked_cases = []
    for tc in tracked_cases:
        temp = {"channel": tc["channel_name"], "case": tc["case"]}
        all_tracked_cases.append(temp)

    if len(all_tracked_cases) > 0:
        client.chat_postEphemeral(
            channel=channel_id,
            user=user_id,
            text=f"Currently tracking cases {all_tracked_cases}")
    else:
        client.chat_postEphemeral(
            channel=channel_id,
            user=user_id,
            text="There are no cases currently being tracked in Slack")

if __name__ == "__main__":
    test_channel_id = os.environ.get("TEST_CHANNEL_ID")
    test_user_id = os.environ.get("TEST_USER_ID")
    list_tracked_cases_all(test_channel_id, test_user_id)


###############################################################################
#                                                                             #
#    This program is free software: you can redistribute it and/or modify     #
#    it under the terms of the GNU General Public License as published by     #
#    the Free Software Foundation, either version 3 of the License, or        #
#    (at your option) any later version.                                      #
#                                                                             #
#    This program is distributed in the hope that it will be useful,          #
#    but WITHOUT ANY WARRANTY; without even the implied warranty of           #
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            #
#    GNU General Public License for more details.                             #
#                                                                             #
#    You should have received a copy of the GNU General Public License        #
#    along with this program. If not, see <http://www.gnu.org/licenses/>.     #
#                                                                             #
###############################################################################

import unittest

from vtracker import VTracker
from vtracker.exceptions import MissingVersion, DuplicateEntity

class TestVTracker(unittest.TestCase):

    def test___init__(self):
        vt = VTracker(('a', 'b', 'c'))
        self.assertDictEqual({'a': 0, 'b': 1, 'c': 2}, vt._ver_to_idx)
        self.assertTupleEqual(('a', 'b', 'c'), vt._idx_to_ver)
        self.assertEqual(0, len(vt._uid_to_node))
        self.assertEqual(0, len(vt._uid_to_edge))

    def test_add(self):
        """
        +--------------+--------+--------------+
        |     1        |   2    |      3       |
        +--------------+--------+--------------+
        | a: x         | a: x   | a: y         |
        |              | b: y,z |              |
        | Missing: y,z |        | Missing: x,z |
        +--------------+--------+--------------+
        """
        vt = VTracker(('1', '2', '3'))
        vt.add('x', {'1': 'a', '2': 'a'})
        vt.add('y', {'2': 'b', '3': 'a'})
        vt.add('z', {'2': 'b'})
        mis = vt.str_na

        # Check the nodes
        self.assertEqual(len(vt._graph._nodes), 6)
        n_1_x = vt._graph.get_node(('1', 'a'))
        self.assertSetEqual(n_1_x._edges_in, set())
        self.assertSetEqual(n_1_x._edges_out, {(('1', 'a'), ('2', 'a'))})
        self.assertSetEqual(n_1_x.attrs['uid'], {'x'})

        n_1_yz = vt._graph.get_node(('1', mis))
        self.assertSetEqual(n_1_yz._edges_in, set())
        self.assertSetEqual(n_1_yz._edges_out, {(('1', mis), ('2', 'b'))})
        self.assertSetEqual(n_1_yz.attrs['uid'], {'y', 'z'})

        n_2_x = vt._graph.get_node(('2', 'a'))
        self.assertSetEqual(n_2_x._edges_in, {(('1', 'a'), ('2', 'a'))})
        self.assertSetEqual(n_2_x._edges_out, {(('2', 'a'), ('3', mis))})
        self.assertSetEqual(n_2_x.attrs['uid'], {'x'})

        n_2_yz = vt._graph.get_node(('2', 'b'))
        self.assertSetEqual(n_2_yz._edges_in, {(('1', mis), ('2', 'b'))})
        self.assertSetEqual(n_2_yz._edges_out, {(('2', 'b'), ('3', 'a')),
                                                (('2', 'b'), ('3', mis))})
        self.assertSetEqual(n_2_yz.attrs['uid'], {'y', 'z'})

        n_3_y = vt._graph.get_node(('3', 'a'))
        self.assertSetEqual(n_3_y._edges_in, {(('2', 'b'), ('3', 'a'))})
        self.assertSetEqual(n_3_y._edges_out, set())
        self.assertSetEqual(n_3_y.attrs['uid'], {'y'})

        n_3_xz = vt._graph.get_node(('3', mis))
        self.assertSetEqual(n_3_xz._edges_in, {(('2', 'a'), ('3', mis)),
                                               (('2', 'b'), ('3', mis))})
        self.assertSetEqual(n_3_xz._edges_out, set())
        self.assertSetEqual(n_3_xz.attrs['uid'], {'x', 'z'})

        # Check the edges
        self.assertEqual(len(vt._graph._edges), 5)
        e_1a_2a = vt._graph.get_edge(('1', 'a'), ('2', 'a'))
        self.assertSetEqual(e_1a_2a.attrs['uid'], {'x'})

        e_mis1_2b = vt._graph.get_edge(('1', mis), ('2', 'b'))
        self.assertSetEqual(e_mis1_2b.attrs['uid'], {'y', 'z'})

        e_2a_3mis = vt._graph.get_edge(('2', 'a'), ('3', mis))
        self.assertSetEqual(e_2a_3mis.attrs['uid'], {'x'})

        e_2b_3mis = vt._graph.get_edge(('2', 'b'), ('3', mis))
        self.assertSetEqual(e_2b_3mis.attrs['uid'], {'z'})

        e_2b_3a = vt._graph.get_edge(('2', 'b'), ('3', 'a'))
        self.assertSetEqual(e_2b_3a.attrs['uid'], {'y'})

        # Check the node indices
        self.assertSetEqual(vt._uid_to_node['x'], {('1', 'a'), ('2', 'a'), ('3', mis)})
        self.assertSetEqual(vt._uid_to_node['y'], {('1', mis), ('2', 'b'), ('3', 'a')})
        self.assertSetEqual(vt._uid_to_node['z'], {('1', mis), ('2', 'b'), ('3', mis)})

        # Check the edge indices
        self.assertSetEqual(vt._uid_to_edge['x'], {(('1', 'a'), ('2', 'a')),
                                                   (('2', 'a'), ('3', mis))})
        self.assertSetEqual(vt._uid_to_edge['y'], {(('1', mis), ('2', 'b')),
                                                   (('2', 'b'), ('3', 'a'))})
        self.assertSetEqual(vt._uid_to_edge['z'], {(('1', mis), ('2', 'b')),
                                                   (('2', 'b'), ('3', mis))})

    def test_add_raises_MissingVersion(self):
        vt = VTracker(('1', '2', '3'))
        self.assertRaises(MissingVersion, vt.add, 'x', {'1': 'a', '9': 'a'})

    def test_add_raises_DuplicateEntity(self):
        vt = VTracker(('1', '2', '3'))
        vt.add('x', {'1': 'a', '2': 'a', '3': 'a'})
        self.assertRaises(DuplicateEntity, vt.add, 'x', {'1': 'a'})

    def test__build_uid_paths(self):
        """
        +--------------+--------+--------------+
        |     1        |   2    |      3       |
        +--------------+--------+--------------+
        | a: x         | a: x   | a: y         |
        |              | b: y,z |              |
        | Missing: y,z |        | Missing: x,z |
        +--------------+--------+--------------+
        """
        vt = VTracker(('1', '2', '3'))
        vt.add('x', {'1': 'a', '2': 'a'})
        vt.add('y', {'2': 'b', '3': 'a'})
        vt.add('z', {'2': 'b'})
        mis = vt.str_na
        nodes, edges = vt._build_uid_paths()

        # Check node highlighting
        node_x = set()
        node_x.add(vt._graph.get_node(('1', 'a'))._node_id)
        node_x.add(vt._graph.get_node(('2', 'a'))._node_id)
        node_x.add(vt._graph.get_node(('3', mis))._node_id)
        self.assertSetEqual(node_x, nodes['x'])

        node_y = set()
        node_y.add(vt._graph.get_node(('1', mis))._node_id)
        node_y.add(vt._graph.get_node(('2', 'b'))._node_id)
        node_y.add(vt._graph.get_node(('3', 'a'))._node_id)
        self.assertSetEqual(node_y, nodes['y'])

        node_z = set()
        node_z.add(vt._graph.get_node(('1', mis))._node_id)
        node_z.add(vt._graph.get_node(('2', 'b'))._node_id)
        node_z.add(vt._graph.get_node(('3', mis))._node_id)
        self.assertSetEqual(node_z, nodes['z'])

        # Check edge highlighting
        edge_x = set()
        edge_x.add(vt._graph.get_edge(('1', 'a'), ('2', 'a'))._edge_id)
        edge_x.add(vt._graph.get_edge(('2', 'a'), ('3', mis))._edge_id)
        self.assertSetEqual(edge_x, edges['x'])

        edge_y = set()
        edge_y.add(vt._graph.get_edge(('1', mis), ('2', 'b'))._edge_id)
        edge_y.add(vt._graph.get_edge(('2', 'b'), ('3', 'a'))._edge_id)
        self.assertSetEqual(edge_y, edges['y'])

        edge_z = set()
        edge_z.add(vt._graph.get_edge(('1', mis), ('2', 'b'))._edge_id)
        edge_z.add(vt._graph.get_edge(('2', 'b'), ('3', mis))._edge_id)
        self.assertSetEqual(edge_z, edges['z'])

    def test_as_sankey_json(self):
        """
        +--------------+--------+--------------+
        |     1        |   2    |      3       |
        +--------------+--------+--------------+
        | a: x         | a: x   | a: y         |
        |              | b: y,z |              |
        | Missing: y,z |        | Missing: x,z |
        +--------------+--------+--------------+
        """
        vt = VTracker(('1', '2', '3'))
        vt.add('x', {'1': 'a', '2': 'a'})
        vt.add('y', {'2': 'b', '3': 'a'})
        vt.add('z', {'2': 'b'})
        mis = vt.str_na
        nodes, edges = vt._build_uid_paths()

        # Convert the sankey results to a dict for easy checking
        sankey_json = vt.as_sankey_json()
        test_nodes = {x['id']: x for x in sankey_json['nodes']}
        test_edges = {x['id']: x for x in sankey_json['links']}

        nodes_exp = dict()
        id_a1 = vt._graph.get_node(('1', 'a'))._node_id
        nodes_exp[id_a1] = {'id': id_a1,
                            'name': 'a',
                            'col': '1',
                            'total': 1,
                            'linkHighlightId': list(edges['x']),
                            'nodeHighlightId': list(nodes['x'])}

        id_mis_1 = vt._graph.get_node(('1', mis))._node_id
        nodes_exp[id_mis_1] = {'id': id_mis_1,
                               'name': mis,
                               'col': '1',
                               'total': 2,
                               'linkHighlightId': list(edges['y'].union(edges['z'])),
                               'nodeHighlightId': list(nodes['y'].union(nodes['z']))}

        id_a2 = vt._graph.get_node(('2', 'a'))._node_id
        nodes_exp[id_a2] = {'id': id_a2,
                            'name': 'a',
                            'col': '2',
                            'total': 1,
                            'linkHighlightId': list(edges['x']),
                            'nodeHighlightId': list(nodes['x'])}

        id_b2 = vt._graph.get_node(('2', 'b'))._node_id
        nodes_exp[id_b2] = {'id': id_b2,
                            'name': 'b',
                            'col': '2',
                            'total': 2,
                            'linkHighlightId': list(edges['y'].union(edges['z'])),
                            'nodeHighlightId': list(nodes['y'].union(nodes['z']))}

        id_a3 = vt._graph.get_node(('3', 'a'))._node_id
        nodes_exp[id_a3] = {'id': id_a3,
                            'name': 'a',
                            'col': '3',
                            'total': 1,
                            'linkHighlightId': list(edges['y']),
                            'nodeHighlightId': list(nodes['y'])}

        id_mis_3 = vt._graph.get_node(('3', mis))._node_id
        nodes_exp[id_mis_3] = {'id': id_mis_3,
                               'name': mis,
                               'col': '3',
                               'total': 2,
                               'linkHighlightId': list(edges['x'].union(edges['z'])),
                               'nodeHighlightId': list(nodes['x'].union(nodes['z']))}

        for exp_node_id, exp_node in nodes_exp.items():
            test_node = test_nodes[exp_node_id]
            self.assertDictEqual(exp_node, test_node)

        edges_exp = dict()
        id_1 = vt._graph.get_edge(('1', 'a'), ('2', 'a'))._edge_id
        edges_exp[id_1] = {'id': id_1,
                           'source': vt._graph.get_node(('1', 'a'))._node_id,
                           'target': vt._graph.get_node(('2', 'a'))._node_id,
                           'value': 1,
                           'linkHighlightId': list(edges['x']),
                           'nodeHighlightId': list(nodes['x'])}

        id_2 = vt._graph.get_edge(('1', mis), ('2', 'b'))._edge_id
        edges_exp[id_2] = {'id': id_2,
                           'source': vt._graph.get_node(('1', mis))._node_id,
                           'target': vt._graph.get_node(('2', 'b'))._node_id,
                           'value': 2,
                           'linkHighlightId': list(edges['y'].union(edges['z'])),
                           'nodeHighlightId': list(nodes['y'].union(nodes['z']))}

        id_3 = vt._graph.get_edge(('2', 'a'), ('3', mis))._edge_id
        edges_exp[id_3] = {'id': id_3,
                           'source': vt._graph.get_node(('2', 'a'))._node_id,
                           'target': vt._graph.get_node(('3', mis))._node_id,
                           'value': 1,
                           'linkHighlightId': list(edges['x']),
                           'nodeHighlightId': list(nodes['x'])}

        id_4 = vt._graph.get_edge(('2', 'b'), ('3', 'a'))._edge_id
        edges_exp[id_4] = {'id': id_4,
                           'source': vt._graph.get_node(('2', 'b'))._node_id,
                           'target': vt._graph.get_node(('3', 'a'))._node_id,
                           'value': 1,
                           'linkHighlightId': list(edges['y']),
                           'nodeHighlightId': list(nodes['y'])}

        id_5 = vt._graph.get_edge(('2', 'b'), ('3', mis))._edge_id
        edges_exp[id_5] = {'id': id_5,
                           'source': vt._graph.get_node(('2', 'b'))._node_id,
                           'target': vt._graph.get_node(('3', mis))._node_id,
                           'value': 1,
                           'linkHighlightId': list(edges['z']),
                           'nodeHighlightId': list(nodes['z'])}

        for exp_edge_id, exp_edge in edges_exp.items():
            test_edge = test_edges[exp_edge_id]
            self.assertDictEqual(exp_edge, test_edge)


'''
Python CSC241
Day 5 - 6/20/2016
'''

'''
**Concept: Importing Random

random.randrange(start, stop) - int
random.uniform(start, stop) - float
random.choice(sequence) - returns random item
random.sample(sequence, k) - returns k items
random.shuffle(sequence) - shuffles

>>> random.random()        # Random float x, 0.0 <= x < 1.0
0.37444887175646646
>>> random.uniform(1, 10)  # Random float x, 1.0 <= x < 10.0
1.1800146073117523
>>> random.randint(1, 10)  # Integer from 1 to 10, endpoints included
7
>>> random.randrange(0, 101, 2)  # Even integer from 0 to 100
26
>>> random.choice('abcdefghij')  # Choose a random element
'c'

>>> items = [1, 2, 3, 4, 5, 6, 7]
>>> random.shuffle(items)
>>> items
[7, 3, 2, 5, 6, 4, 1]

>>> random.sample([1, 2, 3, 4, 5],  3)  # Choose 3 elements
[4, 1, 5]

SHELL PRACTICE (RANDOM)
>>> import random
>>> random.randrange(1,6)
3
>>> random.randrange(1,7)
6
>>> [random.randrange(1,7) for i in range (100)]
[3, 3, 6, 6, 3, 5, 5, 4, 5, 6, 3, 1, 6, 6, 2, 2, 2, 5, 5, 6, 1, 5, 3, 6, 6, 3, 3, 2, 1, 4, 1, 6, 2, 4, 1, 3, 5, 4, 5, 1, 2, 2, 1, 4, 1, 5, 5, 1, 5, 6, 2, 6, 5, 2, 4, 1, 4, 5, 4, 6, 6, 1, 5, 4, 1, 1, 6, 4, 1, 3, 1, 3, 4, 5, 6, 6, 3, 4, 5, 3, 1, 1, 4, 2, 2, 6, 6, 4, 1, 4, 4, 2, 4, 5, 6, 3, 2, 4, 4, 1]
>>> random.uniform(0,1)
0.6177630359885248
>>> random.uniform(0,1)
0.9761416066932117

>>> height = random.uniform(100,200)
>>> height
151.9925745297827

>>> fruit = ['orange','pear','apple']
>>> random.choice(fruit)
'apple'

>>> random.sample(fruit,2)
['pear', 'orange']
>>> random.sample(fruit,2)
['pear', 'apple']

>>> random.randrange(0,2)
1
>>> random.randrange('h','t') - doesnt work
>>> random.choice(['h','t'])
't'

>>> fruit
['orange', 'pear', 'apple']
>>> random.shuffle(fruit)
>>> fruit
['pear', 'orange', 'apple']

**Concept Importing Math

>>> import math
>>> math.pi
3.141592653589793
>>> 'pi' in vars(math)
True
>>> vars(math)['pi']
3.141592653589793

>>> from math import pi
>>> pi
3.141592653589793
>>> from math import *
>>> cos
<built-in function cos>
>>> e
2.718281828459045
'''

#Program 1: Random 2 Dice Rolls | If 'Double' Roll Again | Return total Rolls
#Run Module for Program to execute (F5)
#Type DiceTotal() - Name of Method in Shell to execute.

'''
>>> DiceTotalV2()
5 5
2 4
16
>>> DiceTotal()
4 4
2 5
15
'''

import random

def DiceTotal():
    _total = 0

    dice1 = random.randrange(1,7) #7 not included
    dice2 = random.randrange(1,7)
    print(dice1, dice2)

    _total += dice1 + dice2

    #keep rolling if double
    while(dice1 == dice2):
        dice1 = random.randrange(1,7)
        dice2 = random.randrange(1,7)
        print(dice1, dice2)

        _total += dice1 + dice2

    return _total

#Version 2. DiceTotalV2() - Using Infinite Loop Method.

import random

def DiceTotalV2():
    _total = 0

    #keep rolling if double
    while True:
        dice1 = random.randrange(1,7)
        dice2 = random.randrange(1,7)
        print(dice1, dice2)

        _total += dice1 + dice2

        if(dice1 != dice2):
            return _total

'''
**Concept: Namespace
a dictionary where variables and their values are stored as pairs
when you define a variable, an entry in the current namspace is made
- variable:value

x = 99 # add 'x':99 to namespace

when you access a variable, its value is looked up in the namespace:
print (x)

lookup is in the following order
1) local function namespace
2) global/module namespace
3) builtins namespace
'''

x = 99

def f():
    x = 22
    print(vars())
    print(x)

def g():
    print(vars())
    print(x)

#f() - 'x' will be 22 found locally
#g() - 'x' will be 99 found in module

'''
***OVERVIEW***

method:   l.append('apple')
function: l=len(mylist)

numbers
    bool < int < float
    operators: +,-,*,/,**,%,//
    comparisons: <,>,<=,>=,==,!=
    functions: abs()

string
    containment: in, not in
    +, *(int)
    [][start:stop:step]
    methods - split,upper,lower,capitalize,format,count
    functions: len()
    immutable

list - indexed sequence of items
    in, not in
    [],[::]
    indexed by range
    mutable (editable)
    methods: append,pop,sort,count,remove,find,reverse
    functions: len,min,max,range

dictionary
    {key: val1, key1,val2...}
    d[key] = val
    indexed by keys
    not ordered
    mutable, keys are immutable
    in, not in - works in keys

tuple
    (,,,,)
    like a list but immutable(cant edit,only construct)
    doesnt have append,sort method

set
    {,,,,}
    like a list but,
    no duplicates, not ordered,indexed
    &,|
    ex. {1,3,4} & {1,2}

files
    open,read,write,readlines,close

math
    import math
    math.pi
    math.e
    math.sqrt, math.cos

random
    import random
    randrange, uniform
    choice,shuffle,sample

control structures/logic
    if,elif,else
    for, while
    loop mods: return,break,pass,continue

loop patterns
    iterate over sequence
        for i in lst
        for i in range()
        for char in string
        for key in dict
    indexed
        use when you need location
        for i in range(len(lst))
    nested loop
        2d list
        for row in table:
            for item in row:
    accumulator
        init,accumulate,return/
    for loops that execute at least one time
    loop and a half
    infinite loop

functions
    return vs print
    local vs global variables

'''

#Program 2: Create a Stack of Cups given an Integer
#Run Module for Program to execute (F5)
#Type GetStackHeight(<integer>) - Name of Method in Shell to execute.

'''
>>> GetStackHeight(5)
X
XX
2
>>> GetStackHeight(6)
X
XX
XXX
3
'''

def GetStackHeight(int_cups):
    _length = 1 #for next row to build
    _height = 0

    while(int_cups >= _length):
        print(_length * 'X')

        int_cups -= _length
        _height += 1
        _length += 1

    return _height

#Program 3: Return Set of Index Matching items from two Lists
#Run Module for Program to execute (F5)
#Type RetFindMatch(<list>, <list>) - Name of Method in Shell to execute.

'''
>>> RetIndexFindMatch([1,2,3,4],[2,4,6,8])
(1, 0)
>>> RetIndexFindMatch(['david','gabriel','faizan'],['pedro','faizan','ilia'])
(2, 1)
'''

def RetIndexFindMatch(list1, list2):

    for i in range(len(list1)):
        for j in range(len(list2)):

            if(list1[i] == list2[j]):
                return (i, j)
    return -1,-1

CSC241. Intro Python/CSC241. Class Notes (6)/Py-Day 5 CSC241.py


import pyspark
from operator import add
import time

"""
Transactions layout
0 - block_number: Block number where this transaction was in
1 - from_address: Address of the sender
2 - to_address: Address of the receiver. null when it is a contract creation transaction
3 - value: Value transferred in Wei (the smallest denomination of ether)
4 - gas: Gas provided by the sender
5 - gas_price : Gas price provided by the sender in Wei
6 - block_timestamp: Timestamp the associated block was registered at (effectively timestamp of the transaction)

ScamsCSV layout
0 - Scam ID
1 - Name
2 - URL
3 - Coin
4 - Category
5 - Sub-category
6 - Address
7 - Status
"""

sc = pyspark.SparkContext()

def is_good_line(line):
    try:
        fields = line.split(',')
        if len(fields) == 7: # transactions
            str(fields[2]) # to_addr
            if int(fields[3]) == 0:
                return False
        elif len(fields) == 8: # contracts
            str(fields[6]) # Scam addr
            str(fields[4]) # Scam category
        else:
            return False
        return True
    except:
        return False

def mapper_transactions(line):
    try:
        fields = line.split(',')
        to_addr = fields[2]
        wei = int(fields[3])
        raw_timestamp = int(fields[6])
        year_month = time.strftime('%Y-%m', time.gmtime(raw_timestamp))

        key = to_addr
        value = ( year_month, wei, 1 )
        return (key, value)
    except:
        pass

transactions = sc.textFile('/data/ethereum/transactions').filter(is_good_line)
scams = sc.textFile('scams.csv').filter(is_good_line)

scams_addr = scams.map(lambda l: (l.split(',')[6], l.split(',')[4]))

# calculate Wei received at each address in each month
step1 = transactions.map(mapper_transactions)
step2 = step1.join(scams_addr)
step3 = step2.map(lambda x: ((x[1][1], x[1][0][0]), (x[1][0][1], x[1][0][2])))
step4 = step3.reduceByKey(lambda a, b: (a[0]+b[0], a[1]+b[1]) )
step5 = step4.map(lambda x: '{},{},{},{},{}'.format(x[0][0], x[0][1], float(x[1][0]/1000000000000000000), x[1][1], float(x[1][0]/1000000000000000000)/x[1][1]))

step5.saveAsTextFile('out_monthly_scam_transactions')


# -*- coding: utf-8 -*-
"""
Created on Sat Dec 19 10:17:32 2020

@author: Medyan
"""

# coding: utf-8

"""
process-data.py is a group of functions which
        deal with sqlite database and
        make some manipulation for text
Author: Medyan AbdelHamid
Date: Dec. 2020
"""
import sqlite3
import numpy as np
import string
import re

emoticons_str = r"""
    (?:
        [:=;] # Eyes
        [oO\-]? # Nose (optional)
        [D\)\]\(\]/\\OpP] # Mouth
    )"""

regex_str = [
    emoticons_str,
    r'<[^>]+>', # HTML tags
    r'(?:@[\w_]+)', # @-mentions
    r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)", # hash-tags
    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+', # URLs

    r'(?:(?:\d+,?)+(?:\.?\d+)?)', # numbers
    r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
    r'(?:[\w_]+)', # other words
    r'(?:\S)' # anything else
]

tokens_re = re.compile(r'(' + '|'.join(regex_str) + ')', re.VERBOSE | re.IGNORECASE)
emoticon_re = re.compile(r'^' + emoticons_str + '$', re.VERBOSE | re.IGNORECASE)
db_file = 'C:/Users/ASUS/Documents/Thesis/Code/Python/SoureCode/fdata-2020.db'

def remove_url(text):
    import re
    return re.sub(r'^https?:\/\/.*[\r\n]*', '', text, flags=re.MULTILINE)

def strip_all_entities(text):
    entity_prefixes = ['@','#']
    for separator in  string.punctuation:
        if separator not in entity_prefixes :
            text = text.replace(separator,' ')
    words = []
    for word in text.split():
        word = word.strip()
        if word:
            if word[0] not in entity_prefixes:
                words.append(word)
    return ' '.join(words)

def remove_emoji(text):
    emoji_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text) # no emoji

def preprocess_tweet(text):
    from pyarabic.araby import strip_tashkeel, strip_tatweel, tokenize, is_arabicrange
    text = remove_url(text)
    text = strip_all_entities(text)
    text = strip_tashkeel(text)
    text = strip_tatweel(text)
    tokens = tokenize(text, conditions=is_arabicrange, morphs=strip_tashkeel)
    tokens = [token if emoticon_re.search(token) else token.lower() for token in tokens]
    return ' '.join(tokens)

def reverse_tweet(text):
    # split the string on space
    words = text.split()
    # reverse the words using reversed() function
    words = list(reversed(words))
    # join the words and return
    return ' '.join(words)

def remove_last_word(text):
    return text.rsplit(' ', 1)[0]

def remove_first_word(text):
    return text.split(' ', 1)[1]

def get_test_seq():
    conn = sqlite3.connect(db_file)
    sql = 'SELECT seq FROM SQLITE_SEQUENCE where name=\'TESTS\''
    cur = conn.cursor()
    cur.execute(sql)
    last = cur.fetchone()[0] + 1
    conn.close()
    return last

def add_test_to_database(results, notes):
    conn = sqlite3.connect(db_file)
    sql = "INSERT INTO TESTS (RESULTS, NOTES)  VALUES (?, ?)"
    args = (results, notes,)
    conn.execute(sql, args)
    conn.commit()

def normalize_tweets():
    conn = sqlite3.connect(db_file)
    sql = "SELECT * FROM TRAINSET "
    cur = conn.cursor()
    cur.execute(sql)
    records = cur.fetchall()
    for rec in records:
        # print(rec[0])
        # print(rec[4])
        # print(preprocess_tweet(rec[4]))
        # exit
        sql = "UPDATE TRAINSET SET FULLTEXT = ? WHERE NB = ?"
        args = (preprocess_tweet(rec[4]),rec[0],)
        conn.execute(sql, args)
    conn.commit()

def augment_hate_tweets():
    conn = sqlite3.connect(db_file)
    sql = "SELECT NB, FULLTEXT FROM TRAINSET WHERE ANOMALY = 1 "
    cur = conn.cursor()
    cur.execute(sql)
    records = cur.fetchall()
    for rec in records:
        # print(rec[0])
        # print(rec[4])
        # print(preprocess_tweet(rec[4]))
        # exit
        sql = "INSERT INTO TRAINSET (ANOMALY, FULLTEXT, FULLTEXT) VALUES (1,?,?) "
        rev1 = reverse_tweet(rec[1])
        rev2 = remove_last_word(rec[1])
        rev3 = remove_first_word(rec[1])
        args = (rev1, rev1)
        conn.execute(sql, args)
        args = (rev2, rev2)
        conn.execute(sql, args)
        args = (rev3, rev3)
        conn.execute(sql, args)
    conn.commit()

def get_vec(n_model,dim, token):
    vec = np.zeros(dim)
    if token not in n_model.wv:
        _count = 0
        for w in token.split("_"):
            if w in n_model.wv:
                _count += 1
                vec += n_model.wv[w]
        if _count > 0:
            vec = vec / _count
    else:
        vec = n_model.wv[token]
    return vec

# normalize all tweets
# normalize_tweets()
# augment hate tweets
augment_hate_tweets()


import sys

from PyQt5.QtGui import QIcon
from PyQt5.QtWidgets import QApplication, QWidget, QPushButton

def prepareGui():
    app= QApplication(sys.argv)  # gui for append all component sys.argv attach sys with prg
    window = QWidget()   # create window
    window.resize(400,500)
    # window.setWindowIcon(QIcon('Control.png')) # icon of window
    window.move(150,150) # x ,y of window
    window.setToolTip("Main Program")

    btn = QPushButton('Click',window)

    btn.resize(80,40)
    btn.move(40,60)
    btn.setToolTip("Clik to btn")
    btn.clicked.connect(print_events)
    window.show()
    #window.setGeometry(400,400,600,600)
    window.setWindowTitle("hello by QT")
    app.exec_() # main loop to create all compent in window and display in  app
                # exec_() for Qt and exec() reserverd word for python

def print_events():
    print(" test events ")

if __name__ == "__main__":
    prepareGui()


"""This script creates the reverse complement of a sequence.

The reverse_complement() function is the default function to create the reverse complement.
The reverse_complement_mitos() function is the one that should be used when dealing with MITOS annotations.
The reverse_complement_annotation() function adjusts the annotation coordinates of features to match the ones from the reverse complemented sequence.

"""

import sys
from Bio import SeqIO

def reverse_complement(in_gb, out_gb):

    # open original record and convert it to
    # its reverse complement (rc_record)
    record = SeqIO.read(in_gb, "genbank")
    rc_record = record.reverse_complement(id=record.id + "_rc")
    # needs to set molecule type as DNA because Biopython deletes
    # this info when parsing
    rc_record.annotations["molecule_type"] = "DNA"

    with open(out_gb, "w") as f:
        SeqIO.write(rc_record, f, "genbank")

    out_fasta = out_gb.replace(".gb", ".fasta")
    with open(out_fasta, "w") as f:
        SeqIO.write(rc_record, f, "fasta")

def reverse_complement_mitos(in_fasta, out_fasta):

    # open original record and convert it to
    # its reverse complement (rc_record)
    record = SeqIO.read(in_fasta, "fasta")
    rc_record = record.reverse_complement(id=record.id + "_rc")

    with open(out_fasta, "w") as f:
        SeqIO.write(rc_record, f, "fasta")

def reverse_complement_annotation(mitogenome_annotation, mitogenome_fasta, mitogenome_annotation_rc):

    for record in SeqIO.parse(mitogenome_fasta, "fasta"):
        mitogenome_len = len(record.seq)

    with open(mitogenome_annotation, "r") as f:
        for line in f:
            fields = line.strip().split()
            new_start = mitogenome_len - int(fields[3]) + 1
            new_end = mitogenome_len - int(fields[4]) + 1
            old_strand = fields[6]

            # switches the signal
            if old_strand == "+":
                new_strand = "-"
            else:
                new_strand = "+"

            # sets first coord as start
            # and second coord as end
            if new_start > new_end:
                b = new_start
                new_start = new_end
                new_end = b

            fields[3] = str(new_start)
            fields[4] = str(new_end)
            fields[6] = new_strand
            with open(mitogenome_annotation_rc, "a") as f2:
                f2.write("\t".join(fields) + "\n")

if __name__ == "__main__":
    in_gb = sys.argv[1]
    out_gb = sys.argv[2]
    reverse_complement(in_gb, out_gb)


import logging
from lxml import html
from joj.model.output_variable import OutputVariable
from joj.utils import constants

log = logging.getLogger(__name__)

class JulesOutputVariableParser(object):
    """
    Parser for the Jules documentation
    Parses the html output variables into the database objects
    """

    # Some variables depend on the value of nsmax - keep a record of those names here:
    _DEPENDS_ON_NSMAX = ['snow_ice_gb', 'snow_liq_gb', 'snow_ice_tile', 'snow_liq_tile',
                         'rgrainl', 'snow_ds', 'snow_ice', 'snow_liq', 'tsnow']

    #variable which should appear in the output list (must be lower case)
    _VARIABLES_THAT_SHOULD_BE_USED = \
        ['fqw_gb',
         'ecan_gb',
         'esoil_gb',
         'ei_gb',
         'ftl_gb',
         'cs_gb',
         'cv',
         'gpp_gb',
         'lit_c_mean',
         'npp_gb',
         'resp_p_gb',
         'resp_s_gb',
         'runoff',
         'surf_roff',
         'sub_surf_roff',
         'swet_tot',
         'swet_liq_tot',
         'snow_depth_gb',
         'snow_frac',
         'snow_mass_gb',
         'tstar_gb',
         'smc_tot',
         'rad_net',
         'lw_down',
         'sw_down',
         'wind',
         'tl1',
         'qw1',
         'pstar',
         'precip']

    def parse(self, file_path):
        """
        Run the parser on the output variables file and return a list of OutputVariables
        :param file_path: path of output-variables.html
        :return: list of OutputVariables
        """
        log.info("Parsing %s" % file_path)
        tree = html.parse(file_path)
        output_variables = []
        output_table_rows = tree.xpath(
            '//div[@id="variables-that-have-a-single-value-at-each-land-gridpoint"]/table/tbody/tr[@class="row-even"] '
            '| //div[@id="variables-that-have-a-single-value-at-each-land-gridpoint"]/table/tbody/tr[@class="row-odd"]'
            '| //div[@id="variables-that-have-a-single-value-at-each-gridpoint-land-and-sea"]/table/tbody/tr[@class="row-even"] '
            '| //div[@id="variables-that-have-a-single-value-at-each-gridpoint-land-and-sea"]/table/tbody/tr[@class="row-odd"]')

        for output_row in output_table_rows:
            output_variable = OutputVariable()
            name = str(output_row.xpath('td/tt/span[@class="pre"]/text()')[0])
            if name.lower() in self._VARIABLES_THAT_SHOULD_BE_USED:
                output_variable.name = name
                # Need to reconstruct the description because of <sup> tags
                desc_texts = output_row.xpath('td[2]/text() | td[2]/p[1]/text()')
                desc_tags = output_row.xpath('td[2]/sup/text() | td[2]/p[1]/sup/text() | td[2]/tt/span/text() '
                                             '| td[2]/a/tt/span/text()')
                output_variable.description = unicode(desc_texts[0])
                for i in range(len(desc_tags)):
                    output_variable.description += desc_tags[i]
                    output_variable.description += desc_texts[i+1]
                output_variable.depends_on_nsmax = output_variable.name in constants.DEPENDS_ON_NSMAX
                output_variables.append(output_variable)

        if (len(output_variables) != len(self._VARIABLES_THAT_SHOULD_BE_USED)):
            log.error("'Output variables' length, %s, if different to 'variables to use' length, %s "
                      % (len(output_variables), len(self._VARIABLES_THAT_SHOULD_BE_USED)))
            exit(-1)
        return output_variables

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    parser = JulesOutputVariableParser()
    file_path = '../docs/Jules/user_guide/html/output-variables.html'
    parser.parse(file_path)


import random as rnd
import numpy as np
from Rain import Rain
import threading
from matplotlib import pyplot as plt
import time

class Soil:
    LAYERS_SIZE = []
    NUMBER_OF_LAYERS = 5
    LAYERS_PENETRATION = []
    TIME = 0
    DAY_LIMIT_TIME = 20
    DAY = 0
    MONTH = 0
    SEASON = 'spring'
    SEASONS = ['winter', 'winter', 'spring', 'spring', 'spring', 'summer', 'summer', 'summer', 'autumn', 'autumn',
               'autumn', 'winter']
    SPEED = 0
    # Specific properties of Soil
    VALVE_CAPACITY = 40
    IS_WATERING = False
    LAYERS_MOISTURE = []
    RAIN_CHANCE = 0.2
    SEASON_CHANGING_RAIN_MUTATION = {'spring': +.02 , 'summer': -.01 , 'autumn': +0.01, 'winter': +0.01}
    SEASON_CHANGING_PENETRATION = {'spring': +.01, 'summer': +.04, 'autumn': +.0, 'winter': -.4}
    CHANGING_PENETRATION_CHANCE = 0.02
    DAY_EVAPORATE_BASE = 0.02
    DAY_EVAPORATE_ERROR = 0.008
    NIGHT_EVAPORATE_BASE = 0.001
    NIGHT_EVAPORATE_ERROR = 0.008

    input_water = 0

    def __init__(self):
        self.LAYERS_PENETRATION = [np.random.rand() for i in range(self.NUMBER_OF_LAYERS)]
        self.LAYERS_MOISTURE = [np.random.rand() for i in range(self.NUMBER_OF_LAYERS)]
        self.LAYERS_SIZE = [np.random.rand()*50 for i in range(self.NUMBER_OF_LAYERS)]
        self.rain = Rain(time_limit=100, intensity=3, speed=0.1)
        '''
        self.LAYERS_PENETRATION = [0.5, 0.5, 0.5, 0.5]
        self.LAYERS_MOISTURE = [0.7, 0.2, 0.1, 0.0]
        self.LAYERS_SIZE = [70, 80, 90, 90]
        '''

    def irrigation_handler(self):
        all_data = []
        counter = 0
        while self.DAY < 3:
            self.irrigate()
            counter = counter +1
            all_data.append(self.LAYERS_MOISTURE.copy())
            print(self.LAYERS_MOISTURE)
            '''
            print('TIME: '+str(self.TIME) + ' , DAY: '+str(self.DAY)+' , MONTH: '+
                  str(self.MONTH)+' , SEASON: '+str(self.SEASON) + ' , RAINING: ' + str(self.rain.active)+
                  ' , RAINING_TIME_LIMIT: ' + str(self.rain.time_limit))
            for j in range(0, s.NUMBER_OF_LAYERS):
                print(str(s.LAYERS_MOISTURE[j]) + " , " + str(s.LAYERS_SIZE[j]) + " , " + str(s.LAYERS_PENETRATION[j]))
            print()
            time.sleep(1)
            '''
        all_data = np.array(all_data)
        time_axis = np.arange(0, counter)
        print(all_data)
        for i in range(0, self.NUMBER_OF_LAYERS):
            plt.plot(time_axis, all_data[:, i], label='layer '+str(i))
            plt.xlabel('time')
            plt.ylabel('moisture')
        plt.legend()
        plt.show()

    # Calender
    def increase_time(self):
        if self.TIME >= self.DAY_LIMIT_TIME:
            self.TIME = 0
            self.DAY += 1
            if self.DAY >= 30:
                self.DAY = 0
                self.MONTH += 1
                self.SEASON = self.SEASONS[self.MONTH]
                if self.MONTH >= 12:
                    self.MONTH = 0
        else:
            self.TIME += 1

    def penetration_mutation(self,*args):
        for i in range(self.NUMBER_OF_LAYERS):
            self.LAYERS_PENETRATION[i] = self.LAYERS_PENETRATION[i] if np.random.rand() > self.CHANGING_PENETRATION_CHANCE \
                else (self.LAYERS_PENETRATION[i] + np.random.rand()) / 2

    def rain_operation(self):
        if not self.rain.active:
            self.rain.start_rain(self.RAIN_CHANCE + self.SEASON_CHANGING_RAIN_MUTATION[self.SEASON])
        if self.rain.active:
            self.input_water += self.rain.rain()
        if self.IS_WATERING:
            self.input_water += self.VALVE_CAPACITY

    def make_mutation(self):
        self.penetration_mutation()
        self.rain_operation()

    def start_penetration(self):
        # first layer
        if self.input_water > self.LAYERS_SIZE[0]*(1 - self.LAYERS_MOISTURE[0]):
            self.input_water -= self.LAYERS_SIZE[0]*(1 - self.LAYERS_MOISTURE[0])
            self.LAYERS_MOISTURE[0] = 1
        else:
            self.input_water = 0
            self.LAYERS_MOISTURE[0] =(self.LAYERS_SIZE[0]*self.LAYERS_MOISTURE[0] + self.input_water) / self.LAYERS_SIZE[0]
        # other layers
        for i in range(self.NUMBER_OF_LAYERS-1, 0, -1):
            passed_water = self.LAYERS_MOISTURE[i-1]*self.LAYERS_SIZE[i-1]*self.LAYERS_PENETRATION[i-1]
            if passed_water > self.LAYERS_SIZE[i]*(1-self.LAYERS_MOISTURE[i]):
                passed_water = self.LAYERS_SIZE[i]*(1-self.LAYERS_MOISTURE[i])
            self.LAYERS_MOISTURE[i] += passed_water/self.LAYERS_SIZE[i]
            self.LAYERS_MOISTURE[i-1] -= passed_water/self.LAYERS_SIZE[i-1]

    def decorator(func):
        def wrapper(self, *args):
            self.increase_time()
            return func(self, args)
        return wrapper

    @decorator
    def irrigate(self, *args):
        self.make_mutation()
        self.start_penetration()

s = Soil()
t = threading.Thread(target=Soil.irrigation_handler(s))
t.start()


# Miguel C. Matos
# 89124

from bayes_net import *
from semantic_network import *

class MySN(SemanticNetwork):
    def __init__(self, ldecl=None):
        if ldecl is None:
            ldecl = []
        super().__init__(ldecl)

    def query_dependents(self, entity):
        dep = [self.query_dependents(d.relation.entity1) for d in self.declarations if
               d.relation.entity2 == entity and (isinstance(d.relation, Depends) or isinstance(d.relation, Subtype))]
        deps = [d.relation.entity1 for d in self.declarations if
                d.relation.entity2 == entity and (isinstance(d.relation, Depends))]

        ret = list(set([item for sublist in dep for item in sublist] + deps))
        for r in ret:
            for d in self.query_local(e2=r, rel="subtype"):
                ret.append(d.relation.entity1)
                if d.relation.entity2 in ret:
                    ret.remove(d.relation.entity2)
        return list(set(ret))

    def query_causes(self, entity):
        dep = [self.query_causes(d.relation.entity2) for d in self.declarations if
               d.relation.entity1 == entity and (isinstance(d.relation, Depends) or isinstance(d.relation, Subtype))]

        deps = [d.relation.entity2 for d in self.declarations if
                d.relation.entity1 == entity and isinstance(d.relation, Depends)]

        return list(set([item for sublist in dep for item in sublist] + deps))

    def query_causes_sorted(self, entity):
        dep = self.query_causes(entity)
        ret = [(d.relation.entity1, d.relation.entity2) for d in self.declarations if
               isinstance(d.relation, Association) and d.relation.name == "debug_time" and d.relation.entity1 in dep]

        dic = dict()
        for r in ret:
            key, val = r
            dic.setdefault(key, []).append(val)

        lst = []
        for name in dic:
            lst.append((name, sum(dic[name]) / len(dic[name])))
        lst = sorted(lst, key=lambda x: x[1])
        return lst

class MyBN(BayesNet):

    def markov_blanket(self, var):
        par = self.dependencies.get(var)

        # Dependents of var
        deps = list(set([t[0] for f in par for t in f]))
        # var depends of
        lst = list(set([d for d in self.dependencies for i in self.dependencies[d] for j in i if j[0] == var]))
        # Dependents of what var depends of
        lst_deps = list(set([t[0] for l in lst for f in self.dependencies[l] for t in f if t[0] != var]))

        return deps + lst + lst_deps


import scrapy

class BakerySpider(scrapy.Spider):
  name = 'bakery'
#   start_urls = ['https://keewahsf.com/collections/in-store-pick-up-only',]

# scrapy crawl bakery -a category=in-store-pick-up-only
  def __init__(self, category=None, *args, **kwargs):
    super(BakerySpider, self).__init__(*args, **kwargs)
    self.start_urls = [f'https://keewahsf.com/collections/{category}',]

  def parse(self, response):
    for item in response.css('div.info'):
      yield {
        'name': item.css('span.title::text').get(),
        'price':  item.css('span.money::text').get()
      }

    next = response.css('span.next a::attr("href")').get()
    if next is not None:
      yield response.follow(next, self.parse)


import jsonpath
import requests
import json

# API URL
url = "https://regres.in/api/users?page2"

# send get request -74
response = requests.get(url)

# Parse response to Json format
json_response =  json.loads(response.text)
print(json_response)

# Fetch value using json-path
pages = jsonpath.jsonpath(json_response,'total_page') # this will produce a list of values
print(pages(0)) # the '0' inside the brackets represents the index value
assert pages[0] == 4 # number value we should expect, therefore no error if true - any other number entered will give error

# Fetch value using advance json-path
for i in range(0, 3):
    first_name = jsonpath.jsonpath(json_response, 'data[' + str(i) + '].first_name')
    print((first_name[0]))


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import gym
from matplotlib import pyplot as plt
import os
os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'
env = gym.make('Pendulum-v0').unwrapped
'''Pendulum环境状态特征是三个，杆子的sin(角度)、cos（角度）、角速度，（状态是无限多个，因为连续），动作值是力矩，限定在[-2,2]之间的任意的小数，所以是连续的（动作也是无限个）'''
state_number=env.observation_space.shape[0] #状态的维度或特征数量
action_number=env.action_space.shape[0]
max_action = env.action_space.high[0]
min_action = env.action_space.low[0]
torch.manual_seed(0)#如果你觉得定了随机种子不能表达代码的泛化能力，你可以把这两行注释掉
env.seed(0)
RENDER=False
EP_MAX = 1000
EP_LEN = 200
GAMMA = 0.9
A_LR = 0.0001
C_LR = 0.0003
BATCH = 128
A_UPDATE_STEPS = 10
C_UPDATE_STEPS = 10
METHOD = [
    dict(name='kl_pen', kl_target=0.01, lam=0.5),   # KL penalty
    dict(name='clip', epsilon=0.2),                 # Clipped surrogate objective, find this is better
][1]        # choose the method for optimization
Switch=0
'''由于PPO也是基于A-C框架，所以我把PPO的编写分为两部分，PPO的第一部分 Actor'''
'''PPO的第一步  编写A-C框架的网络，先编写actor部分的actor网络，actor的网络有新与老两个网络'''
class ActorNet(nn.Module):
    def __init__(self,inp,outp):
        super(ActorNet, self).__init__() #这样做可以保证 ActorNet 类正确继承了父类的属性和方法，并且可以添加自己特定的属性和方法。
        self.in_to_y1=nn.Linear(inp,100) # 输入自己可定
        self.in_to_y1.weight.data.normal_(0,0.1) # 网络权值初始化 .normal_(0,0.1) 则是对这些权重参数进行正态分布初始化。其中，(0, 0.1) 表示正态分布的均值为 0，标准差为 0.1。
        self.out=nn.Linear(100,outp) #输出自己定义，用于算均值
        self.out.weight.data.normal_(0,0.1)
        self.std_out = nn.Linear(100, outp) #输出自己定义，用于算标准差
        self.std_out.weight.data.normal_(0, 0.1)
    '''生成均值与标准差，PPO必须这样做，一定要生成分布（所以需要mean与std），不然后续学习策略里的公式写不了，后面更新actor的策略会用到正态分布函数,DDPG是可以不用生成概率分布的'''
    '''在前向传播函数 forward 中，首先将输入状态传递到输入层 in_to_y1，然后通过激活函数ReLU对输出进行非线性变换。接着，通过线性层 out 计算均值（mean），并使用 torch.tanh 函数进行范围映射，以确保均值在合适的范围内。同时，
   通过线性层 std_out 计算标准差（std），并使用 F.softplus 函数确保标准差的值域大于0，以保证正数。'''
    def forward(self,inputstate): #用于构建动作的概率分布
        inputstate=self.in_to_y1(inputstate)
        inputstate=F.relu(inputstate)
        mean=max_action*torch.tanh(self.out(inputstate))#输出概率分布的均值mean
        std=F.softplus(self.std_out(inputstate))#softplus激活函数的值域>0
        return mean,std
'''再编写critic部分的critic网络，PPO的critic部分与AC算法的critic部分是一样，PPO不一样的地方只在actor部分'''
class CriticNet(nn.Module):
    def __init__(self,input,output):
        super(CriticNet, self).__init__()
        self.in_to_y1=nn.Linear(input,100)
        self.in_to_y1.weight.data.normal_(0,0.1)
        self.out=nn.Linear(100,output)
        self.out.weight.data.normal_(0,0.1)
    def forward(self,inputstate):
        inputstate=self.in_to_y1(inputstate)
        inputstate=F.relu(inputstate)
        Q=self.out(inputstate)
        return Q
class Actor():
    def __init__(self):
        self.old_pi,self.new_pi=ActorNet(state_number,action_number),ActorNet(state_number,action_number)#这只是均值mean
        self.optimizer=torch.optim.Adam(self.new_pi.parameters(),lr=A_LR,eps=1e-5) #使用 torch.optim.Adam 创建了一个优化器（optimizer），传入 self.new_pi.parameters() 表示优化器将优化 new_pi 网络的参数
    '''第二步 编写根据状态选择动作的函数'''
    def choose_action(self,s):
        inputstate = torch.FloatTensor(s) #将状态 s 转换为 torch.FloatTensor 类
        mean,std=self.old_pi(inputstate) #该类中只有一个函数直接调用！！！！！！
        #logits = self.old_pi(inputstate)
        #dist = torch.distributions.Categorical(logits=logits)
        dist = torch.distributions.Normal(mean, std) #正太分布可以用来生成动作样本
        action=dist.sample()
        action=torch.clamp(action,min_action,max_action) #对动作进行限制，将其裁剪到一个指定的范围内，以确保动作的取值在合理范围内
        action_logprob=dist.log_prob(action) #将动作 action 和对数概率值 action_logprob 分别转换为NumPy数组，并使用 detach() 方法将其从计算图中分离，返回这两个值
        return action.detach().numpy(),action_logprob.detach().numpy()
    '''第四步  actor网络有两个策略（更新old策略）————————把new策略的参数赋给old策略'''
    def update_oldpi(self):
        self.old_pi.load_state_dict(self.new_pi.state_dict()) #通过调用 self.new_pi.state_dict() 获取新策略网络 new_pi 的状态字典（state dictionary）。状态字典包含了网络的所有参数及其对应的数值。
    '''第六步 编写actor网络的学习函数，采用PPO2，即OpenAI推出的clip形式公式'''
    def learn(self,bs,ba,adv,bap):
        bs = torch.FloatTensor(bs)
        ba = torch.FloatTensor(ba)
        adv = torch.FloatTensor(adv)
        bap = torch.FloatTensor(bap)
        for _ in range(A_UPDATE_STEPS):
            mean, std = self.new_pi(bs) #获取新策略网络 new_pi 的均值 mean 和标准差 std。
            dist_new=torch.distributions.Normal(mean, std)
            action_new_logprob=dist_new.log_prob(ba)
            ratio=torch.exp(action_new_logprob - bap.detach())# 这里log对应自然指数
            surr1 = ratio * adv
            surr2 = torch.clamp(ratio, 1 - METHOD['epsilon'], 1 + METHOD['epsilon']) * adv
            loss = -torch.min(surr1, surr2)
            loss=loss.mean()
            self.optimizer.zero_grad()
            loss.backward()
            nn.utils.clip_grad_norm_(self.new_pi.parameters(), 0.5)
            self.optimizer.step()
class Critic():
    def __init__(self):
        self.critic_v=CriticNet(state_number,1)#改网络输入状态，生成一个Q值
        self.optimizer = torch.optim.Adam(self.critic_v.parameters(), lr=C_LR,eps=1e-5)
        self.lossfunc = nn.MSELoss()
    '''第三步  编写评定动作价值的函数'''
    def get_v(self,s):
        inputstate = torch.FloatTensor(s)
        return self.critic_v(inputstate)
    '''第五步  计算优势——————advantage，后面发现第五步计算出来的adv可以与第七步合为一体，所以这里的代码注释了，但是，计算优势依然算是可以单独拎出来的一个步骤'''
    # def get_adv(self,bs,br):
    #     reality_v=torch.FloatTensor(br)
    #     v=self.get_v(bs)
    #     adv=(reality_v-v).detach()
    #     return adv
    '''第七步  编写actor-critic的critic部分的learn函数，td-error的计算代码（V现实减去V估计就是td-error）'''
    def learn(self,bs,br):
        bs = torch.FloatTensor(bs)
        reality_v = torch.FloatTensor(br)
        for _ in range(C_UPDATE_STEPS):
            v=self.get_v(bs)
            td_e = self.lossfunc(reality_v, v)
            self.optimizer.zero_grad()
            td_e.backward()
            nn.utils.clip_grad_norm_(self.critic_v.parameters(), 0.5)
            self.optimizer.step()
        return (reality_v-v).detach()
if Switch==0:
    print('PPO2训练中...')
    actor=Actor()
    critic=Critic()
    all_ep_r = []
    for episode in range(EP_MAX):
        observation = env.reset() #环境重置
        buffer_s, buffer_a, buffer_r,buffer_a_logp = [], [], [],[]
        reward_totle=0
        for timestep in range(EP_LEN):
            if RENDER:
                env.render()
            action,action_logprob=actor.choose_action(observation)
            observation_, reward, done, info = env.step(action)
            buffer_s.append(observation)
            buffer_a.append(action)
            buffer_r.append((reward+8)/8)    # normalize reward, find to be useful
            buffer_a_logp.append(action_logprob)
            observation=observation_
            reward_totle+=reward
            reward = (reward - reward.mean()) / (reward.std() + 1e-5)
         #PPO 更新
            if (timestep+1) % BATCH == 0 or timestep == EP_LEN-1: #上面先交互一段时间，再学习更新一段时间
                v_observation_ = critic.get_v(observation_)
                discounted_r = []
                for reward in buffer_r[::-1]:
                    v_observation_ = reward + GAMMA * v_observation_
                    discounted_r.append(v_observation_.detach().numpy())
                discounted_r.reverse()
                bs, ba, br,bap = np.vstack(buffer_s), np.vstack(buffer_a), np.array(discounted_r),np.vstack(buffer_a_logp)
                buffer_s, buffer_a, buffer_r,buffer_a_logp = [], [], [],[]
                advantage=critic.learn(bs,br)#critic部分更新
                actor.learn(bs,ba,advantage,bap)#actor部分更新
                actor.update_oldpi()  # pi-new的参数赋给pi-old
                # critic.learn(bs,br)
        if episode == 0:
            all_ep_r.append(reward_totle)
        else:
            all_ep_r.append(all_ep_r[-1] * 0.9 + reward_totle * 0.1)
        print("\rEp: {} |rewards: {}".format(episode, reward_totle), end="")
        #保存神经网络参数
        if episode % 50 == 0 and episode > 100:#保存神经网络参数
            save_data = {'net': actor.old_pi.state_dict(), 'opt': actor.optimizer.state_dict(), 'i': episode}
            torch.save(save_data, "E:\PPO2_model_actor.pth")
            save_data = {'net': critic.critic_v.state_dict(), 'opt': critic.optimizer.state_dict(), 'i': episode}
            torch.save(save_data, "E:\PPO2_model_critic.pth")
    env.close()
    plt.plot(np.arange(len(all_ep_r)), all_ep_r)
    plt.xlabel('Episode')
    plt.ylabel('Moving averaged episode reward')
    plt.show()
else:
    print('PPO2测试中...')
    aa=Actor()
    cc=Critic()
    checkpoint_aa = torch.load("E:\PPO2_model_actor.pth")
    aa.old_pi.load_state_dict(checkpoint_aa['net'])
    checkpoint_cc = torch.load("E:\PPO2_model_critic.pth")
    cc.critic_v.load_state_dict(checkpoint_cc['net'])
    for j in range(10):
        state = env.reset()
        total_rewards = 0
        for timestep in range(EP_LEN):
            env.render()
            action, action_logprob = aa.choose_action(state)
            new_state, reward, done, info = env.step(action)  # 执行动作
            total_rewards += reward
            state = new_state
        print("Score：", total_rewards)
    env.close()


import cv2
import cv
import sklearn
import numpy as np
import os
from Images import Images
from matplotlib import pyplot as plt
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn import metrics
import lasagne
import theano
import theano.tensor as T
theano.config.exception_verbosity='high'
theano.config.traceback.limit=100

path2 = '/home/karna/Karna_Work/Handwriting_Project/Handwriting-Segmentation/testdata2/uppercontour/'

uppercontourimg = Images(path2)
#testimg.applyCanny()
#testimg.invertImages()
#testimg.normalize()
#plt.imshow(testimg.getImage(2),cmap='gray')
#plt.show()
#testimg.createImageFeatures()
uppercontourno = uppercontourimg.cnt

path2 = '/home/karna/Karna_Work/Handwriting_Project/Handwriting-Segmentation/testdata2/lowercontour/'

imglowercont = Images(path2)
#testimg.applyCanny()
#imglowercont.invertImages()
#testimg.normalize()
#plt.imshow(imglowercont.getImage(2),cmap='gray')
#plt.show()
#testimg.createImageFeatures()
lowercontourno = imglowercont.cnt

batchsize = lowercontourno + uppercontourno
#batchsize = uppercontourno
#batchsize = lowercontourno

#CNN Configuration

var_in = T.tensor4('var_in')
var_t = T.imatrix('var_t')
#batchsize = T.scalar('batchsize')
network = lasagne.layers.InputLayer(shape=(batchsize,1,150,150),input_var=var_in)

network1 = lasagne.layers.Conv2DLayer(
        network, num_filters=30, filter_size=(10,10),pad='full',
        nonlinearity=lasagne.nonlinearities.sigmoid,
        W=lasagne.init.GlorotNormal(),b=lasagne.init.Constant(0.))

network = lasagne.layers.Pool2DLayer(network1, pool_size=(2, 2),mode='max')

network = lasagne.layers.Conv2DLayer(network, num_filters=40,pad='full', filter_size=(5,5),nonlinearity=lasagne.nonlinearities.sigmoid,W=lasagne.init.GlorotNormal(),b=lasagne.init.Constant(0.))

network = lasagne.layers.Pool2DLayer(network, pool_size=(2, 2),mode='max')

#network = lasagne.layers.Conv2DLayer(network, num_filters=batchsize*3,pad='same', filter_size=(3,3),nonlinearity=lasagne.nonlinearities.sigmoid,W=lasagne.init.GlorotNormal(),b=lasagne.init.Constant(0.))

#network = lasagne.layers.Pool2DLayer(network, pool_size=(2, 2),mode='max')

network = lasagne.layers.DenseLayer(network,num_units = 2,nonlinearity=lasagne.nonlinearities.softmax,W=lasagne.init.GlorotNormal(), b=lasagne.init.Constant(0.))
#weights = network2.W.get_value()

prediction = lasagne.layers.get_output(network)

loss = lasagne.objectives.squared_error(prediction, var_t).mean()
all_params = lasagne.layers.get_all_params(network, trainable=True)
updates=lasagne.updates.sgd(loss,all_params,0.1)

train = theano.function([var_in, var_t], loss, updates=updates )

#theano.config.optimizer_verbose = 1
#theano.config.compute_test_value = 'warn'
#theano.config.optimizer='fast_compile'

#Training Data preparation

varout=[]
varin=()
#output = np.eye(imglowercont.cnt,batchsize)
#output=output.astype(int)
size=150

for i in range(0,uppercontourno):
    timgi = uppercontourimg.getImage(i)/255.0
    sh = np.shape(timgi[:,:,0])
    dest = np.ones((size,size,1))
    dest[0:sh[0],0:sh[1],:]=timgi[:,:,0,None]
    lst = list(varin)
    lst.append(dest)
    varin=tuple(lst)
    varout.append(tuple([1.0,0.0]))

for i in range(0,lowercontourno):
    timgi = imglowercont.getImage(i)/255.0
    sh = np.shape(timgi[:,:,0])
    dest = np.ones((size,size,1))
    dest[0:sh[0],0:sh[1],:]=timgi[:,:,0,None]
    lst = list(varin)
    lst.append(dest)
    varin=tuple(lst)
    varout.append(tuple([0.0,1.0]))

varin=np.transpose(varin,(0,3,1,2))

#training

maxepoch=10
for epoch in range(maxepoch):
    loss = 0
    loss += train(varin, varout)
    print("Epoch %d: Loss %g" % (epoch + 1, loss))

#test data preparation

path1 = '/home/karna/Karna_Work/Handwriting_Project/Handwriting-Segmentation/data/'
img = Images(path1)
imgcan = img.getImage(0)
sh=np.shape(imgcan)
imgcan[imgcan>180]=255
sh=np.shape(imgcan)

a=imgcan[sh[0]/2+250:sh[0]/2 + 400,sh[1]/2+500:sh[1]/2 + 650]
plt.imshow(a,cmap='gray')
plt.show()

imgcutcan=imgcan[sh[0]/2+250:sh[0]/2 + 400,sh[1]/2+500:sh[1]/2 + 650]/255.0
#imgsh=imgcan[sh[0]/2+250:sh[0]/2 + 400,sh[1]/2+500:sh[1]/2 + 650]

#testing
varin = ()
varout=[]
for i in range(batchsize):
    timgi = imgcutcan[None,95:120,:,0]
    sh = np.shape(timgi[:,:,:])
    dest = np.ones((1,size,size))
    dest[:,0:sh[1],0:sh[2]]=timgi[None,:,:]
    lst = list(varin)
    lst.append(dest)
    varin=tuple(lst)

#Testing
test_prediction = lasagne.layers.get_output(network, deterministic=False)
#predict_fn = theano.function([var_in], T.argmax(test_prediction, axis=1))
predict_fn = theano.function([var_in], test_prediction)
print("Predicted class for first test input: %r" % predict_fn(varin))


import os
import glob

from torch.backends import cudnn

from parameters import get_parameters
from data_loader import Data_Loader
from model import get_dcgan
from utils import Logger

from gan import GanTrainer, BigGanTrainer

def main(config):
    cudnn.benchmark = True

    img_size = config.img_size
    batch_size = config.batch_size
    num_iters = config.iters

    classes = None
    if config.classes is not None:
        classes = config.classes.split(',')
    elif config.dataset == 'lsun':
        raise Exception('Provide class list. Available options: bedroom_train,bridge_train,church_outdoor_train')

    data_loader = Data_Loader(config.dataset, config.data_path, img_size, batch_size, classes=classes, shuffle=True)
    logger_name = '{}_{}'.format(config.dataset, config.model)

    if config.model=='dcgan':
        from torch import nn, optim
        import torch

        sn = config.sn
        sa = config.sa
        device = torch.device(config.device)

        discriminator, generator = get_dcgan(img_size, 3, sn=sn, device = device)
        d_optimizer = optim.Adam(discriminator.parameters(), lr=0.0002, betas = (0.5, 0.999))
        g_optimizer = optim.Adam(generator.parameters(), lr=0.0002, betas = (0.5, 0.999))

        if sn:
            logger_name += '_sn'

        if sa:
            logger_name += '_sa'

        logger = Logger(logger_name + '_{}_bs_{}'.format(img_size, batch_size))
        loss = nn.BCELoss()

        trainer = GanTrainer(discriminator, generator, d_optimizer, g_optimizer, loss, loss, logger, log_iter=config.log_iter, device=device)
    elif config.model == 'biggan':
        gpus = [int(x) for x in config.gpus.split(',')]
        num_classes = data_loader.num_classes
        logger = Logger(logger_name + '_{}_bs_{}'.format(img_size, batch_size))

        trainer = BigGanTrainer(num_classes, logger, log_iter=config.log_iter, gpus=gpus)

    trainer.train(data_loader.loader(), num_iters)

if __name__ == '__main__':
    config = get_parameters()
    main(config)


# Tests containing Python 3.5+ syntax, extracted from test_client_server.py.
# To avoid test discovery, this module's name must not start with test_.

import asyncio

from ..client import *
from ..server import *
from ..test_client_server import handler

class ClientServerContextManager:

    def setUp(self):
        self.loop = asyncio.new_event_loop()
        asyncio.set_event_loop(self.loop)

    def tearDown(self):
        self.loop.close()

    def test_basic(self):
        server = serve(handler, 'localhost', 8642)
        self.server = self.loop.run_until_complete(server)

        async def basic():
            async with connect('ws://localhost:8642/') as client:
                await client.send("Hello!")
                reply = await client.recv()
                self.assertEqual(reply, "Hello!")

        self.loop.run_until_complete(basic())

        self.server.close()
        self.loop.run_until_complete(self.server.wait_closed())


import pandas as pd

from molgenis.capice_resources.core import ColumnEnums, AlleleFrequencyEnums
from molgenis.capice_resources.utilities import split_consequences
from molgenis.capice_resources.balance_dataset import BalanceDatasetEnums
from molgenis.capice_resources.balance_dataset.verbosity_printer import VerbosityPrinter

class Balancer:
    RANDOM_STATE = 5

    def __init__(self, verbose: bool):
        self.printer = VerbosityPrinter(verbose)
        self.bins = [pd.IntervalIndex]
        self.columns: list[str] = []
        self.drop_benign = pd.Index([])
        self.drop_pathogenic = pd.Index([])

    @staticmethod
    def _mark_and_impute(dataset: pd.DataFrame) -> None:
        """
        Function to impute samples where the GNOMAD allele frequency is 0.
        Also marks which samples which are imputed.

        Args:
            dataset:
                The pandas.DataFrame containing the gnomAD allele frequency column to be imputed
                (if required).
                Performed inplace.

        """
        dataset[ColumnEnums.IMPUTED.value] = 0
        dataset.loc[dataset[ColumnEnums.GNOMAD_AF.value].isnull(), ColumnEnums.IMPUTED.value] = 1
        dataset[ColumnEnums.GNOMAD_AF.value].fillna(0, inplace=True)

    @staticmethod
    def _reset_impute(dataset: pd.DataFrame) -> None:
        """
        Function to reset imputed samples where the allele frequency has been imputed.
        Not to confuse native "0" allele frequency from imputed ones.

        Args:
            dataset:
                The pandas.DataFrame containing both the gnomAD allele frequency column and the
                "is_imputed" column. Resets allele frequency to nan where it has been imputed.
                Performed inplace.

        """
        dataset.loc[dataset[ColumnEnums.IMPUTED.value] == 1, ColumnEnums.IMPUTED.value] = None
        dataset.drop(columns=[ColumnEnums.IMPUTED.value], inplace=True)

    def _set_bins(self, gnomad_af: pd.Series) -> None:
        """
        Setter for the init variable self.bins, according to both the bins defined in the
        GlobalEnums and the allele frequency ranges present in the gnomAD_AF column.

        Args:
            gnomad_af:
                pandas.Series instance of the dataset "gnomAD_AF" column.

        """
        self.bins = pd.cut(
            gnomad_af,
            bins=AlleleFrequencyEnums.AF_BINS.value,
            right=False,
            include_lowest=True
        ).dropna().unique()
        self.printer.print(f'Bins set: {self.bins}')

    def _set_columns(self, columns: pd.DataFrame.columns) -> None:
        """
        Function to set the columns available for all methods within the Balancer class.

        Args:
            columns:
                The pandas.DataFrame.columns instance of "dataset" after imputing has been
                performed.

        """
        self.columns = list(columns)

    def balance(self, dataset: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:
        """
        Primary function of the Balancer class. Loops through each (split) consequence,
        samples pathogenic and benign separately, then calls the "_process_consequence" function.
        After that is done, then filters out the drop Indexes that have been generated by
        "_process_bins" to prevent a multi-consequence sample from being sampled multiple times.

        Args:
            dataset:
                The pandas.Dataframe over which should be balanced.
                Requires the GnomAD_AF column and the Consequence column.
        Returns:
            tuple:
                Tuple containing [0] the balanced dataset over each Consequence over each AF bin
                as per defined in GlobalEnums and
                [1] the remainder dataset of all samples that have not been sampled.

        """
        self._mark_and_impute(dataset)
        self._set_columns(dataset.columns)
        self._set_bins(dataset[ColumnEnums.GNOMAD_AF.value])
        pathogenic = dataset.loc[dataset[dataset[ColumnEnums.BINARIZED_LABEL.value] == 1].index, :]
        benign = dataset.loc[dataset[dataset[ColumnEnums.BINARIZED_LABEL.value] == 0].index, :]
        return_dataset = pd.DataFrame(columns=self.columns)
        consequences = split_consequences(dataset[ColumnEnums.CONSEQUENCE.value])
        for consequence in consequences:
            self.printer.print(f'Processing: {consequence}')
            selected_pathogenic = pathogenic[
                pathogenic[ColumnEnums.CONSEQUENCE.value].str.contains(consequence)
            ]
            self.printer.print(
                f'Total amount of pathogenic samples (for consequence): '
                f'{selected_pathogenic.shape[0]}'
            )
            selected_benign = benign[
                benign[ColumnEnums.CONSEQUENCE.value].str.contains(consequence)
            ]
            self.printer.print(
                f'Total amount of benign samples (for consequence): '
                f'{selected_benign.shape[0]}'
            )
            processed_consequence = self._process_consequence(
                selected_pathogenic,
                selected_benign
            )
            counts = processed_consequence.value_counts(  # One-liner in F-string not possible.
                subset=[  # type: ignore
                    ColumnEnums.BINARIZED_LABEL.value,
                    ColumnEnums.GNOMAD_AF.value
                ]
            )
            self.printer.print(
                f'Sampled total for consequence: {consequence}\n'
                f'{counts}'
            )
            processed_consequence[BalanceDatasetEnums.BALANCED_ON.value] = consequence
            return_dataset = pd.concat(
                [
                    return_dataset,
                    processed_consequence
                ], axis=0
            )
            return_dataset.sort_index(inplace=True)
            benign.drop(self.drop_benign, inplace=True)
            self.drop_benign = pd.Index([])
            pathogenic.drop(index=self.drop_pathogenic, inplace=True)
            self.drop_pathogenic = pd.Index([])
        self._reset_impute(return_dataset)
        remainder = pd.concat([benign, pathogenic], ignore_index=True, axis=0)
        self._reset_impute(remainder)
        self.printer.print(f'Balanced set size: {return_dataset.shape[0]}')
        self.printer.print(f'Remainder set size: {remainder.shape[0]}')
        return return_dataset, remainder

    def _sample_variants(self, dataset: pd.DataFrame, n_required: int) -> pd.DataFrame:
        """
        Small function to randomly sample an x amount of samples according to n_required if
        dataset exceeds the n_required.

        Args:
            dataset:
                The pandas.DataFrame that should be checked if the amount of samples is equal to or
                lower than n_required.
            n_required:
                The integer of the amount of samples "dataset" should have at max.
                Does not sample if the amount of samples in "dataset" is lower or equal to
                n_required.

        Returns:
            dataframe:
                Returns a pandas.DataFrame that is the same as input "dataset",
                but randomly sampled to fit a maximum of n_required.

        """
        if dataset.shape[0] > n_required:
            dataset = dataset.sample(n=n_required, random_state=self.RANDOM_STATE)
        return dataset

    def _process_consequence(
            self,
            pathogenic_dataset: pd.DataFrame,
            benign_dataset: pd.DataFrame
    ) -> pd.DataFrame:
        """
        Function to process one specific consequence, which it assumes is already subsetted.
        Does not perform checks if a sample has already been sampled before.
        Does not set the indexes of samples that have to be removed from the initial pathogenic and
        benign subset (without Consequence subset), since it can not know what samples are going
        to be sampled in the allele frequency balancing.

        Args:
            pathogenic_dataset:
                The pandas.DataFrame subset of pathogenic variants, subset further on "Consequence".
            benign_dataset:
                The pandas.DataFrame subset of benign variants, subset further on "Consequence".

        Returns:
            dataframe:
                Returns a pandas.DataFrame that contains variants that have been balanced on the
                allele frequency bins defined in GlobalEnums.
        """
        pathogenic_dataset = self._sample_variants(pathogenic_dataset, benign_dataset.shape[0])
        benign_dataset = self._sample_variants(benign_dataset, pathogenic_dataset.shape[0])
        processed_bins = pd.DataFrame(columns=self.columns)
        for af_bin in self.bins:
            processed_bins = pd.concat(
                [processed_bins, self._process_bins(pathogenic_dataset, benign_dataset, af_bin)],
                axis=0
            )
        return processed_bins

    def _process_bins(self, pathogenic_dataset, benign_dataset, af_bin):
        """
        Function to process an allele frequency bin.
        Does set the indexes of benign and pathogenic sampled variants.
        Calls the "get_variants_within_range", then equalizes between benign and pathogenic by
        randomly sampling.

        Args:
            pathogenic_dataset:
                The subset of pathogenic variants, subset on Consequence, as supplied to
                "_process_consequence" but equalized to the benign subset supplied to
                "_process_consequence".
            benign_dataset:
                The subset of benign variants, subset on Consequence, as supplied to
                "_process_consequence" but equalized to the pathogenic subset supplied to
                "_process_consequence".
            af_bin:
                The pandas.IntervalIndex of the AF bin of interest.

        Returns:
            dataframe:
                Concatenated pandas.DataFrame between pathogenic and benign samples,
                sampled by "af_bin" and then equalized between the amount of benign and
                pathogenic samples.
        """
        selected_pathogenic = self._get_variants_within_range(pathogenic_dataset, af_bin)
        selected_benign = self._get_variants_within_range(benign_dataset, af_bin)
        return_benign = self._sample_variants(selected_benign, selected_pathogenic.shape[0])
        return_pathogenic = self._sample_variants(selected_pathogenic, selected_benign.shape[0])
        self.drop_benign = self.drop_benign.union(return_benign.index).astype(int)
        self.drop_pathogenic = self.drop_pathogenic.union(return_pathogenic.index).astype(int)
        return pd.concat([return_benign, return_pathogenic], axis=0)

    @staticmethod
    def _get_variants_within_range(dataset, af_bin) -> pd.DataFrame:
        """
        Small function to get samples that are within a pandas.IntervalIndex range.

        Args:
            dataset:
                The pandas.DataFrame containing the gnomAD_AF column to get the variants
                that are within the AF bin range.
            af_bin:
                The pandas.IntervalIndex containing the AF bin range.

        Returns:
            dataframe:
                The input "dataset" containing only samples that
                are within the AF range of "af_bin".
        """
        return dataset[dataset[ColumnEnums.GNOMAD_AF.value].apply(lambda value: value in af_bin)]


from django.conf.urls import url
import student
import managementview
import teacher

urlpatterns = [

    url(r'^studentlist$', student.studentlist, name='studentlist'),
    url(r'^ajax/studentdata$', student.data, name='getstudentdata'),
    url(r'^ajax/studentselectdata$', student.selectdata, name='getstudentselectdata'),
    url(r'^ajax/teacherselectdata$', teacher.selectdata, name='getteacherselectdata'),
    url(r'^teacherlist', teacher.teacherlist, name='teacherlist'),
    url(r'^ajax/teacherdata$', teacher.data, name='getteacherdata'),
    url(r'^classlist', managementview.classlist, name='classlist'),
    url(r'^ajax/classdata', managementview.classdata, name='getclassdata'),
    url(r'^majorlist', managementview.majorlist, name='majorlist'),
    url(r'^ajax/majordata', managementview.majordata, name='getmajordata'),
    url(r'^departmentlist', managementview.departmentlist, name='departmentlist'),
    url(r'^ajax/departmentdata', managementview.departmentdata, name='getdepartmentdata'),
    url(r'^administrationlist', managementview.administrationlist, name='administrationlist'),
    url(r'^ajax/administrationdata', managementview.administrationdata, name='getadministrationdata'),
    url(r'^classroomlist', managementview.classroomlist, name='classroomlist'),
    url(r'^ajax/classroomdata', managementview.classroomdata, name='getclassroomdata'),
    url(r'^schoolterm$', managementview.schoolterm, name='schoolterm'),
    url(r'^classtime$', managementview.classtime, name='classtime'),

]


from os import F_OK, access, sep
try:
    from subprocess import PIPE, Popen
except ImportError:
    PIPE, Popen = None, None

# relax module imports.
from lib.compat import SYSTEM
import lib.structure.internal.object
from status import Status; status = Status()

version = "repository commit"
repo_head = None
repo_type = None
repo_url = None

def repo_information():
    """Determine the subversion revision number and URL from svn or git-svn copies of the repository."""

    # The global variables
    global repo_head
    global repo_type
    global repo_url

    # The variables are already set, so do nothing.
    if repo_head != None or repo_type != None or repo_url != None:
        return

    # Python 2.3 and earlier.
    if Popen == None:
        return

    # Command separator.
    symbol = ";"
    if SYSTEM == 'Windows' or SYSTEM == 'Microsoft':
        symbol = "&&"

    # The command to use.
    cmd = None
    if access(status.install_path+sep+'.git'+sep+'svn'+sep+'refs', F_OK):
        cmd = 'cd %s %s git svn info' % (status.install_path, symbol)
        repo_type = 'git-svn'
    elif access(status.install_path+sep+'.git', F_OK):
        cmd = 'cd %s %s git rev-parse HEAD %s git remote -v' % (status.install_path, symbol, symbol)
        repo_type = 'git'
    elif access(status.install_path+sep+'.svn', F_OK):
        cmd = 'svn info %s' % status.install_path
        repo_type = 'svn'

    # Not a repository copy, so do nothing.
    if not cmd:
        return

    # Open the pipe and run the command.
    pipe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False)

    # Loop over the output lines.
    lines = pipe.stdout.readlines()
    for i in range(len(lines)):
        # Decode Python 3 byte arrays.
        if hasattr(lines[i], 'decode'):
            lines[i] = lines[i].decode()

        # Git info.
        if repo_type == 'git':
            # Git hash.
            if i == 0:
                repo_head = lines[i].strip()
                continue

            # Remote URL.
            if repo_url:
                repo_url += '\n'
            else:
                repo_url = ''
            remote_info = lines[i].split()
            repo_url += "%s %s" % (remote_info[1], remote_info[2])

        # SVN and git-svn.
        else:
            # Split up the line.
            row = lines[i].split()

            # Store revision as the global variable.
            if len(row) and row[0] == 'Revision:':
                repo_head = str(row[1])

            # Store URL as the global variable.
            if len(row) and row[0] == 'URL:':
                repo_url = str(row[1])

def version_full():
    """Return the full relax version, including all SVN info for repository versions.

    @return:    The relax version string.
    @rtype:     str
    """

    # The relax version.
    ver = version

    # Repository version.
    if ver == 'repository commit':
        # The global variables
        global repo_head
        global repo_type
        global repo_url

        # Change the version string.
        if repo_head != None:
            if repo_type == 'git':
                ver = version + " " + repo_head
            else:
                ver = version + " r" + repo_head
        if repo_url != None:
            ver += " " + repo_url.replace('\n', '; ')

    # Return the version.
    return ver

# Fetch the repository information, if present.
repo_information()

# Set the version in the relax internal structural object.
lib.structure.internal.object.RELAX_VERSION = version_full()


import pygame
from constants import BALL_RADIUS, SCREEN_HEIGHT, SCREEN_WIDTH, FPS
from support import Support
from math import sin, pi, cos
import numpy as np
import matplotlib.pyplot as plt

def pendulum_func(parameters):
    return np.array([parameters[1], -gravity*sin(parameters[0])/length])

def runge_kutta_step(parameters, step):
    k1 = pendulum_func(parameters)
    intermediate = parameters + step*k1
    k2 = pendulum_func(intermediate)
    return parameters + step*(k1 + k2)/2

length = 1
gravity = 9.8
theta0 = -pi/3
params = np.array([theta0, 0.0])
ceiling = 150
path_radius = 200

pygame.init()
window = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
pygame.display.set_caption("Pêndulo Simples")
clock = pygame.time.Clock()
font = pygame.font.SysFont('Arial', 20, True)
support = Support((SCREEN_WIDTH/2, SCREEN_HEIGHT/2 - path_radius),
                  path_radius, path_radius - BALL_RADIUS, 20, params[0])
running = True

time_history = []
angle_history = []
ideal_history = []
iterations = 0

while running:
    clock.tick(FPS)

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

    window.fill((200, 255, 255))

    params = runge_kutta_step(params, 1/FPS)
    x, y = support.ball_position(params[0])
    support.update_line(params[0])
    pygame.draw.circle(window, (0, 0, 0), (int(x), int(y)), BALL_RADIUS, BALL_RADIUS - 1)
    support.draw(window)
    pygame.display.update()

    time_history.append(iterations/FPS)
    ideal_history.append(theta0*cos(np.sqrt(gravity/length)*iterations/FPS))
    angle_history.append(params[0])
    iterations = iterations + 1

plt.figure()
plt.xlabel('Tempo (s)')
plt.ylabel('Ângulo (rad)')
plt.plot(time_history, angle_history, time_history, ideal_history)
plt.title('Histórico do ângulo')
plt.legend(['real', 'aprox'])
plt.savefig('angle_history.jpg')

pygame.quit()


from config import GOODREADS_DATABASE_NAME
from wsgi.exceptions import NotFoundException
from wsgi.goodreads_api import GoodreadsClient
from wsgi.models import Resume, ReadingList, ResumeSettings

from wsgi import create_app
app = create_app()

shelves_to_pull = {'read-to-share', 'read-to-share-work', 'to-read-work', 'currently-reading-work'}
resume_mapping = {
    'read-to-share': 'finished_reading_general',
    'read-to-share-work': 'finished_reading',
    'to-read-work': 'to_read',
    'currently-reading-work': 'currently_reading'
}

""" goodreads sync.

Handles sync of goodreads book list data from goodreads to a user resume object specified in the
database
"""

def sync_bookshelves_from_goodreads(resume):
    service_link = None
    for sl in resume.service_links_list:
        if sl.name == GOODREADS_DATABASE_NAME:
            service_link = sl
            break

    gr_client = GoodreadsClient(service_link.key,
                                service_link.secret,
                                service_link.oauth_key,
                                service_link.oauth_secret,
                                service_link.user_id)

    shelves = set(gr_client.list_shelves_by_name())
    if len(shelves_to_pull) != len(shelves_to_pull & shelves):
        raise NotFoundException('Error: reading list mismatch ', shelves_to_pull & shelves)

    reading_lists = ReadingList()
    for shelf in shelves_to_pull:
        book_list = [book for book in gr_client.get_books_for_shelf(shelf)]
        setattr(reading_lists, resume_mapping[shelf], book_list)

    resume.reading_list = reading_lists
    resume.save()

if __name__ == '__main__':
    try:
        resume_setting = ResumeSettings.objects.all()[0]
    except IndexError:
        raise NotFoundException('Error: no ResumeSettings model object describing goodreads '
                                'to resume linkage exists.')
    sync_bookshelves_from_goodreads(resume_setting.resume_to_sync)