import os
import sys
import json
import datetime
import numpy as np
import skimage.draw

# Root directory of the project
ROOT_DIR = os.path.abspath("./")

# Import Mask RCNN
sys.path.append(ROOT_DIR)  # To find local version of the library
from mrcnn.config import Config
from mrcnn import model as modellib, utils
from mrcnn import visualize_ext as visualize

import matplotlib.pyplot as plt

# Path to trained weights file
COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")

# Directory to save logs and model checkpoints, if not provided
# through the command line argument --logs
DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")

# Keep the label as 'Motorcyclist' and 'Helmet'
# Set to False if needs to be 'Motorcyclist_with_Helmet' or 'without_Helmet' instead
DEFAULT_LABEL_HELMET_SEPARATE = True
DEBUG = False
NBR_OF_EPOCHS = 100

############################################################
#  Configurations
############################################################

class HelmetConfig(Config):
    """Configuration for training on the toy  dataset.
    Derives from the base Config class and overrides some values.
    """
    # Give the configuration a recognizable name
    NAME = "helmet"

    # We use a GPU with 12GB memory, which can fit two images.
    # Adjust down if you use a smaller GPU.
    IMAGES_PER_GPU = 2

    # Number of classes (including background)
    NUM_CLASSES = 1+2 if DEFAULT_LABEL_HELMET_SEPARATE else 1+3

    # Number of training steps per epoch
    STEPS_PER_EPOCH = 100

    # Skip detections with < 90% confidence
    DETECTION_MIN_CONFIDENCE = 0.9

############################################################
#  Dataset
############################################################

class HelmetDataset(utils.Dataset):

    def load_helmet(self, dataset_dir, subset):
        """Load a subset of the Helmet dataset.
        dataset_dir: Root directory of the dataset.
        subset: Subset to load: train or val
        """

        # Train or validation dataset?
        assert subset in ["train", "val"]
        dataset_dir = os.path.join(dataset_dir, subset)

        annotations = json.load(open(os.path.join(dataset_dir, "annotations-json.json")))
        annotations = list(annotations.values())  # don't need the dict keys

        # For collecting statistics
        number_of_images = 0

        # Preset class_names and instance_count ** To-be-checked
        if DEFAULT_LABEL_HELMET_SEPARATE:
            class_names = {'Motorcyclist': 1,
                           'Helmet': 2}
            instance_count = {'Motorcyclist': 0,
                           'Helmet': 0}
        else :
            class_name = {'Motorcyclist_without_helmet': 1,
                          'Helmet': 2,
                          'Motorcyclist_with_helmet': 3}
            instance_count = {'Motorcyclist_without_helmet': 0,
                          'Helmet': 0,
                          'Motorcyclist_with_helmet': 0}

        def isSegmentConnector(r) :
            """

            Parameters
            ----------
            r : region type with format :
                {
                	"shape_attributes": {
                		"name": "polygon",
                		"all_points_x": [
                			229,231,
                		],
                		"all_points_y": [
                			272,268,
                		]
                	},
                	"region_attributes": {
                		"name": "Motorcyclist",
                		"segment_connector": " "
                	}
                    "ignore" : False
                    "with_helmet" : False
                }

                DESCRIPTION : checks if the given region is Connected Segment or not.

            Returns
            -------
            True if it the given region is a valid connected segment
            False if it is a stand-alone segment

            """
            if ('segment_connector' in r['region_attributes']  and
            r['region_attributes']['segment_connector'].strip()) :
                return True
            else:
                return False

        # The VIA tool saves images in the JSON even if they don't have any
        # annotations. Skip unannotated images.
        annotations = [a for a in annotations if a['regions']]

        nbr_of_ignored = 0
        if DEBUG :
            temp_r = [] # for debugging purpose

        # Add images
        for a in annotations:
            # Get the x, y coordinaets of points of the polygons that make up
            # the outline of each object instance. These are stores in the
            # shape_attributes (see json format above)
            # The if condition is needed to support VIA versions 1.x and 2.x.
            if type(a['regions']) is dict:
                regions = a['regions'].values()
            else :
                regions = a['regions']

            # Connect the regions which belong to the same instance
            reg_len = len(regions)
            for i in range(reg_len):
                regions[i]['ignore'] = False
                regions[i]['with_helmet'] = False
                regions[i]['helmet_annotation'] = "NA"
                regions[i]['shape_attributes'] = [regions[i]['shape_attributes']]

            # Merge connected regions by checking SegmentConnector region attribute
            # Identify Motorcyclist if it has connected Helment annotation to flag accordingly
            for i, r in enumerate(regions):

                assert r['region_attributes']['name'] in ["Motorcyclist", "Helmet"]

                if not isSegmentConnector(r):
                    # check for 'Helmet' annotation found with no SegmentConnector.
                    # this should be corrected in annotation.
                    if r['region_attributes']['name'] == 'Helmet' :
                        print('Helmet annotation found without SegmentConnector : ',
                              'filename : {} annotation# : {}'.format(a['filename'], i+1))
                    continue
                # Check if there are linked instances and combine the regions accordingly
                # Check if the Helmet instance belongs to a motorcyclist to mark 'with_helmet' flag
                for j, rg in zip(range(i+1,reg_len), regions[i+1:]):

                    if (not isSegmentConnector(rg) or
                        rg['ignore'] or
                        (rg['with_helmet'] and rg['region_attributes']['name'] == 'Helmet') or
                        (regions[i]['with_helmet'] and r['region_attributes']['name'] == 'Helmet')):
                        continue

                    if (r['region_attributes']['segment_connector'].strip() ==
                        rg['region_attributes']['segment_connector'].strip()) :

                        if (r['region_attributes']['name'] ==
                            rg['region_attributes']['name']) :
                            regions[i]['shape_attributes'].extend(rg['shape_attributes'])
                            regions[j]['ignore'] = True

                        elif (r['region_attributes']['name'] == 'Motorcyclist' and
                            rg['region_attributes']['name'] == 'Helmet') :
                            regions[i]['with_helmet'] = True
                            regions[j]['with_helmet'] = True
                            regions[i]['helmet_annotation'] = str(j)

                        elif (r['region_attributes']['name'] == 'Helmet' and
                            rg['region_attributes']['name'] == 'Motorcyclist') :
                            regions[i]['with_helmet'] = True
                            regions[j]['with_helmet'] = True
                            regions[j]['helmet_annotation'] = str(i)

            polygons = []

            for i, r in enumerate(regions) :

                if r['ignore']:
                    nbr_of_ignored += 1
                    continue

                class_name = r['region_attributes']['name']

                # Modify class_name to mark with_helmet for combined helmet detection
                if not DEFAULT_LABEL_HELMET_SEPARATE :
                    if r['with_helmet'] and class_name == 'Motorcyclist':
                        class_name = class_name + "_with_helmet"
                    elif class_name == 'Motorcyclist':
                        class_name = class_name + "_without_helmet"

                    if class_name == 'Helmet' and not r['with_helmet'] :
                        print('Helmet annotation found without matching SegmentConnector : ',
                              'filename : {} annotation# : {}'.format(a['filename'], i+1))

                if class_name not in class_names.keys() :
                    if len(class_names.keys()) > 0 :
                        class_names[class_name] = max(class_names.values()) + 1
                    else :
                        class_names[class_name] = 1

                    instance_count[class_name] = 0

                instance_count[class_name] += 1

                polygon = {
                    "polygon": r['shape_attributes'],
                    "catId" : class_names[class_name]
                    }
                polygons.append(polygon)

                # For debugging purpose
                if DEBUG :
                    temp_r.append( {
                        "file_name" : a['filename'],
                        "name" : class_name,
                        'annotation_nbr' : str(i),
                        "with_helmet" : r['with_helmet'],
                        "helmet_annotation_nbr" : r['helmet_annotation']
                        })

            image_path = os.path.join(dataset_dir, a['filename'])
            image = skimage.io.imread(image_path)
            height, width = image.shape[:2]

            self.add_image(
                "helmet",
                image_id=a['filename'],  # use file name as a unique image id
                path=image_path,
                width=width, height=height,
                polygons=polygons)
            number_of_images += 1

        for class_name in list(class_names.keys()):
            self.add_class("helmet", class_names[class_name], class_name)

        # Statistics on datasets
        print ('\n{:*^50}'.format('Dataset statistics for ' + subset))
        print ('{:<20} {}'.format('Number of Images', number_of_images))
        print ('{:<20} {}'.format('class info', class_names))
        print ('{:<20} {}'.format('Nbr of Instances', instance_count))
        print ('{:<20} {}\n'.format('Nbr of ignored Instances', nbr_of_ignored))

        if DEBUG :
            import pandas as pd
            df = pd.DataFrame(data=temp_r)
            df.to_excel("DEBUG_temp.xlsx", index=False)

    def load_mask(self, image_id):
        """Generate instance masks for an image.
       Returns:
        masks: A bool array of shape [height, width, instance count] with
            one mask per instance.
        class_ids: a 1D array of class IDs of the instance masks.
        """
        # If not a helmet dataset image, delegate to parent class.
        image_info = self.image_info[image_id]
        if image_info["source"] != "helmet":
            return super(self.__class__, self).load_mask(image_id)

        # Convert polygons to a bitmap mask of shape
        # [height, width, instance_count]
        info = self.image_info[image_id]

        mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
                        dtype=np.uint8)

        class_ids = []
        for i, p in enumerate(info["polygons"]):
            # Get indexes of pixels inside the polygon and set them to 1
            for each in p['polygon'] :
                rr, cc = skimage.draw.polygon(each['all_points_y'], each['all_points_x'])
                mask[rr, cc, i] = 1
            class_ids.append(p['catId'])

        # Return mask, and array of class IDs of each instance. Since we have
        class_ids = np.array(class_ids, dtype=np.int32)
        return mask.astype(np.bool), class_ids

    def image_reference(self, image_id):
        """Return the path of the image."""
        info = self.image_info[image_id]
        if info["source"] == "helmet":
            return info["path"]
        else:
            super(self.__class__, self).image_reference(image_id)

def train(model, base_dataset_dir):
    """Train the model."""
    # Training dataset.
    dataset_train = HelmetDataset()
    dataset_train.load_helmet(base_dataset_dir, "train")
    dataset_train.prepare()

    # Validation dataset
    dataset_val = HelmetDataset()
    dataset_val.load_helmet(base_dataset_dir, "val")
    dataset_val.prepare()

    # *** This training schedule is an example. Update to your needs ***
    # Since we're using a very small dataset, and starting from
    # COCO trained weights, we don't need to train too long. Also,
    # no need to train all layers, just the heads should do it.
    print("Training network heads")
    model.train(dataset_train, dataset_val,
                learning_rate=config.LEARNING_RATE,
                epochs=NBR_OF_EPOCHS,
                layers='heads')

def color_splash(image, mask):
    """Apply color splash effect.
    image: RGB image [height, width, 3]
    mask: instance segmentation mask [height, width, instance count]

    Returns result image.
    """
    # Make a grayscale copy of the image. The grayscale copy still
    # has 3 RGB channels, though.
    gray = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
    # Copy color pixels from the original color image where mask is set
    if mask.shape[-1] > 0:
        # We're treating all instances as one, so collapse the mask into one layer
        mask = (np.sum(mask, -1, keepdims=True) >= 1)
        splash = np.where(mask, image, gray).astype(np.uint8)
    else:
        splash = gray.astype(np.uint8)
    return splash

def detect_and_color_splash(model, image_path=None, video_path=None):
    assert image_path or video_path

    if DEFAULT_LABEL_HELMET_SEPARATE :
        class_names = ['BG', 'Motorcyclist', 'Helmet']
    else :
        class_names = ['BG', 'Motorcyclist_without_Helmet',
                       'Helmet', 'Motorcyclist_with_Helmet']

    # Image or video?
    if image_path:
        # Run model detection and generate the color splash effect
        # print("Running on {}".format(image_path))
        # Read image
        image = skimage.io.imread(image_path)
        # Detect objects
        r = model.detect([image], verbose=1)[0]
        # Color splash
        splash = color_splash(image, r['masks'])
        ### visualize the results
        #print ('rois, class_ids, scores ',
        #       r['rois'], r['class_ids'], r['scores'])
        print (f"{'rois':<15} - {r['rois']} \n{'class_ids':<15} - {r['class_ids']}")
        #print (f"{'scores':<15} - {r['scores']} \n{'keypoints':<15} - {r['keypoints']}")

        _, ax = plt.subplots(1, 1, figsize=(20, 20))
        visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
                            class_names,
                            r['scores'], ax=ax)
        ####
        # Save output
        file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
        skimage.io.imsave(file_name, splash)
        #ax.savefig(file_name, bbox_inches='tight')
    elif video_path:
        import cv2
        # Video capture
        vcapture = cv2.VideoCapture(video_path)
        width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
        fps = vcapture.get(cv2.CAP_PROP_FPS)

        # Define codec and create video writer
        file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format(datetime.datetime.now())
        vwriter = cv2.VideoWriter(file_name,
                                  cv2.VideoWriter_fourcc(*'MJPG'),
                                  fps, (width, height))

        count = 0
        success = True
        while success:
            print("frame: ", count)
            # Read next image
            success, image = vcapture.read()
            if success:
                # OpenCV returns images as BGR, convert to RGB
                image = image[..., ::-1]
                # Detect objects
                r = model.detect([image], verbose=0)[0]
                # Color splash
                splash = color_splash(image, r['masks'])
                # RGB -> BGR to save image to video
                splash = splash[..., ::-1]
                # Add image to video writer
                vwriter.write(splash)
                count += 1
        vwriter.release()
    print("Saved to ", file_name)

############################################################
#  Training
############################################################

if __name__ == '__main__':
    import argparse

    # Parse command line arguments
    parser = argparse.ArgumentParser(
        description='Train Mask R-CNN to detect helmets.')
    parser.add_argument("command",
                        metavar="<command>",
                        help="'train' or 'splash'")
    parser.add_argument('--dataset', required=False,
                        metavar="/path/to/helmet/dataset/",
                        help='Directory of the Helmet dataset')
    parser.add_argument('--weights', required=True,
                        metavar="/path/to/weights.h5",
                        help="Path to weights .h5 file or 'coco'")
    parser.add_argument('--logs', required=False,
                        default=DEFAULT_LOGS_DIR,
                        metavar="/path/to/logs/",
                        help='Logs and checkpoints directory (default=logs/)')
    parser.add_argument('--image', required=False,
                        metavar="path or URL to image",
                        help='Image to apply the color splash effect on')
    parser.add_argument('--video', required=False,
                        metavar="path or URL to video",
                        help='Video to apply the color splash effect on')

    args = parser.parse_args()

    # Validate arguments
    if args.command == "train":
        assert args.dataset, "Argument --dataset is required for training"
    elif args.command == "splash":
        assert args.image or args.video,\
               "Provide --image or --video to apply color splash"

    print("Weights: ", args.weights)
    print("Dataset: ", args.dataset)
    print("Logs: ", args.logs)

    # Configurations
    if args.command == "train":
        config = HelmetConfig()
    else:
        class InferenceConfig(HelmetConfig):
            # Set batch size to 1 since we'll be running inference on
            # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
            GPU_COUNT = 1
            IMAGES_PER_GPU = 1
        config = InferenceConfig()
    config.display()

    # Create model
    if args.command == "train":
        model = modellib.MaskRCNN(mode="training", config=config,
                                  model_dir=args.logs)
    else:
        model = modellib.MaskRCNN(mode="inference", config=config,
                                  model_dir=args.logs)

    # Select weights file to load
    if args.weights.lower() == "coco":
        weights_path = COCO_WEIGHTS_PATH
        # Download weights file
        if not os.path.exists(weights_path):
            utils.download_trained_weights(weights_path)
    elif args.weights.lower() == "last":
        # Find last trained weights
        weights_path = model.find_last()
    elif args.weights.lower() == "imagenet":
        # Start from ImageNet trained weights
        weights_path = model.get_imagenet_weights()
    else:
        weights_path = args.weights

    # Load weights
    print("Loading weights ", weights_path)
    if args.weights.lower() == "coco":
        # Exclude the last layers because they require a matching
        # number of classes
        model.load_weights(weights_path, by_name=True, exclude=[
            "mrcnn_class_logits", "mrcnn_bbox_fc",
            "mrcnn_bbox", "mrcnn_mask"])
    else:
        model.load_weights(weights_path, by_name=True)

    # Train or evaluate
    if args.command == "train":
        train(model, args.dataset)
    elif args.command == "splash":
        detect_and_color_splash(model, image_path=args.image,
                                video_path=args.video)
    else:
        print("'{}' is not recognized. "
              "Use 'train' or 'splash'".format(args.command))


import torch
from torch.utils.data import Dataset
from PIL import Image
from torchvision import transforms as tf
import numpy as np
import os
from sys import exit
class RegionDataset(Dataset):
    """
    Return the data to feed  to the network
    Return:
        Image (2D tensor)
        region proposals (2D tensor): the region cut out of image based on the box
        label (scalar): the label corresponding to that region
        cooor_offset (tuple): offset coordinate
    """
    def __init__(self, img_dir, image_ids, rois, gtlabels, coord_offsets):
        super(RegionDataset, self).__init__()
        self.img_dir = img_dir
        self.image_ids = image_ids
        self.ROIs = rois # relative to images
        self.gtlabels = gtlabels
        self.coord_offsets= coord_offsets

        self.str2num = {
            'Background': 0,
            'Truck': 1,
            'Bus': 2
        }
        self.transform = self.get_transform()

    def __len__(self):
        return len(self.coord_offsets)

    def __getitem__(self, index):
        image_id = self.image_ids[index]
        image_path = os.path.join(self.img_dir, image_id +'.jpg')
        image = Image.open(image_path)
        label = self.gtlabels[index]
        roi = self.ROIs[index]
        # label string to number
        label = self.str2num[label]
        offset = self.coord_offsets[index]
        image = self.transform(image)

        return (
            image,
            torch.tensor(roi),
            torch.tensor(label),
            torch.tensor(offset)
        )

    def get_transform(self):
        return tf.Compose([
            tf.Resize((224, 224)),
            tf.ToTensor(),
            tf.Normalize(mean=[0.485, 0.456, 0.406], \
                std=[0.229, 0.224, 0.225])
        ])

    # def collate_fn(self, batch):
    #     images, rois, labels, offsets = zip(*batch)
    #     batch_size = len(batch)
    #     images = [image.unsqueeze(0) for image in images]
    #     images = torch.cat(images, dim=0)
    #     rois = torch.from_numpy(np.asarray(rois))
    #     labels = torch.from_numpy(np.asarray(labels))
    #     offsets = torch.from_numpy(np.asarray(offsets))
    #     idxs = torch.arange(0, batch_size).unsqueeze(1)
    #     return images, rois, idxs, labels, offsets


#!/usr/bin/env python3
#import networkx as nx
from numpy import linalg as LA
import numpy as np
import matplotlib.pyplot as plt
import ast
from sys import argv
import networkx as nx

thresh=.08

def check_is_connected(W):
    read_graph=nx.convert_matrix.from_numpy_matrix(W, parallel_edges=False)
    check=nx.is_connected(read_graph)
    return check

def bipartition_graph(W,ii):
    D=np.diag(W.sum(axis=0))
    D_half =np.diag(np.power(np.diag(D),-.5))# element wise for diagonal entries #L=D-A  #D degree matrix,  A  adjacency matrix
    mat=np.dot(np.dot(D_half,D-W),D_half)
    lam, V = LA.eig(mat)
    second_eig=V[:,1]
    print('second length is ',len(second_eig))
    #print(second_eig[:20])
    #print(second_eig[-20:])
    #print(np.mean(second_eig))
    plt.plot(second_eig)
    file_name_nct='second_eig_vector'+str(ii)+'.png'
    plt.savefig(file_name_nct)
    plt.close()
    #plt.show()

    part1_indecis=np.where(second_eig>0)
    part2_indecis=np.where(second_eig<=0)
    part1_list_ind=list(part1_indecis[0])
    part2_list_ind=list(part2_indecis[0])
    #print('part 1 length ',len(part1_list_ind))
    #print('part 2 length ',len(part2_list_ind))
    W_part1=W[part1_list_ind][:,part1_list_ind] #A[[0,1]] extracts first and secnond row of matrix A
    W_part2=W[part2_list_ind][:,part2_list_ind]
    dic_2parts={'W_p1':W_part1,'W_p2':W_part2,'ind_p1':part1_list_ind,'ind_p2':part2_list_ind }
    return dic_2parts

def Ncut_graph(ind_A,ind_B,W): # eq 2 page 889 shi malik paper
    # W 2d numpy array
    W_sliced=W[ind_A][:, ind_B]
    cut=W_sliced.sum()
    W_restA=W[ind_A] # extracting those rows
    W_restB=W[ind_B]
    assoc_A=W_restA.sum()
    assoc_B=W_restB.sum()
    Ncut_graph=cut/assoc_A+cut/assoc_B
    return Ncut_graph

def parse_frag_calcu_weight(file_pfrag_name):
    with open(file_frag_name, 'r') as file_frags:
            for frag in file_frags:
                frag_dic = ast.literal_eval(frag)
                list_frag_dic.append(frag_dic)
    frag_num=len(list_frag_dic)
    print('number of fragments is ',frag_num)
    W=np.zeros((frag_num,frag_num))
    for i in range(frag_num):
        for j in range(frag_num):  # half an hour
            if i<j:
                f1=list_frag_dic[i]
                f2=list_frag_dic[j]
                shared_snp_num=len(set(f1.keys())&set(f2.keys()))
                shared_snp_allel_num= len(set(f1.items()) & set(f2.items()))
                #if shared_snp_num>0:
                #    W[i,j]=(2*shared_snp_allel_num-shared_snp_num)/shared_snp_num
                #else: W[i,j]=0
                #if W[i,j]<0:W[i,j]=0
                W[i,j]=shared_snp_num
                W[j,i]=W[i,j]
            if i==j: W[i,i]=1
    #print(np.shape(W)) # dimension
    name_out=file_frag_name+'weight_shared'
    np.save(name_out, W)
    return W

def partitioning(W):
    list_branch_working=[W]
    list_branch_glob_indices_working=[list(range(np.shape(W)[0]))]

    list_branch_finished=[]
    list_branch_glob_indices_finished=[]
    ii=0
    while len(list_branch_working):
        ii=ii+1
        print('*******',str(ii),'*******')
        print('number of fragments in working is',[np.shape(W1)[0] for W1 in list_branch_working])
        print('number of fragments in finsihed is ',[np.shape(W1)[0] for W1 in list_branch_finished])

        W=list_branch_working[0]
        glob_indices=list_branch_glob_indices_working[0]

        if check_is_connected(W):
            dic_2parts= bipartition_graph(W,ii) # dic_2parts['W_p1'] and dic_2parts['ind_p1']
            ncut=Ncut_graph(dic_2parts['ind_p1'],dic_2parts['ind_p2'],W)
            print('n cut between first and second is third',np.shape(dic_2parts['W_p1'])[0],np.shape(dic_2parts['W_p2'])[0],np.round(ncut,6))

            if ((ncut>thresh) | (np.shape(dic_2parts['W_p1'])[0] ==0) | (np.shape(dic_2parts['W_p2'])[0]==0)):
                list_branch_finished.append(W)
                list_branch_glob_indices_finished.append(glob_indices)
                filename_out='part'+str(len(list_branch_finished))+'.txt'
                with open(filename_out, 'w+') as file_out:
                    for line_num in glob_indices:
                        file_out.write(str(line_num+1)+'\n')
                print('it goes to finsihed')
            else:
                list_branch_working.append(dic_2parts['W_p1'])
                glob_indices_p1=[glob_indices[i] for i in dic_2parts['ind_p1']]
                #print('glob ind 1 is',glob_indices_p1)
                list_branch_glob_indices_working.append(glob_indices_p1)
                list_branch_working.append(dic_2parts['W_p2'])
                glob_indices_p2=[glob_indices[i] for i in dic_2parts['ind_p2']]
                list_branch_glob_indices_working.append(glob_indices_p2)
                #print('glob ind 2 is',glob_indices_p2)
                #print(list_branch_glob_indices_working)
        else:
            list_branch_finished.append(W)
            list_branch_glob_indices_finished.append(glob_indices)
            filename_out='part'+str(len(list_branch_finished))+'.txt'
            with open(filename_out, 'w+') as file_out:
                for line_num in glob_indices:
                    file_out.write(str(line_num+1)+'\n')
            print('it is not connected and goes to finished')

        del list_branch_glob_indices_working[0]
        del list_branch_working[0]

    return list_branch_finished

if __name__ == "__main__":
    file_frag_name =argv[1]  # 'old_data/small_exmp2.txt' #'old_data/frag2_edit_dic_a0.txt'# argv[1]   #file_frag_name ='frag2_edit_dic_a1.txt' #'old/frag_dic.txt'
    list_frag_dic=[]
    if file_frag_name[-3:]=='txt':
        W=parse_frag_calcu_weight(file_frag_name)
    if file_frag_name[-3:]=='npy':
        W=np.load(file_frag_name)
    a= partitioning(W)
    print('>>><<<')
    print('number of partitions is ',len(a))


#!/usr/bin/env python
import essential_module
import pprint
import requests
from bs4 import BeautifulSoup
from lxml import etree
import re
import sqlite3
var = {
    'base': {
        "name": '信广财行',
        "db": r'C:\Users\Administrator\Desktop\spider\db_file\XGCH.db3',
        "headers": '''''',
        'partten': 2  # 1 Synchronization acquisition ;2 extract the uri and save db first and then collect detailed imformathon
    },
    'url': {
        'sourceUri': 'http://www.xgcaihang.com/investinfo?p={}',
        'varS': 1,
        'varE': 1,
        'varStep': 1,
        'requestMethod': 'get',
        'headers?': False,
        'autoExt': True, 'mustContain': '/invest/', 'noContain': '',
        # 'manualExt': True, 'extRegex': '', 'combine': '',
        'test': False
    },
    'detali': {
        'test': {
            'state': False,
            'uri': 'http://www.xgcaihang.com/invest/3262.html'
        },
        'content': {
            'requestMethod': 'get',
            'pageAttribute': 'html'
        },
        'associate': {
            'requestMethod': 'post',
            'posturi': 'http://www.xgcaihang.com/home/investinfo/investlog',
            'rule': 'http://www.xgcaihang.com/invest/(\w+).html',
            'postval': 'borrow_id=variable_a',
            'associated': []
        },
    },
    'tag': {
        "标题": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'xpath', 'selector': "//div[@class='touzwr']/h3",
            'dataHandle': {'htmlTagClear': 'yes'},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "完成度": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': 'body > div.qxWrapper > div.qxBox > div.zwj_xgch.content > div.touzye > div.toyelf > div:nth-of-type(5) > div > div > font',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '100%',
                'noContain': ''
            }
        },
        "借款金额": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "借款期限": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "年利率": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "作者": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "还款方式": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "回复内容": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "发标时间": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 3, 'values': None, 'tag': '回复内容',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "时间": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 3, 'values': None, 'tag': '回复内容',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "类型图片": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "网站编号": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "网址": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "编号": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 3, 'values': None, 'tag': '网址',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "年龄": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
        "性别": {
            # 1 source code; 2 fiexd format data ; 3 existing tag data
            'accessMethod': 1, 'values': None, 'tag': '',
            # 1 default page code; 2 associated page 1 ; 3 associated page 2;...;0 sorce uri
            'dataSource': 1,
            'extractMethod': 'css', 'selector': '',
            'dataHandle': {},
            'contentFilter': {
                'noNone': '',
                'mustContent': '',
                'noContain': ''
            }
        },
    }
}

def mydef(para):
    print("start extract ", para['base']['name'])
    essential_module.initialize_db(para['base']['db'])
    # initizlize databases,create table Content
    essential_module.update_db_field(para['base']['db'], para['tag'])
    # update table Content fields
    headers = essential_module.format_headers(para['base']['headers'])
    # format request headers info.
    for i in range(para['url']['varS'], para['url']['varE'] + 1, para['url']['varStep']):
        url = var['url']['sourceUri'].format(str(i))
        print('start extract list page:', url)
        listUrl = []
        num = 0
        dup = 0
        if para['url']['requestMethod'] == 'get':
            rep = requests.get(url, headers=headers)
            if para['url']['autoExt']:
                soup = BeautifulSoup(rep.text, "lxml")
                for linkEle in soup.find_all('a'):
                    link = str(linkEle.get('href'))
                    if re.match(para['url']['mustContain'], link):
                        if para['url']['noContain']:
                            if not re.match(para['url']['noContain'], link):
                                if re.match('/', link):
                                    g = re.match(
                                        '(^.*com)', para['url']['sourceUri'])
                                    link = g[0] + link
                                    listUrl.append(link)
                                    num += 1
                        else:
                            if re.match('/', link):
                                g = re.match(
                                    '(^.*com)', para['url']['sourceUri'])
                                link = g[0] + link
                                listUrl.append(link)
                                num += 1
        else:
            pass
        if listUrl:
            conn = sqlite3.connect(para['base']['db'])
            db = conn.cursor()
            for i in listUrl:
                s = 'select * from Content where 网址="%s"' % i
                db.execute('select * from Content where 网址="%s"' % i)
                if db.fetchall():
                    dup += 1
                else:
                    if not para['url']['test']:
                        db.execute(
                            'insert into Content (网址) values ("%s")' % i)
            conn.commit()
            db.close()
            conn.close()
        pprint.pprint(listUrl)
        print('extract links num is', num, ',duplicate is ',
              dup, 'new links is', num - dup)
    if para['url']['test']:
        exit()
    if para['base']['partten'] == 2:
        conn = sqlite3.connect(para['base']['db'])
        db = conn.cursor()
        db.execute("select 网址 from Content where 已采 is null")
        l = db.fetchall()
        targetUrl = []
        if l:
            for x in l:
                targetUrl.append(x[0])
            print('need collected data is', len(targetUrl))
            for x in targetUrl:
                if para['detali']['test']['state']:
                    requestUrl = para['detali']['test']['uri']
                else:
                    requestUrl = x
                print('start requests Uri:', requestUrl)
                if para['detali']['content']['requestMethod'] == 'get':
                    page = requests.get(requestUrl, headers=headers)
                    print(page.status_code)
                    if para['detali']['associate']['requestMethod'] == 'post':
                        s = re.search(
                            para['detali']['associate']['rule'], requestUrl)
                        source = re.sub(
                            'variable_a', s[1], para['detali']['associate']['postval'])
                        data = essential_module.format_postval(source)
                        print('start requests associated uri: ',
                              para['detali']['associate']['posturi'], data)
                        associatedPage = requests.post(
                            para['detali']['associate']['posturi'], data=data, headers=headers)
                        print(associatedPage.status_code)
                    else:
                        # get method ----------------------------------------------
                        pass
                    entry = {}
                    if para['detali']['content']['pageAttribute'] == 'html':
                        html = etree.HTML(page.text)
                        soup = BeautifulSoup(page.text, 'lxml')
                    else:
                        # para['detali']['content']['pageAttribute'] == 'json':
                        pass
                    for tag_name, tag_rule in para['tag'].items():
                        if tag_rule['accessMethod'] == 1:
                            if tag_rule['dataSource'] == 1:
                                if tag_rule['extractMethod'] == 'css':
                                    try:
                                        if para['detali']['test']['state']:
                                            if not tag_rule['selector']:
                                                entry[tag_name] = None
                                            else:
                                                if soup.select(tag_rule['selector']):
                                                    temporary_a = ''
                                                    for i in soup.select(tag_rule['selector']):
                                                        temporary_a += i.text
                                                    entry[tag_name] = temporary_a
                                        else:
                                            if soup.select(tag_rule['selector']):
                                                temporary_a = ''
                                                for i in soup.select(tag_rule['selector']):
                                                    temporary_a += i.text
                                                entry[tag_name] = temporary_a
                                            else:
                                                entry[tag_name] = None
                                    except NotImplementedError:
                                        print(
                                            tag_name, ' css selector is error, please change "nth-child" to "nth-of-type"')
                                        exit()
                                else:
                                    # tag_rule['extractMethod'] == 'xpath':
                                    if html.xpath(tag_rule['selector']):
                                        result = ''
                                        for ele in html.xpath(tag_rule['selector']):
                                            result += ele.text
                                        if tag_rule['dataHandle']['htmlTagClear'] == 'yes':
                                            result = re.subn(
                                                r'[\s\r\n]', '', result)[0]
                                        entry[tag_name] = result
                                    else:
                                        entry[tag_name] = None
                            elif tag_rule['dataSource'] == 2:
                                pass
                            else:
                                # tag_rule['dataSource'] == 3:
                                pass
                            if tag_rule['contentFilter']['noNone']:
                                if not entry[tag_name]:
                                    if para['detali']['test']['state']:
                                        entry[tag_name] = 'do not satisfy the necessary conditions--noNone'
                                        print(requestUrl, '\n', tag_name,
                                              ' extract result is None')
                                    else:
                                        print(requestUrl, '\n', tag_name,
                                              ' extract result is None')
                                        next
                            if tag_rule['contentFilter']['mustContent']:
                                f = re.match(
                                    tag_rule['contentFilter']['mustContent'], entry[tag_name])
                                if not f:
                                    print(
                                        requestUrl, '\n', tag_name, 'do not satisfy the necessary conditions--mustContent')
                                    if para['detali']['test']['state']:
                                        entry[tag_name] = None
                                    else:
                                        next
                            if tag_rule['contentFilter']['noContain']:
                                pass
                            # print(entry[tag_name])
                        elif tag_rule['accessMethod'] == 2:
                            pass
                        else:
                            # tag_rule['accessMethod']==3
                            pass
                    # ----------------------------------------------------
                    print(entry)
                else:
                    # post method------------------------------------------------------
                    pass
                if para['detali']['test']['state']:
                    exit()
        else:
            print('need collected data is 0')
        conn.commit()
        db.close()
        conn.close()

mydef(var)


import numpy as np

songs = ['Dame Da Ne',
 'National Anthem of USSR',
 'A$AP Rocky - L$D (LOVE x $EX x DREAMS)',
 'BTS(방탄소년단)DOPE(쩔어)',
 'Noak Hellsing - Lost',
 'Gurenge - Lisa',
 'Mac Miller - Good News',
 'Bastille - Those Nights',
 'Martin Garrix feat. Macklemore & Patrick Stump of Fall Out Boy - Summer Days',
 'Lauv - Changes',
 'ODESZA - Line Of Sight (feat. WYNNE & Mansionair)',
 'Never Gonna Give You U',
 'In The End - Linkin Park',
 'Lauv - Sims',
 'Doja Cat - Say So',
 'Lil Mosey - Blueberry Faygo (Dir. by ColeBennett)',
 'heck Wes - Mo Bamba',
 '마마무 (MAMAMOO) - 딩가딩가 (Dingga)',
 'Disturbed - The Sound Of Silence',
 'teve Jablonsky - Arrival To Earth',
 'LeeSSang(리쌍) Pursuing The Happiness(행복을 찾아서) (Feat. Jo Hyuna of Urban Zakapa)',
 "Richard Clayderman - Mariage d'amour",
 'Jake Scott - Man Who Stays',
 'Pharrell Williams - Happy',
 'love ride - christian french',
 'Justin Bieber & benny blanco - Lonely',
 'DaBaby - BOP on Broadway',
 'am Smith - To Die For',
 'Emilee - Heaven',
 'Rich & Sad',
 'John Barrowman - I know him so well (duet with Daniel Boys)',
 'erendipity',
 'Hippie Sabotage - Devil Eyes',
 'Bruno Mars - It Will Rain',
 'Cartoon - On & On (feat. Daniel Levi)',
 'Lil Nas X - HOLIDAY',
 'Dreamville - Down Bad feat. J.I.D, Bas, J. Cole, EarthGang, & Young Nudy',
 'NAV & Gunna ft. Travis Scott - Turks',
 'Mako - Smoke Filled Roo']

songs_vector = np.load('./songs.npy')

def image_to_song(image):
    # image values
    amusement = image[:, 0]
    anger = image[:, 1]
    awe = image[:, 2]
    contentment = image[:, 3]
    disgust = image[:, 4]
    excitement = image[:, 5]
    fear = image[:, 6]
    sadness = image[:, 7]

    # make song from image
    song = [
        fear,
        amusement,
        awe,
        contentment,
        anger,
        excitement,
        disgust,
        sadness
    ]

    return np.column_stack(song)

def closest_song(song, all_songs, labels):
    # closest in terms of euclidean distance
    distances = np.linalg.norm(all_songs-song, axis=1)
    return labels[np.argmin(distances)]

def predict_song_from_img(image):
    # predict a song given the output of the image emotion model
    # ex: predict(np.array([[0, 0, 1, 2, 2, 0, -2, 0]]))

    if request.method == 'OPTIONS':
        # Allows GET/POST requests from any origin with the Content-Type
        # header and caches preflight response for an 3600s
        headers = {
            'Access-Control-Allow-Origin': '*',
            'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
            'Access-Control-Allow-Headers': 'Content-Type',
            'Access-Control-Max-Age': '3600'
        }

        return ('', 204, headers)

    # Set CORS headers for the main request
    headers = {
        'Access-Control-Allow-Origin': '*'
    }

    song = image_to_song(image)
    return closest_song(song, songs_vector, songs), 200, headers


# -*- mode: python ; coding: utf-8 -*-
from PyInstaller.utils.hooks import collect_all
from shutil import which

datas = [('static', 'static'), ('templates', 'templates')]
binaries = []
hiddenimports = ['srsly.msgpack.util']
tmp_ret = collect_all('textblob')
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
tmp_ret = collect_all('spacy')
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
tmp_ret = collect_all('flask_socketio')
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]
tmp_ret = collect_all('en_core_web_sm')
datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2]

block_cipher = None

a = Analysis(
    ['app.py'],
    pathex=[],
    binaries=[*binaries, (which("ffmpeg"), "bin"), (which("ffprobe"), "bin")],
    datas=datas,
    hiddenimports=hiddenimports,
    hookspath=['hooks'],
    hooksconfig={},
    runtime_hooks=[],
    excludes=[],
    win_no_prefer_redirects=False,
    win_private_assemblies=False,
    cipher=block_cipher,
    noarchive=False,
)
pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)

exe = EXE(
    pyz,
    a.scripts,
    [],
    exclude_binaries=True,
    name='Fudgify',
    debug=False,
    bootloader_ignore_signals=False,
    strip=False,
    upx=False,
    console=False,
    disable_windowed_traceback=False,
    argv_emulation=False,
    target_arch=None,
    codesign_identity=None,
    entitlements_file=None,
    icon=['static/img/App_Icon.png'],
)

coll = COLLECT(
    exe,
    a.binaries,
    a.zipfiles,
    a.datas,
    strip=False,
    upx=True,
    upx_exclude=[],
    name='Fudgify',
)

app = BUNDLE(coll,
             name='Fudgify.app',
             icon='static/img/App_Icon.png',
             bundle_identifier='com.fudgify.Fudgify')


from datetime import date
from bs4 import BeautifulSoup
import asyncio
import aiohttp
import json
import hashlib
from .find_dates_in_string import find_date_in_string, normalise_str
from datetime import datetime

result = []

headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
              'application/signed-exchange;v=b3;q=0.9',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/109.0.0.0 Safari/537.36'
}

async def make_queue_vstu(un_id, url, filter_date):
    try:
        async with aiohttp.ClientSession() as session:
            tasks = []

            year = filter_date.year - 1
            while True:
                year += 1
                if year > date.today().year + 1:
                    break
                url_ = f'{url}/{year}/'
                try:
                    async with session.get(url=url_, headers=headers, timeout=20) as response:
                        if response.status == 404:
                            print(f'{response.status} - Нет такой страницы {url_}')
                            return
                        response_text = await response.text()
                        soup = BeautifulSoup(response_text, 'lxml')
                        main_container = soup.find('div', class_='content-wrapper').find('div', class_='unit-75')
                        try:
                            _ = int(soup.find('div', class_='page-header-wrapper').find('div', class_='unit-100').text)
                        except:
                            print(f'Конференции за {year} год отсутствуют.')
                            continue

                        if not main_container.find('dl', class_='conf'):
                            print(f'Конференции за {year} год отсутствуют.')
                            continue

                        print(f'{response.status} Обрабатываем страницу {year} года, {url_}')
                        conf_block = main_container.find_all('dl', class_='conf')

                        for n, conf in enumerate(conf_block):
                            title = normalise_str(conf.find('a').text)
                            # print(title)
                            if 'конфер' in title:
                                conf_url = f"https://www.vstu.ru{conf.find('a').get('href').strip()}"
                                # print(title, conf_url)
                                task = asyncio.create_task(parser_vstu_pages(session, un_id, conf_url, filter_date, n, year))
                                tasks.append(task)
                                # break

                except asyncio.TimeoutError as e:
                    print(f'Отказ по таймауту, страница {year} года, {url_}', e)
                except Exception as e:
                    raise Exception(f'Не удалось обработать ссылку {n} года {year} в {__name__} для {url_}\n{e}')

            await asyncio.gather(*tasks)

    except Exception as e:
        raise Exception(f'Не удалось обработать очередь для {__name__}\n{e}')

async def parser_vstu_pages(session, un_id, url, filter_date, n, year):
    def date_str_prep(str_):
        words = str_.split()
        for n, word in enumerate(words):
            if '—' in word:
                words.insert(n, str(year))
                break
        words.append(str(year))
        str_ = ' '.join(words)
        return str_

    try:
        async with session.get(url=url, headers=headers, timeout=20) as response:
            if response.status == 404:
                print(f'{response.status} - Нет такой страницы {url}')
                return
            response_text = await response.text()

            print(f'Выполняется ссылка {n + 1} года {year}, {url}')
            soup = BeautifulSoup(response_text, 'lxml')
            try:
                conf_block = soup.find('div', class_='content-wrapper').find('div', class_='unit-75')
            except:
                print(f'Не найден нужный блок в html, не обработана ссылка {url}')
                return

            un_name = 'Волгоградский государственный технический университет'

            conf_name = normalise_str(soup.find('div', class_='page-header-wrapper').find('div', class_='unit-100').text)
            local = False if 'международн' in conf_name.lower() else True

            conf_id = f"{un_id}_{url.split('/')[-2]}"
            hash_ = str(hashlib.md5(bytes(conf_id, 'utf-8')).hexdigest())
            conf_card_href = url

            prep_dates = date_str_prep(normalise_str(conf_block.find('p').text))
            # print(prep_dates)

            conf_date_begin = str(list(find_date_in_string(prep_dates))[0].date()) if \
                list(find_date_in_string(prep_dates)) else ''
            conf_date_end = str(list(find_date_in_string(prep_dates))[1].date()) if \
                len(list(find_date_in_string(prep_dates))) > 1 else ''
            # print(conf_date_begin, conf_date_end)

            conf_s_desc = conf_name
            reg_date_begin = ''
            reg_date_end = ''

            reg_href = ''
            conf_desc = ''
            org_name = ''
            themes = ''
            online = False
            conf_href = ''
            offline = False
            conf_address = ''
            contacts = ''
            rinc = False

            try:
                lines_tr = conf_block.find('table', id='table1').find_all(['tr'])
            except:
                lines_tr = []

            for tr in lines_tr:
                lines = tr.find_all(['td', 'li'])
                for line in lines:
                    # print(line.text)
                    if conf_s_desc == '':
                        conf_s_desc = normalise_str(line.text)

                    if ('заявк' in line.text.lower() or 'принимаютс' in line.text.lower() or 'участи' in line.text.lower()
                                or 'регистрац' in line.text.lower() or 'регистрир' in line.text.lower()) and reg_date_begin == '':
                        reg_date_begin = str(list(find_date_in_string(line.text.lower()))[0].date()) if \
                            list(find_date_in_string(line.text.lower())) else ''
                        reg_date_end = str(list(find_date_in_string(line.text.lower()))[1].date()) if \
                            len(list(find_date_in_string(line.text.lower()))) > 1 else ''

                    if not online and ('онлайн' in line.text.lower() or 'трансляц' in line.text.lower()):
                        online = True
                        conf_href = line.find('a').get('href') if line.find('a') else 'отсутствует'
                    if not offline and ('место' in line.text.lower() or 'адрес' in line.text.lower()):
                        offline = True
                        conf_address = normalise_str(line.get_text(separator=" "))

                    conf_desc = conf_desc + ' ' + normalise_str(line.get_text(separator=" "))

                    if reg_href == '' and ('регистрац' in line.text.lower() or 'зарегистр' in line.text.lower()
                                           or 'участия' in line.text.lower() or 'заявк' in line.text.lower()):
                        reg_href = line.find('a').get('href') if line.find('a') \
                                                                 and ('http:' in line.find('a').get('href') or
                                                                      'https:' in line.find('a').get('href')) and\
                                                                 ('.pdf' not in line.find('a').get('href') or
                                                                  '.doc' not in line.find('a').get('href') or
                                                                  '.xls' not in line.find('a').get('href')) else 'отсутствует'

                    if org_name == '' and 'организатор' in line.text.lower():
                        org_name = normalise_str(line.get_text(separator=" "))

                    if not online and ('онлайн' in line.text.lower() or 'трансляц' in line.text.lower() or
                                       'ссылка' in line.text.lower()):
                        conf_href = line.find('a').get('href') if line.find('a') else 'отсутствует'
                        online = True

                    if not offline and ('место' in line.text.lower() or 'адрес' in line.text.lower() or
                                        'очно' in line.text.lower()):
                        conf_address = normalise_str(line.text)
                        offline = True

                    if ('тел.' in line.text.lower() or 'контакт' in line.text.lower() or 'mail' in line.text.lower()
                            or 'почта' in line.text.lower() or 'почты' in line.text.lower()):
                        contacts = contacts + ' ' + normalise_str(line.text)

                    if line.find('a') and 'mailto' in line.find('a').get('href'):
                        contacts = contacts + ' ' + normalise_str(line.find('a').text)

                    if not rinc:
                        rinc = True if 'ринц' in line.text.lower() else False

            if (conf_date_begin != '' and datetime.strptime(conf_date_begin,
                                                            '%Y-%m-%d') >= filter_date) or conf_date_begin == '':
                result.append(
                        {'conf_id': conf_id,
                         'hash': hash_,
                         'un_name': un_name,
                         'local': local,
                         'reg_date_begin': reg_date_begin,
                         'reg_date_end': reg_date_end,
                         'conf_date_begin': conf_date_begin,
                         'conf_date_end': conf_date_end,
                         'conf_card_href': conf_card_href,
                         'reg_href': reg_href,
                         'conf_name': conf_name,
                         'conf_s_desc': conf_s_desc,
                         'conf_desc': conf_desc,
                         'org_name': org_name,
                         'themes': themes,
                         'online': online,
                         'conf_href': conf_href,
                         'offline': offline,
                         'conf_address': conf_address,
                         'contacts': contacts.strip(),
                         'rinc': rinc,
                         }
                    )

    except asyncio.TimeoutError as e:
        print(f'Отказ по таймауту, ссылка {n + 1} года {year}, {url}', e)

def parser_vstu(un_id, urls, date_):
    try:
        for url in urls:
            asyncio.run(make_queue_vstu(un_id, url, date_))
        with open(f'Conference_data/Parsers/JSON_log/{un_id}_{date.today()}.json', 'w', encoding="utf-8") as file:
            json.dump(result, file, indent=4, ensure_ascii=False)
        return result
    except Exception as e:
        print(e)

if __name__ == '__main__':
    print(parser_vstu('vstu', ['https://www.vstu.ru/nauka/konferentsii'], datetime.strptime('2023.01.01', '%Y.%m.%d')))


def solution(answers):
    answer = []
    one = [1, 2, 3, 4, 5]
    oCount = 0
    two = [2, 1, 2, 3, 2, 4, 2, 5]
    tCount = 0
    thr = [3, 3, 1, 1, 2, 2, 4, 4, 5, 5]
    thCount = 0
    while len(answers) > 0:
        cur = answers[0]
        o = one.pop(0)
        one.append(o)
        if o == cur:
            oCount += 1
        t = two.pop(0)
        two.append(t)
        if t == cur:
            tCount += 1
        tc = thr.pop(0)
        thr.append(tc)
        if tc == cur:
            thCount += 1

        answers.pop(0)

    m = max(oCount, tCount, thCount)
    if m == oCount:
        answer.append(1)
    if m == tCount:
        answer.append(2)
    if m == thCount:
        answer.append(3)

    return answer


from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
from io import BytesIO
from PIL import Image
import numpy as np
import cv2
import os

def start(bot, update):
    update.message.reply_text('I am extract contrast bot! send me some picture and I will apply the extract black to white Filter!!')

def hello(bot, update):
    update.message.reply_text(
        'Hello {}'.format(update.message.from_user.first_name))

def applyclaheBGR(img, clipLimit=3., tileGridSize=(8, 8)):
    clahe = cv2.createCLAHE(clipLimit, tileGridSize)
    lab = cv2.cvtColor(img, cv2.COLOR_BGR2LAB)  # convert from BGR to LAB color space
    l, a, b = cv2.split(lab)  # split on 3 different channels
    l2 = clahe.apply(l)  # apply CLAHE to the L-channel
    lab = cv2.merge((l2, a, b))  # merge channels
    img = cv2.cvtColor(lab, cv2.COLOR_LAB2BGR)  # convert from LAB to BGR
    return img

def applyotsuBGR(img_orig):
    img_orig = cv2.cvtColor(img_orig, cv2.COLOR_BGR2GRAY)
    hist, bins = np.histogram(img_orig.flatten(), 256, [0, 256])

    sum1 = np.sum(range(0, 256) * hist)
    total = img_orig.shape[0] * img_orig.shape[1]
    sumB = 0
    wB = 0
    maximum = 0
    threshold1 = 0
    threshold2 = 0
    for i in range(0, 256):
        wB = wB + hist[i]
        if (wB == 0): continue
        wF = total - wB
        if (wF == 0): break
        sumB = sumB + i * hist[i]
        mB = sumB / wB
        mF = (sum1 - sumB) / wF
        between = wB * wF * (mB - mF) * (mB - mF)
        if (between >= maximum):
            threshold1 = i
            if (between > maximum):
                threshold2 = i
            maximum = between

    thresh = (threshold1 + threshold2) / 1.3

    tr = np.zeros(256, dtype=np.uint8)
    for i in range(0, 256):
        if (i < thresh):
            tr[i] = 0
        else:
            tr[i] = 255

    img_res = tr[img_orig]
    img_res = 255-img_res

    return img_res

def applycustomcontrastBGR(img):
    colors = np.array([[37, 1, 1],
                       [120, 1, 1],
                       [1, 1, 1]])
    colotsu = colors * np.array([[1, 255, 240],
                                 [1, 255, 240],
                                 [255, 255, 255]])
    colotsu = colotsu + np.array([[20, 0, 0],
                                  [20, 0, 0],
                                  [0, 0, 0]])
    colotsl = colors * np.array([[1, 50, 150],
                                 [1, 50, 100],
                                 [1, 50, 220]])
    colotsl = colotsl - np.array([[20, 0, 0],
                                  [20, 0, 0],
                                  [0, 0, 0]])

    phi = 1
    theta = 1
    maxInt = 255
    nimg = (maxInt / phi) * (img / (maxInt / theta)) ** 2
    nimg = np.array(nimg, dtype="uint8")
    nimg = (maxInt / phi) * (nimg / (maxInt / theta)) ** 2
    nimg = np.array(nimg, dtype="uint8")
    # CLAHE (Contrast Limited Adaptive Histogram Equalization)
    img2 = applyclaheBGR(nimg, 3., (8, 8))
    # cv2.imshow("msk0",img2)
    # cv2.waitKey()
    converted = cv2.cvtColor(img2, cv2.COLOR_BGR2HSV)

    msk0 = cv2.inRange(converted, colotsl[0], colotsu[0])
    msk1 = cv2.inRange(converted, colotsl[1], colotsu[1])
    msk2 = cv2.inRange(converted, colotsl[2], colotsu[2])

    final = 255 - cv2.merge((msk2 - msk0, msk2, msk2 - msk0))
    return final

def getimg(bot, update):
    photo = update.message.photo[-1].file_id
    im = bot.getFile(file_id=photo)
    im.download()
    img = cv2.imread(os.path.basename(im.file_path))

    final=applyotsuBGR(img)

    pil_im = Image.fromarray(final) # actually convert to pil image
    bio = BytesIO()
    bio.name = 'image.jpeg'
    pil_im.save(bio, 'JPEG')
    bio.seek(0)
    update.message.reply_photo(photo=bio)

updater = Updater('361559041:AAFHJzj7wN8Rn_PBMmGt8NNuXyJ9owvB6lQ')

updater.dispatcher.add_handler(CommandHandler('start', start))
updater.dispatcher.add_handler(CommandHandler('hello', hello))
updater.dispatcher.add_handler(MessageHandler(Filters.photo, getimg))

updater.start_polling()
updater.idle()


# -*- coding: utf-8 -*-
"""
Created on Thu Feb  5 14:20:55 2015

@author: damienrj
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('ggplot')
import seaborn as sns

#Set the display precision
pd.set_option('precision', 4)

df = pd.DataFrame.from_csv('atus_00005.csv')
df['weighted_sleep']=df.BLS_PCARE_SLEEP * df.WT06
df.EMPSTAT.astype('category')
df['SEX'] = df['SEX'].replace([1, 2], ['Male', 'Female'])
df.SEX.astype('category')
df.AGE = pd.cut(df.AGE, [15, 20, 25, 35, 45, 55, 65, 90], right = 'False')

#Load the state names
num,name = np.loadtxt('state_id.txt',unpack=True,dtype=str,delimiter=',')

# replace with state names
df['STATEFIP'] = df['STATEFIP'].replace(num.astype(np.int64), name)

age_sum = df.groupby(['YEAR',  'AGE', 'SEX' ]).sum()
results = age_sum.weighted_sleep / age_sum.WT06
results = results.unstack(0)/60

f = plt.figure(figsize=(12, 8))
data = results[2013].unstack(1)
data.plot(ax=f.gca(), kind='bar', ylim=(7,10), y = ['Male', 'Female'])
print(data)

#This section finds the number of people in eac state, each year.
grouped_state = df.groupby(['YEAR', 'STATEFIP'])
num_per_state = grouped_state.size()
num_per_state = num_per_state.unstack(0)

d = {'Mean' : num_per_state.mean(1), 'STD' : num_per_state.std(1)}
pd.DataFrame(d)

#This section groups the data by metro area
grouped = df.groupby(['YEAR', 'METAREA', 'SEX' ] )
num_LA_NY=grouped.size().unstack(0).loc[[4481, 5601]]
results = grouped.weighted_sleep.sum() / grouped.WT06.sum()
results = results.unstack(0)/60
la_ny_sleep = results.loc[[4481, 5601]]
la_ny_sleep = la_ny_sleep.rename(index={4481:"LA Metro", 5601: "NY Metro"})
la_ny_sleep = la_ny_sleep.stack(0).unstack(1)

#Error calulation using replicate weights
column_names = df.columns.values
indices = [i for i, s in enumerate(column_names) if 'RWT06_' in s]
column_names = column_names[indices]

Y0 = grouped.weighted_sleep.sum() / grouped.WT06.sum();
data = []

def wavg(group, name):
    d = group['BLS_PCARE_SLEEP']
    w = group[name]
    return (d * w).sum() / w.sum()

for name in column_names:
     data.append((grouped.apply(wavg, name)-Y0)**2)

error = np.sqrt(sum(data)*(4./160))/60

la_error = error.loc[[4481, 5601]]
la_error = la_error.rename(index={4481:"LA Metro", 5601: "NY Metro"})
la_error = la_error.stack(0).unstack(1)

f = plt.figure()
f.gca().get_xaxis().get_major_formatter().set_useOffset(False)
la_ny_sleep['Male'].unstack(0).plot(ax = f.gca(), yerr = la_error['Male'].unstack(0))
plt.ylabel("Hours Slept")
plt.title('Males')


import matplotlib
matplotlib.use('Agg')
from numpy import loadtxt,diff#,savetxt,shape,argsort
from pylab import *
#from glob import glob
import os
import sys
from scipy import integrate
sys.path.append('/home/fbrasili/.python')
from init_file import *

def compute_pot(g):
    #g = where( g==0 , 1.e-18 , g )
    return -log(g)

def compute_Fr(r,g):
    #g = where( g==0 , 1.e-18 , g )
    return gradient(g)/gradient(r)/g

def compute_threshold_gr_pot(fol=None):
    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    gr_path = os.path.join( fol , 'gr_ions_npart_nopbc.dat' )
    r,g = loadtxt( gr_path, skiprows=1 , unpack=True )
    cond = g != 0
    r,g = r[cond],g[cond]
    U = -log(g[1:])
    F = diff(g)/diff(r)/g[1:]
    r,g = r[1:],g[1:]
    savetxt( os.path.join( fol , 'Fr.dat' ) , c_[r,F] )
    cond = r<20
    r,F,U = r[cond],F[cond],U[cond]
    cond = ( r>max(r[F==F.min()]) ) #* ( U+1>0 )
    return r[cond].min()

def compute_threshold_gr_fact(frac=0.2 , fol=None):
    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    gr_path = os.path.join( fol , 'gr_ions_npart_nopbc.dat' )

    init_fname = os.path.join( fol , "mgel_nanoparts_eq10e6.dat" )
    _,_,box,_,_ = read_init(init_fname)
    V = box[0]**3
    _,N = check_init(init_fname, retPartType=3, ret=True, verb=False)

    r,g = loadtxt( gr_path, skiprows=1 , unpack=True )
    cond = ( r > max(r[ g == g.max() ]) ) * ( g <= g.max() * frac )
    return r[cond].min()

def compute_threshold_gr_Coul(fol=None):
    gr_paths = {
            2 : 'gr_neg-cions_npart.dat',
            3 : 'gr_ions_npart_nopbc.dat',
            4 : 'gr_npart_npart_nopbc.dat',
            5 : 'gr_pos-cions_npart.dat'
            }
    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    init_fname = os.path.join( fol , "mgel_nanoparts_eq10e6.dat" )
    gr_paths = { i: os.path.join( fol , p ) for i,p in gr_paths.items() }

    _,_,box,_,_ = read_init(init_fname)
    V = box[0]**3
    Zs,Ns = check_init(init_fname, retPartType="all", ret=True, verb=False)
    Zr = 0
    for key in gr_paths.keys():
        Z,N = Zs[key],Ns[key]
        if key==4: N=N-1
        r,g = loadtxt( gr_paths[key] , skiprows=1 , unpack=True )
        Zr += 4*pi * Z*N/V * integrate.cumtrapz( g * r**2 , r , initial=0 )
    Ir = integrate.cumtrapz( Zr/r**2 , r , initial=0 )
    Ur = Zs[4]/r - Ir + Ir[-1]

    figure()
    plot(r,Ur)
    savefig("fol"); colse()
    return r[Ur+1<0].max()

def myPlot(observable , system , fol=None , norm=False , ret=False):
    filename = {
            'profile': {
                'nanoparticles': 'nanoparts_profile.dat',
                'microgel': 'monomers_profile.dat',
                'charge': 'charge_profile.dat',
                'positive counterions': 'pos_cions_profile.dat',
                'negative counterions': 'neg_cions_profile.dat'
                },
            'g(R)': {
                'nanoparticle - nanoparticle': 'gr_npart_npart_nopbc.dat', #nr_npart_npart_pbc.dat
                'charged monomer - charged monomer': 'gr_ions_ions_nopbc.dat', #nr_ions_ions_pbc.dat
                'nanoparticle - charged monomer': 'gr_ions_npart_nopbc.dat', #nr_ions_npart_pbc.dat
                'nanoparticle - positive counterion': 'gr_pos-cions_npart.dat',
                'nanoparticle - negative counterion': 'gr_neg-cions_npart.dat',
                'charged monomer - negative counterion': 'gr_ions_neg-cions.dat',
                'charged monomer - positive counterion': 'gr_ions_pos-cions.dat',
                'positive counterion - negative counterion': 'gr_neg-cions_pos-cions.dat'
                },
            'gyration radius': {
                'chull': os.path.join( "chull_rh" , "rh.dat" ),
                'zeno': 'swelling'
                },
            'hydrodynamic radius': {
                'chull': os.path.join( "chull_rh" , "rh.dat" ),
                'zeno': 'swelling'
                }
            }
    args = {
            'profile': [4,(1,3)],
            'g(R)': [1,(0,1)],
            'gyration radius': [0,(0,1)],
            'hydrodynamic radius': [0,(0,2)]
            }

    if   observable == 'prof'    : observable = 'profile'
    elif observable == 'gr'      : observable = 'g(R)'
    elif observable == 'Rg'      : observable = 'gyration radius'
    elif observable == 'Rh'      : observable = 'hydrodynamic radius'

    if observable == 'profile':
        if system in [ 'nanoparticle' , 'nanoparts' , 'nanopart' , 'nps' , 'np' ]: system,retPartType = 'nanoparticles',4
        elif system in [ 'mgel' , 'monomer' , 'monomers' ]: system,retPartType = 'microgel',None
        elif system == 'pos_cions': system,retPartType = 'positive counterions',5
        elif system == 'neg_cions': system,retPartType = 'negative counterions',2
    elif observable == 'g(R)':
        if system in [ 'nanoparticle' , 'nanoparts' , 'nanopart' , 'nps' , 'np' ]:
            system = 'nanoparticle - nanoparticle'
            retPartType = 4
        elif system in [ 'ion' , 'ions' , 'monomer' , 'monomers' ]:
            system = 'charged monomer - charged monomer'
            retPartType = 3
        elif system == 'np-ion':
            system = 'nanoparticle - charged monomer'
            retPartType = 3
        elif system == 'np-pos_cion':
            system = 'nanoparticle - positive counterion'
            retPartType = 5
        elif system == 'np-neg_cion':
            system = 'nanoparticle - negative counterion'
            retPartType = 2
        elif system == 'ion-neg_cion':
            system = 'charged monomer - negative counterion'
            retPartType = 2
        elif system == 'ion-pos_cion':
            system = 'charged monomer - positive counterion'
            retPartType = 5
        elif system == 'pos_cion-neg_cion':
            system = 'positive counterion - negative counterion'
            retPartType = 2
    if system == 'swelling': system = 'chull'

    skiprows,usecols = args[observable]
    if observable == 'profile' and system == 'charge': skiprows = 1

    if fol is None:
        os.system( "pwd >> path.dat" )
        fol = str( loadtxt( "path.dat" , dtype=str ) )
        os.system( "rm path.dat" )
    if 'alpha' not in fol:
        if system in ['chull', 'zeno']: fol = os.path.join( fol , 'swelling' )
        else                          : fol = os.path.join( fol , 'alpha_0.00' )

    path = os.path.join( fol , filename[observable][system] )
    x,y = loadtxt( path , skiprows=skiprows , usecols=usecols , unpack=True )

    if system in ['chull', 'zeno']: label = observable
    else                          : label = system

    if type(norm) is float:
        y /= norm
    elif norm:
        init_fname = os.path.join( fol , "mgel_nanoparts_eq10e6.dat")
        if not os.path.isfile( init_fname ): init_fname = os.path.join( fol , "mgel_cions_eq10e6.dat")
        if system == 'microgel':
            _,Nions = check_init(init_fname, retPartType=3, ret=True, verb=False)
            _,Nneut = check_init(init_fname, retPartType=1, ret=True, verb=False)
            Nparts = Nions + Nneut
        else:
            _,Nparts = check_init(init_fname, retPartType=retPartType, ret=True, verb=False)
        #_,_,box,_,_ = read_init(init_fname)
        y /= Nparts

    if ret:
        return x, y, label
    else:
        plot(x, y, label=label)

'''
non ha senso scrivere una funzione per i due/tre sublplot
def plot_swellingCurve(fol='./', withZeno=False, ret=False):
    Xrg,Yrg,label = myPlot('Rg', 'swelilng' , fol=fol , ret=True)
    plot(x,y)
'''


from flask import request, url_for
from flask_api import status
import json

class ReqResController:

	def __init__(self):

		self.status_codes = {
			"200": status.HTTP_200_OK,
			"201": status.HTTP_201_CREATED,
			"204": status.HTTP_204_NO_CONTENT,
			"400": status.HTTP_400_BAD_REQUEST,
			"403": status.HTTP_403_FORBIDDEN,
			"404": status.HTTP_404_NOT_FOUND
		}

	def response(self, status_code, content=False):
		if content:
			resp_content = json.dumps(content)
			return resp_content, self.status_codes[status_code]
		else:
			return self.status_codes[status_code]

	def check_request_format(self, request_body):
		body = request_body.data
		if not body:
			return self.response(400, {"error": "Invalid JSON"})
		return False


# Solution 1, simulation
class Solution:
    def change(self, word, idx):
        if word[0].lower() in 'aeiou':
            res = word
        else:
            res = word[1:] + word[0]

        res = res + 'ma' + 'a' * idx

        return res

    def toGoatLatin(self, S: str) -> str:

        words = S.split()

        res = ' '.join(self.change(word, i + 1) for i, word in enumerate(words))

        return res


from keys import CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET, DROPBOX_ACCESS_TOKEN
from tweepy import Stream, OAuthHandler
from tweepy.streaming import StreamListener
import json
import dropbox
from datetime import datetime
from time import sleep
from netifaces import interfaces, ifaddresses, AF_INET
import RPi.GPIO as GPIO
from flask import Flask, render_template
import threading

webapp = Flask(__name__)

auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
TRIGGER_MESSAGE = open('trigger','r').readline().rstrip("\n")
print("Using", TRIGGER_MESSAGE, "as trigger message")

class myThread (threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
    def run(self):
        uploadIPtoDropbox()
        listenForTweets()

class LightsController:
    STAR_LIGHTS = 18
    TREE_LIGHTS = 23
    BALL_LIGHTS = 24
    STATUS_LIGHT = 25

    def __init__(self):

        self.lightsBusy = False

        GPIO.setmode(GPIO.BCM)

        GPIO.setup(18, GPIO.OUT)

        GPIO.setup(self.STAR_LIGHTS, GPIO.OUT)
        GPIO.setup(self.TREE_LIGHTS, GPIO.OUT)
        GPIO.setup(self.BALL_LIGHTS, GPIO.OUT)
        # GPIO.setup(self.STATUS_LIGHT, GPIO.OUT)
        print("Initialised GPIO pins")
        self.flashAllTogether(5)
        self.off()

    def off(self):

        print("**All lights off**")

        GPIO.output(self.STAR_LIGHTS, False)
        GPIO.output(self.TREE_LIGHTS, False)
        GPIO.output(self.BALL_LIGHTS, False)
        # GPIO.output(self.STATUS_LIGHT, False)

    def treeOn(self):

        GPIO.output(self.TREE_LIGHTS, True)

    def treeOff(self):

        GPIO.output(self.TREE_LIGHTS, False)

    def starOn(self):

        GPIO.output(self.STAR_LIGHTS, True)

    def starOff(self):

        GPIO.output(self.STAR_LIGHTS, False)

    def ballsOn(self):

        GPIO.output(self.BALL_LIGHTS, True)

    def ballsOff(self):

        GPIO.output(self.BALL_LIGHTS, False)

    def statusOn(self):

        print("**Status light on**")

        # GPIO.output(self.STATUS_LIGHT, True)

    def statusOff(self):

        print("**Status light off**")

        # GPIO.output(self.STATUS_LIGHT, False)

    def perm_on(self):
        self.lightsBusy = True
        self.treeOn()
        self.ballsOn()
        self.starOn()

    def on(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime

        self.lightsBusy = True

        print("**All lights on**")
        for n in range(0, int(numberFlashes)):
            # print("flash number: " + str(n))
            self.treeOn()
            self.starOn()
            self.ballsOn()
            sleep(flashTime)

        self.off()

    def flashTree(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            # print("flash number: " + str(n))
            self.treeOn()
            sleep(flashTime)
            self.treeOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashBalls(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.ballsOn()
            sleep(flashTime)
            self.ballsOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashStar(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.starOn()
            sleep(flashTime)
            self.starOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashStarAndBalls(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.starOn()
            self.ballsOn()
            sleep(flashTime)
            self.starOff()
            self.ballsOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashAllTogether(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 2.0

        self.lightsBusy = True

        for n in range(0, int(numberFlashes)):
            self.ballsOn()
            self.treeOn()
            self.starOn()
            sleep(flashTime)
            self.ballsOff()
            self.treeOff()
            self.starOff()
            sleep(flashTime)

        self.lightsBusy = False

    def flashAllSequence(self, duration):

        flashTime = 0.2
        numberFlashes = duration / flashTime / 3.0

        self.lightsBusy = True
        # print (datetime.datetime.now())

        for n in range(0, int(numberFlashes)):
            self.starOff()
            self.treeOn()
            sleep(flashTime)
            self.treeOff()
            self.ballsOn()
            sleep(flashTime)
            self.ballsOff()
            self.starOn()
            sleep(flashTime)
            self.off()

        # print (datetime.datetime.now())
        self.lightsBusy = False

# Set up Twitter listener
class listener(StreamListener):

    def on_data(self, data):
        d = json.loads(data)
        print("trigger message found in tweet: %s" % d['text'])
        lc.flashAllSequence(20)
        return True

    def on_error(self, status):
        print (status)
        return False

def uploadIPtoDropbox():

    # Establish dropbox connection - useful for uploading IP address to DB

    try:
        dbx = dropbox.Dropbox(DROPBOX_ACCESS_TOKEN)
        ip_message = "Socialite Christmas Jumper\n\n"

        # from http://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib
        for ifaceName in interfaces():
            addresses = [i['addr'] for i in ifaddresses(ifaceName).setdefault(AF_INET, [{'addr':'No IP addr'}] )]
            ip_message = ip_message +  '%s: %s\n' % (ifaceName, ', '.join(addresses))

        ip_message = ip_message + "Updated: {1}".format("xx.xx.xx.xx", datetime.now())
        dbx.files_upload(ip_message.encode('utf-8'), '/jumper_ip.txt', mode=dropbox.files.WriteMode('overwrite'))
        print("IP address uploaded to Dropbox")

    except:
        print("Dropbox connection error. Attempting again in 10 seconds.")
        lc.on(10)
        uploadIPtoDropbox()

# Set up global lights controller object
lc = LightsController()

def listenForTweets():
    try:
        twitterStream = Stream(auth, listener())
        twitterStream.filter(track=[TRIGGER_MESSAGE])
        print("Twitter stream listener started.")

    except:
        print("Twitter connection error. Trying again in 10 seconds.")
        lc.on(10)
        listenForTweets()

# API routes
@webapp.route('/')
def index():
    return render_template('index.html')

@webapp.route('/on/')
def web_on():
    global lc
    lc.perm_on()
    return render_template('index.html', message="Message received! You made my lights shine!")

@webapp.route('/off/')
def web_off():
    global lc
    lc.off()
    return render_template('index.html', message="Message received! Had enough of my lights, I see...")

@webapp.route('/flash/')
def web_flash():
    global lc
    lc.flashAllTogether(20)
    return render_template('index.html', message="Message received! Flashing has commenced.")

@webapp.route('/chase/')
def web_chase():
    global lc
    lc.flashAllSequence(20)
    return render_template('index.html', message="Message received! You're into the fancy stuff I see.")

# Disabled due to SSL certificate error in school
#listenThread = myThread()
#listenThread.start()

if __name__ == '__main__':
    webapp.run(host="0.0.0.0", port=5000)


from fastapi import APIRouter, Request
from typing import Optional

from ..modules.KNN import predict_data

import time

router = APIRouter()

@router.get("/predict", tags = ["predict"])
def check_predict(gender: int = 0, age: int = 0, hypertenstion: int = 0, heart_disease: int = 0,ever_marries: int = 0, Residence_type: int = 0, avg_glucose_level: int = 0, bmi: int = 0, work_type_Govt_job: int = 0, work_type_Never_worked: int = 0, work_type_Private: int = 0, work_type_Self_employed: int = 0, work_type_children: int = 0, smoking_status_Unknown: int = 0, smoking_status_formerly_smoked: int = 0, smoking_status_never_smoked: int = 0, smoking_status_smokes: int = 0):
    start_time = time.time()
    try:

        if (gender or age or hypertenstion or heart_disease or ever_marries or Residence_type or avg_glucose_level or bmi or work_type_Govt_job or work_type_Never_worked or work_type_Private or work_type_Self_employed or work_type_children or smoking_status_Unknown or smoking_status_formerly_smoked or smoking_status_never_smoked or smoking_status_smokes):
            data = [gender, age, hypertenstion, heart_disease,ever_marries, Residence_type, avg_glucose_level, bmi, work_type_Govt_job, work_type_Never_worked, work_type_Private, work_type_Self_employed, work_type_children, smoking_status_Unknown, smoking_status_formerly_smoked, smoking_status_never_smoked, smoking_status_smokes]

            clasify = predict_data(data)
            # clasify = predict_data(data)

            if clasify:
                result = { "Predict_class:": int (clasify[0]),
                            "Status": "Success" }

            else:
                result = { "Status" : "Error" }
        else:
            result =  { "Status" : "Error",
                        "Message" : "Invalid get parameters" }
    except:
        result =  { "Status" : "Error",
                    "Message" : "Something went wrong" }
    finally:
        end_time = time.time()
        result["Respone_time"] = end_time - start_time
        return result