from airflow import DAG
from airflow.operators.python import PythonOperator
from datetime import date, datetime, timedelta
import glob
import hashlib
import pandas as pd
import re
import time
import os

# set up input and output folders
input_folder = '/Users/yanqingfu/airflow/in'
successful_folder = '/Users/yanqingfu/airflow/successful'
unsuccessful_folder = '/Users/yanqingfu/airflow/unsuccessful'

# get today's date at 12:00 AM
today = datetime.now()
start_date = today.replace(hour=0, minute=0, second=0, microsecond=0)

# define DAG arguments
default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=3),
    'start_date': start_date,
}

# define DAG
dag = DAG(
    'membership_data_pipeline',
    default_args=default_args,
    description='Pipeline to process membership applications',
    schedule=timedelta(hours=1),
    catchup=False,
)

# get first name and last name from full name
def extract_name_parts(name):
    regex = r'([A-Z][a-z]+) ([A-Z][a-z]+)'
    match = re.search(regex, name)
    if match:
        return match.group(1), match.group(2)

# format brithday as YYYYMMDD for different date formats
def format_dob(dob):
    formats = ['%d-%m-%Y', '%Y-%m-%d', '%m/%d/%Y', '%Y/%m/%d']
    for date_format in formats:
        try:
            return datetime.strptime(dob, date_format).strftime('%Y%m%d')
        except ValueError:
            pass

# create memebership id based on last name and birthday hash
def generate_membership_id(row):
    hash_dob = hashlib.sha256(row['date_of_birth'].encode()).hexdigest()[:5]
    return f"{row['last_name']}_{hash_dob}"

# check for new csv files and load them into one dataframe
def get_new_csv_files():
    df = pd.DataFrame()
    # buffer time of 10 minutes
    minutes_ago = time.time() - 600
    all_files = glob.glob(input_folder + "/*.csv")
    # get all files that were added within the last 10 minutes
    new_files = [file for file in all_files if os.path.getmtime(file) >= minutes_ago]
    if new_files:
        df_list = [pd.read_csv(file) for file in new_files]
        df = pd.concat(df_list, ignore_index=True)
        return df

# clean and transform data
def format_data(df):
    if df is not None and not df.empty:
        # drop rows with missing values
        df.dropna(subset=['name', 'date_of_birth'], inplace=True)
        # remove duplicate rows based on all columns
        df.drop_duplicates(inplace=True)
        # get first name and last name
        df[['first_name', 'last_name']] = df['name'].apply(
            extract_name_parts).apply(pd.Series)
        # format date of birth
        df['date_of_birth'] = df['date_of_birth'].apply(format_dob)
        # remove spaces in mobile number
        df['mobile_no'] = df['mobile_no'].str.replace(' ', '')
        # check if age is above 18 years old as of 1 Jan 2022
        df['above_18'] = (pd.to_datetime('20220101',
                                        format='%Y%m%d') - pd.to_datetime(df['date_of_birth'],
                                                                        format='%Y%m%d')).dt.days > 18 * 365
        return df
    else:
        return pd.DataFrame()

# perform validity checks and create membership IDs for successful applications
def check_validity(df):
    result_dict = {}
    if df is not None and not df.empty:
        # check if mobile number is 8 digits, age is above 18 and email ends with .com or .net
        valid_mobile = df['mobile_no'].str.len() == 8
        valid_age = df['above_18']
        valid_email = df['email'].str.endswith(('.com', '.net'))
        # create membership id for successful applications
        successful_applications = df[valid_mobile & valid_age & valid_email].copy()
        successful_applications['membership_id'] = successful_applications.apply(
            generate_membership_id, axis=1)
        unsuccessful_applications = df[~(valid_mobile & valid_age & valid_email)]
        result_dict = {
            'successful_applications': successful_applications,
            'unsuccessful_applications': unsuccessful_applications
        }
        return result_dict
    else:
        return result_dict

# write results as csv files to different status folders
def output_csv_files(result_dict):
    if result_dict:
        successful_applications = result_dict['successful_applications']
        unsuccessful_applications = result_dict['unsuccessful_applications']
        # get the current datatime as YYYYMMDDHH
        now = datetime.now()
        formatted_datetime = now.strftime('%Y%m%d%H')
        # create folders if they don't exist
        for folder in [successful_folder, unsuccessful_folder]:
            if not os.path.exists(folder):
                os.makedirs(folder)
        # write csv files
        successful_applications.to_csv(
            successful_folder +
            '/successful_applications' +
            '_' +
            formatted_datetime +
            '.csv',
            index=False)
        unsuccessful_applications.to_csv(
            unsuccessful_folder +
            '/unsuccessful_applications' +
            '_' +
            formatted_datetime +
            '.csv',
            index=False)

# define task to get new csv files and load them into one dataframe
get_new_csv_files_task = PythonOperator(
    task_id='get_new_csv_files',
    python_callable=get_new_csv_files,
    dag=dag
)

# define task to format data and clean it
format_data_task = PythonOperator(
    task_id='format_data',
    python_callable=format_data,
    op_kwargs={'df': get_new_csv_files_task.output},
    dag=dag
)

# define task to check validity and create membership IDs
check_validity_task = PythonOperator(
    task_id='check_validity',
    python_callable=check_validity,
    op_kwargs={'df': format_data_task.output},
    dag=dag
)

# define task to write csv files to different status folders
output_csv_files_task = PythonOperator(
    task_id='output_csv_files',
    python_callable=output_csv_files,
    op_kwargs={'result_dict': check_validity_task.output},
    dag=dag
)


import setuptools
from setuptools import setup

with open("README.md", "r", encoding="utf-8") as fh:
    long_description = fh.read()

setup(
    name="PartisanAssociations", # Replace with your own username
    version="0.0.1",
    author="Patrick Y. Wu, Walter R. Mebane, Jr., Logan Woods, Joseph Klaver, and Preston Due",
    author_email="pywu@umich.edu",
    description="Partisan associations using Twitter user bios",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/patrickywu/PartisanAssociations",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.6',
)


import datetime
import threading

import cv2
import mediapipe as mp
import pika
import screeninfo

class NewsConsumer:
    """
    This class implements a RabbitMQ consumer.
    """
    articles = []
    titles = 'There are no news to display...'

    def __init__(self, host):
        """
        This method initializes the RabbitMQ consumer.
        """
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(host))
        self.channel = self.connection.channel()
        self.channel.queue_declare(queue='news_stream')
        self.channel.queue_declare(queue='news_titles')

    def consume(self):
        """
        This method starts consuming messages from the RabbitMQ queue.
        """
        self.channel.basic_consume(queue='news_stream', on_message_callback=self.add_new_article)
        self.channel.basic_consume(queue='news_titles', on_message_callback=self.update_titles)
        self.channel.start_consuming()

    def add_new_article(self, ch, method, properties, body):
        """
        This method adds a new article to the list of articles.
        """
        new_article = body.decode('utf-8')
        # concatenate the new article to the text
        NewsConsumer.articles.append(new_article)

    def update_titles(self, ch, method, properties, body):
        """
        This method updates the titles of the news.
        """
        global titles
        titles = body.decode('utf-8')
        print(
            f"Received titles: {titles} at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
        )
        self.titles += titles
        ch.basic_ack(delivery_tag=method.delivery_tag)

    def close(self):
        """
        This method closes the connection to the RabbitMQ server.
        """
        if self.connection.is_open:
            self.connection.close()

def start_consumer():
    """
    This method starts the RabbitMQ consumer.
    """
    consumer = NewsConsumer('localhost')
    try:
        consumer.consume()
    except KeyboardInterrupt:
        consumer.close()

thread = threading.Thread(target=start_consumer)
thread.start()

camera_id = 0

def start_streaming():

    """
    This method starts the video streaming.
    """

    # Using OpenCV to display the image
    cap = cv2.VideoCapture(camera_id)

    w = screeninfo.get_monitors()[0].width
    h = screeninfo.get_monitors()[0].height

    cap.set(cv2.CAP_PROP_FRAME_WIDTH, w)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, h)
    cap.set(cv2.CAP_PROP_FPS, 60)

    # mp_drawing = mp.solutions.drawing_utils
    # mp_drawing_styles = mp.solutions.drawing_styles
    mp_hands = mp.solutions.hands

    zoomed_text_color = (0, 255, 0)
    standard_text_color = (255, 255, 255)
    color = standard_text_color

    direction = 0
    font_size = 12
    box_height = 50
    # box width is the same as the image width
    box_width = 1280

    # is_queue_requested = False

    with mp_hands.Hands(
            model_complexity=0,
            min_detection_confidence=0.6,
            min_tracking_confidence=0.5) as hands:

        while cap.isOpened():

            success, image = cap.read()

            if not success:
                print("Ignoring empty camera frame.")
                # If loading a video, use 'break' instead of 'continue'.
                continue

            image = cv2.flip(image, 1)

            # To improve performance, optionally mark the image as not writeable to
            # pass by reference.
            image.flags.writeable = False
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            results = hands.process(image)

            # # Draw the hand annotations on the image.
            image.flags.writeable = True
            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

            # create a background for the text with a border
            cv2.rectangle(image, (0, image.shape[0] - box_height), (image.shape[1], image.shape[0]), (0, 0, 0), -1)
            # border
            cv2.rectangle(image, (0, image.shape[0] - box_height), (image.shape[1], image.shape[0]), (255, 255, 255), 2)

            if results.multi_hand_landmarks:
                for hand_landmarks in results.multi_hand_landmarks:

                    # if left hand is raised then move the text to the left
                    if hand_landmarks.landmark[mp_hands.HandLandmark.WRIST].x < 0.2 and len(results.multi_hand_landmarks) == 1:
                        NewsConsumer.titles = NewsConsumer.titles[1:] + NewsConsumer.titles[0]
                        direction = 0
                        font_size = 12
                        color = standard_text_color
                        box_height = 50

                    # if right hand is raised then move the text to the right
                    elif hand_landmarks.landmark[mp_hands.HandLandmark.WRIST].x > 0.8 and len(results.multi_hand_landmarks) == 1:
                        direction = 1
                        font_size = 12
                        color = standard_text_color
                        box_height = 50

                    # if both hands are raised then increase the font size to 36pt and change the color
                    elif 0.2 < hand_landmarks.landmark[mp_hands.HandLandmark.WRIST].x < 0.8 and len(results.multi_hand_landmarks) == 2:
                        font_size = 36
                        color = zoomed_text_color
                        box_height = 100

            else:
                # if no hands are detected then move the text to the left
                font_size = 12
                color = standard_text_color
                box_height = 50

            if direction == 0:
                NewsConsumer.titles = NewsConsumer.titles[1:] + NewsConsumer.titles[0]
            elif direction == 1:
                NewsConsumer.titles = NewsConsumer.titles[-1] + NewsConsumer.titles[:-1]

            # draw the text in the bottom left corner with a 2px border in black color. Get the font size from the variable font_size
            cv2.putText(image, NewsConsumer.titles, (10, image.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, font_size / 12, color, 2, cv2.LINE_AA)

            # draw another text at the right top corner to show the current time of the day in 24h format (HH:MM:SS) and in white color with a 1px border in black color. Change to font-size to 14pt with bold style
            cv2.putText(image, datetime.datetime.now().strftime("%H:%M:%S"), (image.shape[1] - 100, 20), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)

            # put a logo in the top left corner
            logo = cv2.imread('news/logo.png')

            # resize the logo
            # match the size of the logo and time in the top right corner
            logo = cv2.resize(logo, (50, 50))

            # add the logo to the image
            image[0:logo.shape[0], 0:logo.shape[1]] = logo

            # Flip the image horizontally for a selfie-view display.
            cv2.imshow('AI News', image)

            # wait for the 'q' key to be pressed
            if cv2.waitKey(1) & 0xFF == ord('q') or cv2.waitKey(1) & 0xFF == ord('Q'):
                break

            # if 's' is pressed, save the image
            if cv2.waitKey(1) & 0xFF == ord('s') or cv2.waitKey(1) & 0xFF == ord('S'):
                cv2.imwrite('news.jpg', image)

    cap.release()

    # stop the video
    cv2.destroyAllWindows()

    # stop the consumer thread
    thread.join()

def main():
    """
    Main function
    """
    start_streaming()

if __name__ == '__main__':
    main()


### TASK ###
# Write a program that queries Covalent's API ticker endpoint.
# Collect null contract address results for 60 samples at a rate of 1 sample per minute.
# After 60 samples, display a histogram of null results. Only include currencies that have >0 null address responses.

### AUTHOR ###
# Kristopher Buote
# September 26, 2021

### Use python's requests library to query API
import requests
import time
import matplotlib.pyplot as plt
import numpy as np
import csv
import sys

### SET UP ###
# Test connection to the Covalent API with error handling.
# Organize the response data into dictionaries.
# Create a dictionary to store the contract addresses from the API response.
# This program uses the Ticker to associate with sample responses.

### Receive API key as user input
mykey = input("Enter Covalent API Key:")

### Try an initial connection to the API
try:
    response = requests.get('https://api.covalenthq.com/v1/pricing/tickers/',
                            auth=(mykey, ''))

# Exit program if initial connection failed.
except:
    print(f"Connection error. Program exited. Error:{sys.exc_info()[0]}")
    exit()

### Collect the API's response error and message
json_response = response.json()
error = json_response["error"]
error_message = json_response["error_message"]

### Check if response was good (i.e. 200 response code and error is false).
# If good, continue. Else, print error code and message.
if response.status_code == 200 and error is False:
    print(f"API Response: {response.status_code}. No error. Program continuing.")

    ### Organize API response JSON data
    data = json_response["data"]
    timestamp = data["updated_at"]
    items = data["items"]

    ### Create a dictionary using the currency Ticker symbols to store results
    # *** This program assumes there will be no errors returned by the contract_ticker_symbol. It also assumes the same ticker symbols will be returned every sample***
    results = {}

    # Collect the ticker symbols and store them in results
    for item in items:
        ticker_symbol = item["contract_ticker_symbol"]
        results[ticker_symbol] = {}

else:
    print(f"API Response Code: {response.status_code} \nAPI Error Message: {error_message}\nProgram failed and exited.")
    exit()

### DATA COLLECTION ###
# Collect 60 samples, approximately 1 sample per minute
# The results are stored in a nested dictionary using the Ticker and sample count as keys.
# e.g. {'Ticker1': {1: 'abcd', 2: null, ..., 60: 'abcd'}, 'Ticker2': {1: 'efg', 2: 'efg', ..., 60: null}}

for sample in range(1, 61):

    # Query API
    response = requests.get('https://api.covalenthq.com/v1/pricing/tickers/',
                            auth=(mykey, ''))

    json_response = response.json()
    error = json_response["error"]
    error_message = json_response["error_message"]

    # Organize API response JSON data
    data = json_response["data"]
    timestamp = data["updated_at"]
    all_items = data["items"]

    ### Check if an error was thrown by the API response
    # If error message is false, continue with program. Else, skip sample.
    if error is False:

        # Collect results in a nested dictionary using the Ticker symbol as key
        # A new entry will be added for each sample. If an address is returned, it will be stored. If null is returned, it will be stored as null.
        for item in all_items:
            item_ticker = item["contract_ticker_symbol"]
            item_address = item["contract_address"]
            results[item_ticker][sample] = item_address

        print(f"Sample {sample} collected.")

    else:
        print(f"Sample {sample} failed and skipped. API error message: {error_message}")

    # Wait 1 minute for another sample. This is usually commented out for developing.
    time.sleep(60)

### DATA ANALYSIS ###
# Count the amount of null responses for each ticker across the 60 samples
# Note that None is the variable for null in python

# Create a new dictionary to store the null count
ticker_null_count = {}

# Count the number of null address responses for each ticker
for ticker, samples in results.items():
    null_count = sum(value == None for value in samples.values())
    ticker_null_count[ticker] = null_count

# Exclude the tickers with a zero null count
ticker_null_count_non_zero = {key: value for key, value in ticker_null_count.items() if value != 0}

# Sort the dictionary in ascending order
ticker_null_count_ordered = {key: value for key, value in sorted(ticker_null_count_non_zero.items(), key=lambda item: item[1])}

### DATA PRESENTATION AND SAVING ###
### Save the null count data to a csv file
with open('null_count.csv', 'w') as csv_file:
    writer = csv.writer(csv_file)
    for key, value in ticker_null_count_non_zero.items():
       writer.writerow([key, value])

### Create a histogram for the null counts in ascending order. Save the figure as a png file.
plt.bar(ticker_null_count_ordered.keys(), ticker_null_count_ordered.values(), width=1.0, color='g')
plt.title("Null Address Count over 60 samples")
plt.xlabel("Currency Ticker")
plt.ylabel("Null Address Count")

# Save the figure
plt.savefig('null_count_histogram.png')
plt.show()

print("Program complete.")


class Mobile:
    fp = "Yes"              #Class variable

    def __init__(self) -> None:
        self.model = "Realme X"         # Instance variable

    def show_model(self):               # Instance method
        print("Model :",self.model)

    @classmethod
    def is_fp(cls):                     # Class method
        print("finger Print :",cls.fp)

realme = Mobile()
redme = Mobile()
realme.show_model()
realme.model = "Realme 7s"
realme.show_model()
redme.show_model()

Mobile.is_fp()
Mobile.fp = "NO"
Mobile.is_fp()


import pandas as pd
import os
import numpy as np

#----------------------------- q
path_image_MIMIC = "/scratch/gobi2/projects/ml4h/projects/mimic_access_required/MIMIC-CXR/"

train_df_path_MIMIC ="/scratch/gobi2/projects/ml4h/datasets/new_split/8-1-1/new_train.csv"
test_df_path_MIMIC ="/scratch/gobi2/projects/ml4h/datasets/new_split/8-1-1/new_test.csv"
val_df_path_MIMIC = "/scratch/gobi2/projects/ml4h/datasets/new_split/8-1-1/new_valid.csv"

path_image_NIH = "/scratch/gobi2/projects/ml4h/datasets/NIH/images/"

train_df_path_NIH ="/scratch/gobi2/projects/ml4h/datasets/NIH/split/July16/train.csv"
test_df_path_NIH = "/scratch/gobi2/projects/ml4h/datasets/NIH/split/July16/test.csv"
val_df_path_NIH = "/scratch/gobi2/projects/ml4h/datasets/NIH/split/July16/valid.csv"

path_image_CXP = "/scratch/gobi2/projects/ml4h/datasets/CheXpert/"

train_df_path_CXP ="/scratch/gobi2/projects/ml4h/datasets/CheXpert/split/July19/new_train.csv"
test_df_path_CXP ="/scratch/gobi2/projects/ml4h/datasets/CheXpert/split/July19/new_test.csv"
val_df_path_CXP = "/scratch/gobi2/projects/ml4h/datasets/CheXpert/split/July19/new_valid.csv"

val_df_NIH = pd.read_csv(val_df_path_NIH)
val_df_CXP = pd.read_csv(val_df_path_CXP)
val_df_MIMIC = pd.read_csv(val_df_path_MIMIC)

test_df_NIH = pd.read_csv(test_df_path_NIH)
test_df_CXP = pd.read_csv(test_df_path_CXP)
test_df_MIMIC = pd.read_csv(test_df_path_MIMIC)

train_df_NIH = pd.read_csv(train_df_path_NIH)
train_df_CXP = pd.read_csv(train_df_path_CXP)
train_df_MIMIC = pd.read_csv(train_df_path_MIMIC)

def preprocess_MIMIC(split):

    details = pd.read_csv("/scratch/gobi2/projects/ml4h/datasets/new_split/mimic-cxr-metadata-detail.csv")
    details = details.drop(columns=['dicom_id', 'study_id', 'religion', 'race', 'insurance', 'marital_status', 'gender'])
    details.drop_duplicates(subset="subject_id", keep="first", inplace=True)
    df = pd.merge(split, details)

    copy_sunbjectid = df['subject_id']
    df.drop(columns = ['subject_id'])

    df = df.replace(
            [[None], -1, "[False]", "[True]", "[ True]", 'UNABLE TO OBTAIN', 'UNKNOWN', 'MARRIED', 'LIFE PARTNER',
             'DIVORCED', 'SEPARATED', '0-10', '10-20', '20-30', '30-40', '40-50', '50-60', '60-70', '70-80', '80-90',
             '>=90'],
            [0, 0, 0, 1, 1, 0, 0, 'MARRIED/LIFE PARTNER', 'MARRIED/LIFE PARTNER', 'DIVORCED/SEPARATED',
             'DIVORCED/SEPARATED', '0-20', '0-20', '20-40', '20-40', '40-60', '40-60', '60-80', '60-80', '80-', '80-'])

    df['subject_id'] = copy_sunbjectid
    df['Age'] = df["age_decile"]
    df['Sex'] = df["gender"]
    df = df.drop(columns=["age_decile", 'gender'])

    return df

def preprocess_NIH(split):
    split['Patient Age'] = np.where(split['Patient Age'].between(0,19), 19, split['Patient Age'])
    split['Patient Age'] = np.where(split['Patient Age'].between(20,39), 39, split['Patient Age'])
    split['Patient Age'] = np.where(split['Patient Age'].between(40,59), 59, split['Patient Age'])
    split['Patient Age'] = np.where(split['Patient Age'].between(60,79), 79, split['Patient Age'])
    split['Patient Age'] = np.where(split['Patient Age']>=80, 81, split['Patient Age'])

    copy_sunbjectid = split['Patient ID']
    split.drop(columns = ['Patient ID'])

    split = split.replace([[None], -1, "[False]", "[True]", "[ True]", 19, 39, 59, 79, 81],
                            [0, 0, 0, 1, 1, "0-20", "20-40", "40-60", "60-80", "80-"])

    split['subject_id'] = copy_sunbjectid
    split['Sex'] = split['Patient Gender']
    split['Age'] = split['Patient Age']
    split = split.drop(columns=["Patient Gender", 'Patient Age'])

    return split

def preprocess_CXP(split):
    split['Age'] = np.where(split['Age'].between(0,19), 19, split['Age'])
    split['Age'] = np.where(split['Age'].between(20,39), 39, split['Age'])
    split['Age'] = np.where(split['Age'].between(40,59), 59, split['Age'])
    split['Age'] = np.where(split['Age'].between(60,79), 79, split['Age'])
    split['Age'] = np.where(split['Age']>=80, 81, split['Age'])

    copy_sunbjectid = split['subject_id']
    split.drop(columns = ['subject_id'])
    split = split.replace([[None], -1, "[False]", "[True]", "[ True]", 19, 39, 59, 79, 81],
                            [0, 0, 0, 1, 1, "0-20", "20-40", "40-60", "60-80", "80-"])

    split['subject_id'] = copy_sunbjectid
    split['Sex'] = np.where(split['Sex']=='Female', 'F', split['Sex'])
    split['Sex'] = np.where(split['Sex']=='Male', 'M', split['Sex'])

    return split

test_df_MIMIC = preprocess_MIMIC(test_df_MIMIC)
test_df_CXP = preprocess_CXP(test_df_CXP)
test_df_NIH = preprocess_NIH(test_df_NIH)

train_df_MIMIC = preprocess_MIMIC(train_df_MIMIC)
train_df_CXP = preprocess_CXP(train_df_CXP)
train_df_NIH = preprocess_NIH(train_df_NIH)

val_df_MIMIC = preprocess_MIMIC(val_df_MIMIC)
val_df_CXP = preprocess_CXP(val_df_CXP)
val_df_NIH = preprocess_NIH(val_df_NIH)

test_df_MIMIC['Jointpath'] = path_image_MIMIC + test_df_MIMIC['path'].astype(str)
test_df_CXP['Jointpath']   = path_image_CXP   + test_df_CXP['Path'].astype(str)
test_df_NIH['Jointpath']   = path_image_NIH   + test_df_NIH['Image Index'].astype(str)

train_df_MIMIC['Jointpath'] = path_image_MIMIC + train_df_MIMIC['path'].astype(str)
train_df_CXP['Jointpath']   = path_image_CXP   + train_df_CXP['Path'].astype(str)
train_df_NIH['Jointpath']   = path_image_NIH   + train_df_NIH['Image Index'].astype(str)

val_df_MIMIC['Jointpath'] = path_image_MIMIC + val_df_MIMIC['path'].astype(str)
val_df_CXP['Jointpath']   = path_image_CXP   + val_df_CXP['Path'].astype(str)
val_df_NIH['Jointpath']   = path_image_NIH   + val_df_NIH['Image Index'].astype(str)

# val_df_NIH["subject_id"] = "NIH-" + val_df_NIH['Patient ID'].astype(str)
# val_df_CXP["subject_id"] = "CXP-" + val_df_CXP['subject_id'].astype(str)
# val_df_MIMIC["subject_id"] = "CXR-" + val_df_MIMIC['subject_id'].astype(str)

# test_df_NIH["subject_id"] = "NIH-" + test_df_NIH['Patient ID'].astype(str)
# test_df_CXP["subject_id"] = "CXP-" + test_df_CXP['subject_id'].astype(str)
# test_df_MIMIC["subject_id"] = "CXR-" + test_df_MIMIC['subject_id'].astype(str)

# train_df_NIH["subject_id"] = "NIH-" + train_df_NIH['Patient ID'].astype(str)
# train_df_CXP["subject_id"] = "CXP-" + train_df_CXP['subject_id'].astype(str)
# train_df_MIMIC["subject_id"] = "CXR-" + train_df_MIMIC['subject_id'].astype(str)

val_df_NIH["subject_id"] =  val_df_NIH['subject_id'].astype(int)
val_df_CXP["subject_id"] =  val_df_CXP['subject_id'].astype(int)
val_df_MIMIC["subject_id"] = val_df_MIMIC['subject_id'].astype(int)

test_df_NIH["subject_id"] =  test_df_NIH['subject_id'].astype(int)
test_df_CXP["subject_id"] = test_df_CXP['subject_id'].astype(int)
test_df_MIMIC["subject_id"] =  test_df_MIMIC['subject_id'].astype(int)

train_df_NIH["subject_id"] = train_df_NIH['subject_id'].astype(int)
train_df_CXP["subject_id"] =  train_df_CXP['subject_id'].astype(int)
train_df_MIMIC["subject_id"] =  train_df_MIMIC['subject_id'].astype(int)

val_df_CXP["Effusion"] = val_df_CXP["Pleural Effusion"]
val_df_MIMIC["Effusion"] = val_df_MIMIC["Pleural Effusion"]
val_df_CXP['Airspace Opacity'] = val_df_CXP['Lung Opacity']

train_df_CXP["Effusion"] = train_df_CXP["Pleural Effusion"]
train_df_MIMIC["Effusion"] = train_df_MIMIC["Pleural Effusion"]
train_df_CXP['Airspace Opacity'] = train_df_CXP['Lung Opacity']

test_df_CXP["Effusion"] = test_df_CXP["Pleural Effusion"]
test_df_MIMIC["Effusion"] = test_df_MIMIC["Pleural Effusion"]
test_df_CXP['Airspace Opacity'] = test_df_CXP['Lung Opacity']

val_df_CXP_new = val_df_CXP[['subject_id','Jointpath','Sex',"Age",'No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]
train_df_CXP_new = train_df_CXP[['subject_id','Jointpath','Sex',"Age",'No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]

val_df_NIH_new = val_df_NIH[['subject_id','Jointpath','Sex',"Age",'No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]
train_df_NIH_new = train_df_NIH[['subject_id','Jointpath','Sex',"Age",'No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]

val_df_MIMIC_new = val_df_MIMIC[['subject_id','Jointpath','Sex',"Age",'No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]
train_df_MIMIC_new = train_df_MIMIC[['subject_id','Jointpath','Sex',"Age",'No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]

test_df_CXP_new = test_df_CXP[['subject_id','Jointpath','Sex','Age','No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]
test_df_NIH_new = test_df_NIH[['subject_id','Jointpath','Sex','Age','No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]
test_df_MIMIC_new = test_df_MIMIC[['subject_id','Jointpath','Sex','Age','No Finding', 'Atelectasis', 'Cardiomegaly',  'Effusion',  'Pneumonia', 'Pneumothorax', 'Consolidation','Edema' ]]

test_df = test_df_CXP_new.append([test_df_NIH_new, test_df_MIMIC_new])
train_df = train_df_CXP_new.append([train_df_NIH_new, train_df_MIMIC_new])
val_df = val_df_CXP_new.append([val_df_NIH_new, val_df_MIMIC_new])


import json
import hashlib
import unittest
from bin.audit_log_entry import AuditLogEntry

class TestAuditLogEntry(unittest.TestCase):

    def test_no_document_id(self):
        mock_entry = {
            "@timestamp":"1614697638660","org":"org-demo","business":"poizen-inc","repo":"org-demo/public-repo","action":"git.fetch","transport_protocol_name":"http","transport_protocol":1,"repository":"org-demo/public-repo","repository_public":"true"
        }
        expected_output = "61ff3c1a135b3c6cac7ccabe2c6fe209"
        entry = AuditLogEntry(**mock_entry)
        self.assertEqual(entry.document_id, expected_output)

    def test_document_id(self):
        mock_entry = {
            "@timestamp":"1614697638660","_document_id":"wSuRpMciieZn4qkaR4YUtg","business":"poizen-inc","org":"org-demo","repo":"org-demo/public-repo","action":"git.fetch","transport_protocol_name":"http","transport_protocol":1,"repository":"org-demo/public-repo","repository_public":"true"
        }
        expected_output = "wSuRpMciieZn4qkaR4YUtg"
        entry = AuditLogEntry(**mock_entry)
        self.assertEqual(entry.document_id, expected_output)


import re
from typing import Final

def _create_simple_pattern() -> re.Pattern[str]:
    NO_CLOSE_PAREN_CHAR = r'[^)]'
    return re.compile(fr"""
        [^a-zA-Z]       # Some non alphabetic character (prevent format! false positives)
        t!\("           # t!("
        ([^"]+)         # the translation key as a captured group
        "               # "
        (?:,
            {NO_CLOSE_PAREN_CHAR}+
        )?               # optional non-capturing group with a comma and several characters more, excepting close parens.
                \)              # )
    """, re.VERBOSE)

simple_pattern: Final = _create_simple_pattern()

def test_match_simple_pattern_1() -> None:
    KEY = "una_key"
    m = simple_pattern.search(f'let s = t!("{KEY}")')
    assert m
    assert len(m.groups()) == 1
    assert m.groups()[0] == KEY

def test_match_simple_pattern_2() -> None:
    KEY = "una_key_con_arg_${name}_"
    m = simple_pattern.search(f'let s = t!("{KEY}", name=name)')
    assert m
    assert len(m.groups()) == 1
    assert m.groups()[0] == KEY


import os
from argparse import ArgumentParser

intro = '''# Use useful_python_programs/scars_thingy.py for ease of usage.

# TODO: (use CTRL + F to jump between these)
# 1) TITLE
# 2) INITIALS
# 3) DESCRIPTION

################
### 1) TITLE ###
################

defined_text = {
	name = Statelore_Get_Name'''
title_block = '''
	text = {
		trigger = {
			check_variable = { selected_lore = 1 }
		}
		localization_key = state_lore_name.1
	}'''
initial = '''
}

###################
### 2) INITIALS ###
###################

defined_text = {
	name = Statelore_Get_Initials'''
initial_block = '''
	text = {
		trigger = {
			check_variable = { lore_entries_onscreen^sl_i = 1 }
		}
		localization_key = state_lore_initials.1
	}'''
description = '''
}

######################
### 3) DESCRIPTION ###
######################
defined_text = {
	name = Statelore_Get_Desc'''
description_block = '''
	text = {
		trigger = {
			check_variable = { selected_lore = 1 }
		}
		localization_key = state_lore_desc.1
	}'''

end = '''
}
'''

if __name__ == "__main__":
	parser = ArgumentParser()
	parser.add_argument('number', metavar='number', type=int, help='''The number of blocks you want to do.''')
	args = parser.parse_args()
	number = args.number
	text = intro
	for i in range(1, number+1):
		text += title_block.replace('1', str(i))
	text += initial
	for i in range(1, number+1):
		text += initial_block.replace('1', str(i))
	text += description
	for i in range(1, number+1):
		text += description_block.replace('1', str(i))
	text += end

	output = open(os.path.join('..', 'common', 'scripted_localisation', 'TNO_state_lore_scripted_localisation.txt'), 'w+')
	output.write(text)


from datetime import datetime, timedelta
from .models import Allocation, AllocationDetail, Employee, Project
from django.db.models import Q

class TimeLineCalendar:
    class Booking:
        allocation_id = 0
        project = ''
        allocation_type = ''
        day_of_week = 0
        start_date = datetime.now()
        end_date = datetime.now()
        no_of_hours = 0
        color_code = ''

    def __init__(self, start_date=None, end_date=None):
        self.start_date = start_date
        self.end_date = end_date

    @staticmethod
    def get_month_header(start_date, end_date):
        delta = timedelta(days=1)
        month_td = ''
        current_month = start_date.strftime("%b")
        month_col_span = 0

        while start_date <= end_date:
            if current_month == start_date.strftime("%b"):
                month_col_span += 1
            else:
                month_td += '{}	'\
                    .format(month_col_span, current_month + "/" + start_date.strftime('%y'))
                current_month = start_date.strftime("%b")
                month_col_span = 1

            start_date += delta

        # loop ends once look reaches end_date hence not getting opportunity to add last month name. Due to that
        # added this manual entry in tail end of the loop
        month_td += '{}	'.format(
            month_col_span, current_month + "/" + start_date.strftime('%y'))

        return month_td

    @staticmethod
    def get_week_header(start_date, end_date):
        delta = timedelta(days=1)
        week_td = ''
        current_week = start_date.strftime("%W")
        week_col_span = 0

        while start_date <= end_date:
            if current_week == start_date.strftime("%W"):
                week_col_span += 1
            else:
                wc_date = start_date - delta * 2
                week_commencing = wc_date - timedelta(days=wc_date.weekday() % 7)

                week_td += '{}	'.format(
                    week_col_span, week_commencing.strftime("%d-%m"))
                current_week = start_date.strftime("%W")
                week_col_span = 1

            start_date += delta

        wc_date = start_date - delta * 2
        week_commencing = wc_date - timedelta(days=wc_date.weekday() % 7)
        # loop ends once look reaches end_date hence not getting opportunity to add last month name. Due to that
        # added this manual entry in tail end of the loop
        week_td += '{}	'.format(
            week_col_span, week_commencing.strftime("%d-%m"))

        return week_td

    @staticmethod
    def get_week_header_v2(start_date, end_date):
        delta = timedelta(days=1)
        week_td = ''
        current_week = start_date.strftime("%W")

        while start_date < end_date:
            if current_week != start_date.strftime("%W"):
                wc_date = start_date - delta * 2
                week_commencing = wc_date - timedelta(days=wc_date.weekday() % 7)

                week_td += '{}	'.format(
                    week_commencing.strftime("%d-%m"))
                current_week = start_date.strftime("%W")
            start_date += delta

        wc_date = start_date - delta * 2
        week_commencing = wc_date - timedelta(days=wc_date.weekday() % 7)
        # loop ends once look reaches end_date hence not getting opportunity to add last month name. Due to that
        # added this manual entry in tail end of the loop
        week_td += '{}	'.format(
            week_commencing.strftime("%d-%m"))

        return week_td

    @staticmethod
    def get_date_range_header(start_date, end_date):
        delta = timedelta(days=1)
        date_td = ''

        while start_date <= end_date:
            date_td += '{}	'.format(
                start_date.strftime("%a")[0])
            start_date += delta

        return date_td

    @staticmethod
    def is_weekend(date_to_check):
        if date_to_check.strftime("%a") == "Sat" or \
                date_to_check.strftime("%a") == "Sun":
            return True
        else:
            return False

    @staticmethod
    def get_week_commencing_date(date_to_check):
        delta = timedelta(days=1)
        wc_date = date_to_check - delta * 2
        return wc_date - timedelta(days=wc_date.weekday() % 7)

    def is_bank_holiday(self, date_to_check):
        bank_holidays = self.get_bank_holidays()

        if date_to_check.strftime("%d-%m-%Y") in bank_holidays:
            return True
        else:
            return False

    @staticmethod
    def get_bank_holidays():
        # TODO: Replace constant values with database
        bank_holidays = {}

        bank_holidays['21-04-2023'] = 'Eid-ul-Fitr'
        bank_holidays['15-08-2023'] = 'Independence Day'
        bank_holidays['30-08-2023'] = 'Rakshabandhan'
        bank_holidays['24-10-2023'] = 'Dusshera'
        bank_holidays['13-11-2023'] = 'New Year'
        bank_holidays['14-11-2023'] = 'Bhai Duj'
        bank_holidays['25-12-2023'] = 'Christmas'
        bank_holidays['15-01-2024'] = 'Vasi Uttrayan'
        bank_holidays['26-01-2024'] = 'Republic Day'
        bank_holidays['25-03-2024'] = 'Dhuleti'

        return bank_holidays

    def get_employee_allocation_detail(self, start_date, end_date, employee):

        query = Q(Q(employee=employee) & Q(end_date__gte=start_date) & Q(start_date__lte=end_date))

        allocations = Allocation.objects.filter(query)

        bookings = []
        for allocation in allocations:
            for detail in AllocationDetail.objects.filter(allocation=allocation):
                b = self.Booking()
                b.allocation_id = allocation.id
                b.project = allocation.project.name
                b.allocation_type = allocation.allocation_type.name
                b.day_of_week = detail.day_of_week
                b.start_date = allocation.start_date
                b.end_date = allocation.end_date
                b.no_of_hours = allocation.no_of_hours
                b.color_code = allocation.allocation_type.color_code
                bookings.append(b)

        return bookings

    def get_blank_tds(self, start_date, end_date):
        delta = timedelta(days=1)
        tr = ''
        while start_date <= end_date:
            if self.is_weekend(start_date):
                tr += '	'
            elif self.is_bank_holiday(start_date):
                tr += '	'
            else:
                tr += '	'

            start_date += delta
        return tr

    def get_employee_allocation_tr(self, start_date, end_date, allocation):
        delta = timedelta(days=1)
        timeline_tr = ''
        total_allocation_hours = 0
        total_hours_between_dates = 0

        allocation_detail = AllocationDetail.objects.filter(allocation=allocation)

        while start_date <= end_date:
            allocation_data = ''

            for detail in allocation_detail:
                if start_date.weekday() == detail.day_of_week and \
                        start_date >= allocation.start_date and \
                        start_date <= allocation.end_date:

                    class_name = ''
                    if self.is_bank_holiday(start_date):
                        class_name = 'bank-holiday'

                    allocation_data = '	'\
                        .format(allocation.allocation_type.color_code, class_name, allocation.id)

                    total_allocation_hours += allocation.no_of_hours
                    break
                else:
                    if self.is_weekend(start_date):
                        allocation_data = '	'
                    elif self.is_bank_holiday(start_date):
                        allocation_data = '	'
                    else:
                        allocation_data = '	'

            if not self.is_weekend(start_date):
                total_hours_between_dates += 8

            timeline_tr += allocation_data
            start_date += delta

        return timeline_tr

    def gettimelinecalendar(self):

        empty_tds = self.get_blank_tds(self.start_date, self.end_date)

        cal = '
'

        cal += '
'

        cal += '
'
        # Creating space for Name, Job Title, Date Of Joining, Skills, Date, etc... columns
        # cal += '		'
        cal += '{}	'\
            .format('Name')
        cal += '{}	'\
            .format('Job Title')
        cal += self.get_week_header(self.start_date, self.end_date)
        cal += '
'

        cal += '
'
        cal += self.get_date_range_header(self.start_date, self.end_date)
        cal += '
'

        cal += '
'

        cal += '
'

        timeline_tr = ''
        row_span = 0

        days_difference = (self.end_date-self.start_date)
        row_span = days_difference.days + 3

        flex_squad = '
'
        flex_squad += '{}	'.format(row_span, 'Flex Suad')
        flex_squad += '
'
        # Fetch employees who are not allocated in selected time frame and add them in the Flex Squad project
        emps = [allocation.employee.id for allocation in Allocation.objects.filter(start_date__lte=self.end_date,
                                                                                end_date__gte=self.start_date)]

        for emp in Employee.objects.all().exclude(id__in=emps):
            flex_squad += '
'
            flex_squad += '{}	'.format(emp.first_name)
            flex_squad += '{}	'.format(emp.job_title.name)
            flex_squad += empty_tds
            flex_squad += '
'

        cal += flex_squad

        for project in Project.objects.filter(start_date__lte=self.end_date, end_date__gte=self.start_date)\
                .order_by('name'):
            timeline_tr = '
'
            timeline_tr += '{}- Project Type:{}, Start date:{}, ' \
                           'End date:{}, Commercial Status: {}	'.format(
                            row_span, project.name, project.project_type.name,
                            project.start_date, project.end_date, project.commercial_status)

            timeline_tr += '
'

            for allocation in Allocation.objects.filter(project=project, start_date__lte=self.end_date,
                                                        end_date__gte=self.start_date).order_by('employee__first_name'):
                timeline_tr += '
'
                timeline_tr += "{}	".format(allocation.employee.first_name)
                timeline_tr += "{}	".format(allocation.employee.job_title.name)
                timeline_tr += self.get_employee_allocation_tr(self.start_date, self.end_date, allocation)
                timeline_tr += '
'

            cal += timeline_tr

        cal += '
'
        cal += '
'

        return cal

    def capacitycalendar(self):

        cal = '
'
        cal += '
'

        cal += '
'
        cal += '{}	' \
            .format('Name')
        cal += '{}	' \
            .format('Job Title')
        cal += '{}	' \
            .format('Skills')
        cal += self.get_week_header_v2(self.get_week_commencing_date(self.start_date), self.get_week_commencing_date(self.end_date))
        cal += '
'
        cal += '
'

        cal += '
'

        delta = timedelta(days=1)
        start_date = self.get_week_commencing_date(self.start_date)
        end_date = self.get_week_commencing_date(self.end_date)

        for employee in Employee.objects.all():

            allocation_detail = self.get_employee_allocation_detail(self.get_week_commencing_date(self.start_date),
                                                                    self.get_week_commencing_date(self.end_date),
                                                                    employee)
            total_allocation_hours = 0
            employee_tr = '
'
            employee_tr += '{}	'.format(employee.first_name)
            employee_tr += '{}	'.format(employee.job_title.name)
            employee_tr += '{}	'.format(','.join([s.name for s in employee.skills.all()]))

            start_date = self.get_week_commencing_date(self.start_date)
            end_date = self.get_week_commencing_date(self.end_date)
            current_week = self.start_date.strftime("%W")
            while start_date <= end_date:

                for detail in allocation_detail:
                    if start_date.weekday() == detail.day_of_week and \
                            start_date >= detail.start_date and \
                            start_date <= detail.end_date:

                        total_allocation_hours += detail.no_of_hours

                if start_date.strftime("%a") == 'Sun':
                    allocation_percentage = (total_allocation_hours / 40)

                    # progressbar = '
'
                    # progressbar += '
'.format(allocation_percentage)
                    # progressbar += '{:.0%}
'.format(allocation_percentage)
                    # progressbar += '
'

                    current_week = start_date.strftime("%W")

                    # employee_tr += "{}	".format(progressbar)
                    employee_tr += "{} %	".format(self.get_heatmap_color(allocation_percentage*100), allocation_percentage*100)
                    total_allocation_hours = 0

                start_date += delta

            employee_tr += "
"
            cal += employee_tr

        cal += '
'
        cal += '
'

        return cal

    @staticmethod
    def get_heatmap_color(value):
        heatmap_color = ''

        if value>=0 and value<=10:
            heatmap_color='#cc0000'
        elif value>=11 and value<=20:
            heatmap_color='#ff0000'
        elif value>=21 and value<=30:
            heatmap_color = '#ff6600'
        elif value>=31 and value<=40:
            heatmap_color = '#ff9900'
        elif value>=41 and value<=50:
            heatmap_color = '#ffc000'
        elif value>=51 and value<=60:
            heatmap_color = '#ffd966'
        elif value>=61 and value<=70:
            heatmap_color = '#ffe699'
        elif value>=71 and value<=80:
            heatmap_color = '#c6e0b4'
        elif value>=81 and value<=90:
            heatmap_color = '#a9d08e'
        elif value>=91 and value<=100:
            heatmap_color = '#339966'
        elif value>100:
            heatmap_color = '#1654b0'

        return heatmap_color


from .base import PipBaseRecipe

class LxmlRecipe(PipBaseRecipe):
    def __init__(self, *args, **kwargs):
        super(LxmlRecipe, self).__init__(*args, **kwargs)
        self.sha256 = '736f72be15caad8116891eb6aa4a078b' \
                      '590d231fdc63818c40c21624ac71db96'
        self.name = 'lxml'
        self.version = '3.8.0'  # < 4 for apache-airflow


from unittest import mock

import pytest
from shipyard_airflow.plugins import concurrency_check_operator as operator
from shipyard_airflow.plugins.concurrency_check_operator import (
    ConcurrencyCheckOperator
)
from airflow.exceptions import AirflowException

COLOR_SETS = [
    set(['blue', 'green']),
    set(['blue', 'purple']),
    set(['red', 'purple']),
    set(['red', 'orange']),
    set(['yellow', 'green']),
    set(['yellow', 'orange']),
]

CONFLICT_SET = set(['cow', 'monkey', 'chicken'])

def test_find_conflicting_dag_set():
    """
    Ensure that the right values are determined by find_conflicting_dag_set
    """
    # Should not be found in the default set - no conflicts
    assert operator.DAG_RUN_SELECT_RUNNING_SQL is not None
    assert not operator.find_conflicting_dag_set("this_is_completely_cheese")

    # Check for contents vs the COLOR_SETS
    not_in_green_response_set = set(['purple', 'red', 'orange'])
    response_set = operator.find_conflicting_dag_set(
        dag_name='green', conflicting_dag_sets=COLOR_SETS)
    assert 'blue' in response_set
    assert 'yellow' in response_set
    assert not_in_green_response_set.isdisjoint(response_set)
    assert len(response_set) == 2

def get_executing_dags_stub_running_twice():
    return [
        ('buffalo', 'now'),
        ('buffalo', 'earlier'),
        ('squirrel', 'ages ago'),
    ]

def get_executing_dags_stub():
    return [
        ('buffalo', 'now'),
        ('chicken', 'earlier'),
        ('monkey', 'ages ago'),
    ]

def get_executing_dags_stub_no_conflicts():
    return [
        ('buffalo', 'now'),
        ('hedgehog', 'earlier'),
        ('panda', 'ages ago'),
    ]

def test_find_conflicting_dag():
    """
    Ensure that:
    1) responds with a found conflict
    2) responds None if there is no conflict
    3) responds with the dag_id_to_check being searched for if it is running
       more than once.
    """
    cco = ConcurrencyCheckOperator(
        conflicting_dag_set=CONFLICT_SET,
        task_id='bogus')

    # no conflicts
    cco.get_executing_dags = get_executing_dags_stub_no_conflicts
    assert cco.find_conflicting_dag('buffalo') is None

    # self is running twice
    cco.get_executing_dags = get_executing_dags_stub_running_twice
    assert cco.find_conflicting_dag('buffalo') != 'squirrel'
    assert cco.find_conflicting_dag('buffalo') == 'buffalo'

    # a conflict from the list
    cco.get_executing_dags = get_executing_dags_stub
    assert cco.find_conflicting_dag('buffalo') != 'monkey'
    assert cco.find_conflicting_dag('buffalo') == 'chicken'

@mock.patch('shipyard_airflow.plugins.concurrency_check_operator.XcomPusher')
def test_execute_exception(xcom_pusher):
    """
    Run the whole execute function for testing
    """
    cco = ConcurrencyCheckOperator(
        conflicting_dag_set=CONFLICT_SET,
        task_id='bogus')
    # dag_id of cow should cause monkey to conflict.
    cco.check_dag_id = 'cow'
    cco.get_executing_dags = get_executing_dags_stub
    try:
        context = {'task_instance': None}
        cco.execute(context)
        pytest.fail('AirflowException should have been raised')
    except AirflowException as airflow_exception:
        assert 'Aborting run' in airflow_exception.args[0]

@mock.patch('shipyard_airflow.plugins.concurrency_check_operator.XcomPusher')
def test_execute_success(xcom_pusher):
    """
    Run the whole execute function for testing - successfully!
    """
    cco = ConcurrencyCheckOperator(
        conflicting_dag_set=set(['car', 'truck']),
        task_id='bogus')

    # dag_id of airplane should have no conflicts
    cco.check_dag_id = 'airplane'
    cco.get_executing_dags = get_executing_dags_stub
    try:
        context = {'task_instance': None}
        cco.execute(context)
        assert True
    except AirflowException:
        pytest.fail('AirflowException should not have been raised')


"""
The following module receives user's coordinates, a year and returns a map
with points of the locations of filming on that year that are the nearest to
the user's location.

To do that, the module uses data.csv file that is generated by gen_csv module.
"""

import pandas as pd
import folium as fl
from geopy.geocoders import Nominatim
from geopy.exc import GeocoderUnavailable
import haversine

def gen_address(coodrdinates: tuple):
    '''
    The function generates an address by the entered by user coordinates.
    '''
    address = ''

    for elem in coodrdinates:
        address += str(elem) + ', '

    geolocator = Nominatim(user_agent="tito")
    location = geolocator.reverse(address[:-2], language='en')

    location = str(location).split(', ')
    return location[-1]

def gen_location(location: str):
    '''
    The function returns latitude and longitude of the entered location.
    '''
    data = ''
    check = False
    counter = 0
    try:
        while check != True:
            if counter == 6:
                return ''

            geolocator = Nominatim(user_agent='tito').geocode(location)

            try:
                data = (geolocator.latitude, geolocator.longitude)
                check = True

            except AttributeError:

                location = location.split(', ')[1:]
                new_location = ''

                for elem in location:
                    new_location += elem + ', '

                location = new_location
                counter += 1

    except GeocoderUnavailable:
        return data
    return data

def gen_distance(coord1: tuple, coord2: tuple):
    '''
    The function returns the great-circle distance between two points on a sphere.
    >>> gen_distance((45.7597, 4.8422), (48.8567, 2.3508))
    392.2172595594006
    '''
    distance = haversine.haversine(coord1, coord2)

    return distance

def calculate_distances(year: int, point: tuple):
    '''
    The function filters a DataFrame by the year entered by user,
    calculates the distances from the entered by the user point to all the
    coordinates and returns the nearest spots.
    '''

    data = pd.read_csv("data.csv")
    address = gen_address(point)  # used to narrow down the number of options

    data = data[data['Location'].str.contains(address) == True]
    data = data[data['Year'] == year]

    data.to_csv('beta.csv', index=False)

    print('\nSearching the nearest points...')

    data['Coordinates'] = data['Location'].apply(gen_location)
    data = data[data['Coordinates'] != '']

    data.to_csv('beta.csv', index=False)

    data['Lat'] = data['Coordinates'].apply(lambda x: x[0])
    data['Lon'] = data['Coordinates'].apply(lambda x: x[1])

    data['Distance'] = data['Coordinates'].apply(
        lambda x: gen_distance(point, x))

    data = data.sort_values(by=['Distance'])

    print('\nGenerating the map...')

    data = data.head(10)
    data.to_csv('beta.csv', index=False)

    return data[['Title', 'Year', 'Location', 'Lat', 'Lon', 'Distance']]

def create_map(data, address: tuple):
    '''
    The following function generates a map with the filming points based
    on a DataFrame.
    '''
    lat = data['Lat'].tolist()
    lon = data['Lon'].tolist()
    title = data['Title'].tolist()
    point_address = data['Location'].tolist()
    year = data['Year'].tolist()
    dist = data['Distance'].tolist()
    length = data.shape[0]

    location = [address[0], address[1]]

    film_map = fl.Map(location=location,
                      zoom_start=5)

    film_points = fl.FeatureGroup(name='Film map')

    html = """

{}

Year: {}
Address: {}

Distance: {} km

        """

    for _ in range(length):

        coordinates = [location, [float(lat[_]), float(lon[_])]]

        film_points.add_child(fl.Marker(location=[lat[_], lon[_]],
                                        popup=html.format(
            title[_], year[_], point_address[_], round(dist[_], 2)),
            icon=fl.Icon()))

        connector = fl.PolyLine(locations=coordinates, weight=3)
        film_map.add_child(connector)

    fl.Marker(
        location=location,
        popup="You are here!",
        icon=fl.Icon(icon="home", color="red"),
    ).add_to(film_map)

    film_map.add_child(film_points)
    film_map.save('FilmsAroundYou.html')

    text = '\nOpen your browser to view the generated map!'

    return text

def main():
    '''
    The function executes aa process of creating a map based on the info,
    entered by user.
    '''
    example = '2002 -37.9064862 144.7533059'
    user_info = input(
        '\nPlease, enter the year and your coordinates (latitude & longitude) \
like this:\n{}\nEXAMPLE: {}\n{}\n'
        .format('='*37, example, '='*37))

    user_year, lat, lon = user_info.split(' ')
    user_coord = (float(lat), float(lon))
    user_year = int(user_year)

    ten_points = calculate_distances(user_year, user_coord)

    if ten_points.shape[0] == 0:
        print('\n\n\nOops, looks like there are no near films points in that year!')
        print('\nTry again with another coordinates/year or regenerate the file!')
        main()

    result = create_map(ten_points, user_coord)
    print(result)

    return result

if __name__ == "__main__":

    main()


from PyQt5 import QtWidgets, QtGui, QtCore
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
import sys
import platform
import sqlite3
import requests
import smtplib
import ssl
from bs4 import BeautifulSoup
import time
# import windows
from MainWindow import Ui_MainWindow
from TrackerWindow import Ui_TrackerWindow
from DialogWindow import Ui_DialogWindow
from Database import DatabaseManager
from SelectWindow import Ui_SelectWindow

class MainWindow(QMainWindow):
    def __init__(self):
        QMainWindow.__init__(self)
        self.window = Ui_MainWindow()
        self.window.setupUi(self)
        self.show()
        ####################################

        self.Tracker_Window = TrackerWindow()

        ####################################
        self.window.AccesButton.clicked.connect(self.show_tracker_window)
        self.window.helpButton.installEventFilter(self)

    def show_tracker_window(self):
        if len(self.window.emailText.text()) == 0:
            QMessageBox.about(self, "eMag Price Tracker",
                              "Please enter a valid email address!")
        else:
            self.hide()
            self.Tracker_Window.email = self.window.emailText.text()
            self.Tracker_Window.show()

    def eventFilter(self, object, event):
        if object == self.window.helpButton and event.type() == QtCore.QEvent.HoverEnter:
            QMessageBox.about(self, "eMag Price Tracker",
                              "Enter your valid email and start tracking. Add links for your products, create a preset, load it, and start tracking!")
        return False

class SelectWindow(QWidget):
    def __init__(self):
        QWidget.__init__(self)
        self.window = Ui_SelectWindow()
        self.window.setupUi(self)

class DialogWindow(QWidget):
    def __init__(self):
        QWidget.__init__(self)
        self.window = Ui_DialogWindow()
        self.window.setupUi(self)

class pricetracker_thread(QThread):
    def __init__(self, products, Notif_List, email):
        super(pricetracker_thread, self).__init__()
        self.threadactive = True
        self.products = products
        self.Notif_List = Notif_List
        self.email = email

    @pyqtSlot()
    def run(self):
        self.check_price(self.products, self.Notif_List, self.email)

    def number(self, string):
        number = 0
        for letter in string:
            if letter.isalnum() == True:
                number = number*10+int(letter, 10)
        return number

    def send_mail(self, message, email):
        context = ssl.create_default_context()
        server = smtplib.SMTP("smtp.gmail.com", 587)
        server.ehlo()
        server.starttls(context=context)
        server.ehlo()
        server.login("emagpricetracker@gmail.com", "YAMAsiro12")
        server.sendmail("emagpricetracker@gmail.com", email, message)

    def initialize_prices(self, products):
        # initialize the prices
        previous_prices = [x[1] for x in products]
        links = [x[0] for x in products]
        header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
        for i in range(0, len(links)):
            page = requests.get(str(links[i]), headers=header)
            main = BeautifulSoup(page.content, 'html.parser')
            price = main.find(class_="product-new-price").get_text()
            price = price.strip()
            price = self.number(price[0:len(price)-6])
            previous_prices[i] = price
        db_manager = DatabaseManager('Database.db')
        db_manager.check_database()
        cursor = db_manager.conn.cursor()
        for i in range(0, len(previous_prices)):
            cursor.execute(
                "UPDATE Products SET previous_price =? WHERE link=?", (previous_prices[i], links[i]))
        db_manager.conn.commit()
        db_manager.close_connection()

    def check_price(self, products, Notif_List, email):

        header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"}
        # initialize the prices
        previous_prices = [x[1] for x in products]
        links = [x[0] for x in products]
        if 0 in previous_prices:
            self.initialize_prices(products)
        # checking the prices every 900sec
        db_manager = DatabaseManager('Database.db')
        db_manager.check_database()
        cursor = db_manager.conn.cursor()
        while True:
            for i in range(0, len(links)):
                page = requests.get(links[i], headers=header)
                main = BeautifulSoup(page.content, 'html.parser')
                price = main.find(class_="product-new-price").get_text()
                name = main.find(class_="page-title").get_text()
                name = name.strip()
                price = price.strip()
                price = self.number(price[0:len(price)-6])
                if price < previous_prices[i]:
                    aux = previous_prices[i]-price
                    print_message = "Price has been dropped by {0} lei for product {1}".format(
                        aux, name)
                    previous_prices[i] = price
                    subject = "Price was decreased by {0} lei for product {1}".format(
                        aux, name)
                    body = "Check the link {}".format(links[i])
                    message = f"Subject: {subject}\n\n{body}"
                    self.send_mail(message, email)
                    ########################################
                    cursor.execute(
                        "UPDATE Products SET previous_price =? WHERE link=?", (previous_prices[i], links[i]))
                    Notif_List.addItem(QListWidgetItem(print_message))
                    db_manager.conn.commit()
                else:
                    if price < previous_prices[i]:
                        print_message = "Price was increased by {0} lei for product {1}".format(price-previous_prices[i],
                                                                                name)
                        Notif_List.addItem(QListWidgetItem(print_message))
                    else:
                        print_message = "Price it's the same for product {0}".format(
                            name)
                        Notif_List.addItem(QListWidgetItem(print_message))
                print(print_message)
            time.sleep(300)

    def stop(self):
        self.threadactive = False
        self.terminate()

class TrackerWindow(QMainWindow):
    def __init__(self):
        QMainWindow.__init__(self)
        self.window = Ui_TrackerWindow()
        self.window.setupUi(self)
        ####################################
        self.Dialog_Window = DialogWindow()
        self.Select_Window = SelectWindow()
        self.Select_Window2 = SelectWindow()
        ####################################
        self.window.AddButton.clicked.connect(self.add_Item)
        self.window.SaveButton.clicked.connect(self.show_popup)
        self.window.LoadButton.clicked.connect(self.show_Presets)
        self.window.DeleteButton.clicked.connect(self.show_Presets2)
        self.Dialog_Window.window.SaveButton.clicked.connect(
            self.save_Preset)
        self.Select_Window.window.Button.clicked.connect(self.load_Preset)
        self.Select_Window2.window.Button.clicked.connect(self.delete_Preset)
        self.preset_is_loaded = False
        ####################################

    def add_Item(self):
        text = self.window.Link_Input.text()
        if len(text) == 0:
            QMessageBox.about(self, "eMag Price Tracker",
                              "You must add the products URL!")
        else:
            item = QListWidgetItem(text)
            self.window.Tracker_List.addItem(item)
            widget = QtWidgets.QWidget(self.window.Tracker_List)
            button = QtWidgets.QPushButton(widget)
            button.setText("X")
            button.setFixedSize(25, 25)
            button.clicked.connect(lambda: self.remove_Item(item))
            layout = QtWidgets.QHBoxLayout(widget)
            layout.setContentsMargins(0, 1, 1, 0)
            layout.addStretch()
            layout.addWidget(button)
            self.window.Tracker_List.setItemWidget(item, widget)
            self.window.Link_Input.clear()

    def remove_Item(self, item):
        self.window.Tracker_List.takeItem(self.window.Tracker_List.row(item))

    def show_popup(self):
        if self.window.Tracker_List.count() == 0:
            QMessageBox.about(self, "eMag Price Tracker",
                              "You must add at least one product!")
        else:
            self.Dialog_Window.show()

    def get_Presets(self):
        db_manager = DatabaseManager('Database.db')
        db_manager.check_database()
        cursor = db_manager.conn.cursor()
        cursor.execute("SELECT DISTINCT(preset) FROM Products")
        list = cursor.fetchall()
        list = [x[0] for x in list]
        return list

    def show_Presets(self):
        self.Select_Window.window.List.clear()
        if self.preset_is_loaded == True:
            self.thread.stop()

        self.Select_Window.window.Button.setText("Load Preset")
        self.Select_Window.show()
        list = self.get_Presets()
        for i in range(0, len(list)):
            item = QListWidgetItem(str(list[i]))
            self.Select_Window.window.List.addItem(item)

    def show_Presets2(self):
        self.Select_Window2.window.List.clear()
        if self.preset_is_loaded == True:
            QMessageBox.about(self, "eMag Price Tracker",
                              "Preset is already loaded")
        else:
            self.Select_Window2.window.Button.setText("Delete Preset")
            self.Select_Window2.show()
            list = self.get_Presets()
            for i in range(0, len(list)):
                item = QListWidgetItem(str(list[i]))
                self.Select_Window2.window.List.addItem(item)

    def save_Preset(self):
        if len(self.Dialog_Window.window.Name_Input.text()) == 0:
            QMessageBox.about(self, "eMag Price Tracker",
                              "You must choose a name for the preset!")
        list = self.get_Presets()
        if str(self.Dialog_Window.window.Name_Input.text()) in list:
            QMessageBox.about(self, "eMag Price Tracker",
                              "The preset already exists!")
        else:
            db_manager = DatabaseManager('Database.db')
            if db_manager.check_database():
                cursor = db_manager.conn.cursor()
            cursor.execute("""create table if not exists Products(
                link text,
                preset text,
                previous_price integer
            )""")

            for i in range(self.window.Tracker_List.count()):
                cursor.execute("""INSERT INTO Products VALUES(?,?,?)""", (str(
                    self.window.Tracker_List.item(i).text()), str(self.Dialog_Window.window.Name_Input.text()), 0))
            cursor.execute("SELECT * FROM Products")

            print(cursor.fetchall())
            db_manager.conn.commit()
            db_manager.close_connection()
            self.window.Tracker_List.clear()
            self.Dialog_Window.hide()

    def load_Preset(self):
        self.window.Notif_List.clear()
        preset_name = self.Select_Window.window.List.currentItem().text()
        db_manager = DatabaseManager('Database.db')
        db_manager.check_database()
        cursor = db_manager.conn.cursor()
        cursor.execute(
            "SELECT link,previous_price FROM Products WHERE preset=?", (preset_name,))
        self.Select_Window.hide()
        if str(preset_name) is not None:
            self.window.Status_Label.setText(str(preset_name)+" loaded!")
        #########################################
        self.start_thread(cursor.fetchall(),
                          self.window.Notif_List, self.email)
        self.preset_is_loaded = True

    def start_thread(self, products, notif_list, email):
        print(email)
        self.thread = pricetracker_thread(products, notif_list, email)
        self.thread.setTerminationEnabled(True)
        self.thread.start()

    def stop_thread(self):
        self.thread.stop()

    def delete_Preset(self):
        preset_name = self.Select_Window2.window.List.currentItem().text()
        db_manager = DatabaseManager('Database.db')
        db_manager.check_database()
        cursor = db_manager.conn.cursor()
        cursor.execute("DELETE FROM Products WHERE preset=?", (preset_name,))
        cursor.execute("SELECT * FROM Products")
        print(cursor.fetchall())
        db_manager.conn.commit()
        db_manager.close_connection()
        self.Select_Window2.hide()
    ####################################################

app = QApplication(sys.argv)
Main_Window = MainWindow()
sys.exit(app.exec_())


from langchain.vectorstores import Qdrant
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.chains import RetrievalQA
from langchain.llms import OpenAI
import qdrant_client
import os

# Initialize a Qdrant client. Returns an object used to interact with the Qdrant server
client = qdrant_client.QdrantClient(
    os.getenv("QDRANT_HOST"),
    api_key= os.getenv("QDRANT_API_KEY")
)

#create a qdrant collection
def create_collection():
    os.environ['QDRANT_COLLECTION_NAME'] = "Tenaris"
    #create vectors configuration object
    vectors_config = qdrant_client.http.models.VectorParams(
        size=1536,
        distance=qdrant_client.http.models.Distance.COSINE
    )
    client.create_collection(
        collection_name=os.getenv("QDRANT_COLLECTION_NAME"),
        vectors_config=vectors_config
    )

#create vector store object
def add_texts_to_collection():
    embeddings = OpenAIEmbeddings()

    vector_store = Qdrant(
            client=client,
            collection_name=os.getenv("QDRANT_COLLECTION_NAME"),
            embeddings=embeddings
        )

    # Add documents to the cloud collection
    # get_chunks function takes a text and returns a list of chunks
    def get_chunks(text):
        splitter = CharacterTextSplitter(
            separator="\n",
            chunk_size=1000,
            chunk_overlap=150,
            length_function=len
            )
        chunks = splitter.split_text(text)
        return chunks

    with open("data/paper.txt", "r") as f:
        raw_text = f.read()

    texts = get_chunks(raw_text)
    return vector_store.add_texts(texts)

# plug vector store into retrieval qa chain

qa = RetrievalQA.from_chain_type(
    llm=OpenAI(),
    chain_type="stuff",
    retriever=Qdrant(
            client=client,
            collection_name=os.getenv("QDRANT_COLLECTION_NAME"),
            embeddings=OpenAIEmbeddings()
        ).as_retriever()

)
query = "What is the text about?"
print(qa.run(query))


import numpy as np
import pandas as pd
import sys, json

class NaiveBayes:
	def __init__(self):
		self.pos_class = 0
		self.neg_class = 0
		self.class_prob = []
		self.cond_prob = {} # Dictionary to store the conditional probabilities of each attribute given the class 0 or 1

	#Computes the number of positive and negative class and calculate its corresponding probabilities and store in class_prob
	def count_y(self, y):
		for val in y:
			if val==1:
				self.pos_class = self.pos_class+1
		self.neg_class = len(y)-self.pos_class

		self.class_prob.append(self.neg_class/len(y))
		self.class_prob.append(self.pos_class/len(y))

	#Computes the conditional probability for the given attribute given the class
	def compute_freq(self, X, feature, y):
		freq = {}
		for i in range(len(X)):
			val = X[i]
			if val in freq.keys():
				if y[i] == 0:
					freq[val][0] = freq[val][0]+1
				else:
					freq[val][1] = freq[val][1]+1
			else:
				freq[val] = [0,0]
				if y[i] == 0:
					freq[val][0] = freq[val][0]+1
				else:
					freq[val][1] = freq[val][1]+1
		return freq

	#Function to train the training data by computing the conditional probabilities of all attribute given each class output
	def train(self, X_train, y_train):
		self.count_y(y_train)

		for col in X_train.columns:
			#print(col)
			freq = self.compute_freq(X_train[col], col, y_train)
			#print(freq)
			prob = {}
			for key in freq:
				prob[key] = [0,0]
				if freq[key][0] == 0:
					freq[key][0] = 2
				if freq[key][1] == 0:
					freq[key][1] = 2

				prob[key][0] = freq[key][0]/self.neg_class
				prob[key][1] = freq[key][1]/self.pos_class
			#print(prob)
			self.cond_prob[col] = prob

	#returns predicted class y for the given test data
	def test(self, test_data):
		pred_y = []
		for index, row in test_data.iterrows():
			pos_prob = 1
			neg_prob = 1
			for col in test_data.columns:
				neg_prob = neg_prob*self.cond_prob[col][row[col]][0]
				pos_prob = pos_prob*self.cond_prob[col][row[col]][1]
			if pos_prob > neg_prob:
				pred_y.append(1)
			else:
				pred_y.append(0)
		return pred_y

if __name__ == "__main__":
	args = sys.argv
	train_file = str(args[1])
	test_file = str(args[2])
	MFile_name = str(args[3])
	RFile_name = str(args[4])

	train_data = pd.read_csv(train_file)
	NB = NaiveBayes()
	y = train_data['class']
	#NB.count_y(y)

	X_train = train_data.drop(['class'], axis=1)

	NB.train(X_train, y)

	MFile = open(MFile_name, "a")
	MFile.write("\nClass probabilities\n")
	MFile.write("\nP(Class 0) = "+str(NB.class_prob[0]))
	MFile.write("\nP(Class 1) = "+str(NB.class_prob[1]))

	MFile.write("\n\nConditional probabilities for each attribute")
	for col in X_train.columns:
		MFile.write("\n\nAttribute : "+col)
		MFile.write("\n\t Class 0 \t\t Class 1")
		for key in NB.cond_prob[col]:
			MFile.write("\n"+str(key))
			MFile.write("\t"+str(NB.cond_prob[col][key][0]))
			MFile.write("\t"+str(NB.cond_prob[col][key][1]))
		#df = pd.DataFrame(NB.cond_prob[col])
		#df.to_json(MFile)
		#df.to_csv(MFile)

	test_data  = pd.read_csv(test_file)
	test_y = test_data['class'].values
	test_X = test_data.drop(['class'], axis=1)
	pred_y = NB.test(test_X)

	RFile = open(RFile_name, "a")
	RFile.write("Predicted class\n")
	json.dump(pred_y, RFile)

	#Computing confusion matrix
	TP = 0
	TN = 0
	FP = 0
	FN = 0

	for i in range(len(test_y)):
		if test_y[i] == 1 and pred_y[i] == 1:
			TP = TP + 1
		if test_y[i] == 0 and pred_y[i] == 0:
			TN = TN + 1
		if test_y[i] == 0 and pred_y[i] == 1:
			FP = FP + 1
		if test_y[i] == 1 and pred_y[i] == 0:
			FN = FN + 1

	RFile.write("\n\nConfusion Matrix: \n")
	RFile.write("\t True \t False \n")
	RFile.write("True \t "+str(TP)+"\t "+str(FP)+"\n")
	RFile.write("False \t "+str(FN)+"\t "+str(TN)+"\n")


import logging
import json
import os
import math

logger = logging.getLogger()
logger.setLevel(logging.INFO)

class FileStorageClient:

    def __init__(self, client, bucket_name):
        self.client = client
        self.bucket_name = bucket_name

    def upload_local_file(self, file_name):
        s3_file_key = file_name.replace('_', '/')

        logging.info(f"Opening file {file_name}.")
        file_content = open(file_name, 'rb').read()

        logging.info(f"Saving file in s3 bucket with file_key s3://{self.bucket_name}/{s3_file_key}.")
        self._upload(file_key=s3_file_key, file_content=file_content)

    def get(self, file_key):
        logger.info(f'Getting file content for the file: {file_key} from the bucket: {self.bucket_name}')
        response = self.client.get_object(
            Bucket=self.bucket_name,
            Key=file_key
        )
        return response['Body']

    def _upload(self, file_key, file_content):
        logger.info(f'Uploading file {file_key} into bucket {self.bucket_name}...')
        self.client.put_object(
            Bucket=self.bucket_name,
            Key=file_key,
            Body=file_content
        )

    def download_directory(self, directory_prefix, local_directory):
        os.system(f"mkdir {local_directory}")
        os.system(f"aws s3 sync s3://{self.bucket_name}/{directory_prefix} {local_directory}")

class QueueClient:

    BATCH_REQUEST_LIMIT = 10

    def __init__(self, resource, queue_name):
        self.resource = resource
        self.queue_name = queue_name
        self.queue = self.resource.get_queue_by_name(QueueName=self.queue_name)

    def receive_message(self):
        logger.info(f"Receiving messages from SQS queue")
        messages = self.queue.receive_messages(
            AttributeNames=['SentTimestamp'],
            MaxNumberOfMessages=1,
            MessageAttributeNames=['All'],
            VisibilityTimeout=900,
            WaitTimeSeconds=3
        )
        if messages:
            return messages[0].body, messages[0].receipt_handle
        else:
            return None, None

    def delete_message(self, receipt_handle):
        logger.info(f"Deleting consumed message from SQS queue: {self.queue_name}")
        self.resource.meta.client.delete_message(
            QueueUrl=self.queue.url,
            ReceiptHandle=receipt_handle
        )

    def send_messages(self, messages):
        logger.info(f"Number of messages to send to SQS queue {self.queue_name} - {len(messages)}")

        request_count = math.ceil(len(messages) / self.BATCH_REQUEST_LIMIT)

        logger.info(f"Messages will be sent in following number of requests: {request_count}")
        for i in range(request_count):
            start = i * self.BATCH_REQUEST_LIMIT
            end = (i + 1) * self.BATCH_REQUEST_LIMIT

            messages_page = messages[start:end]

            logger.info(f"Sending {i+1}. batch of messages[{start}:{end}] to SQS...")
            self._send_messages_batch(messages_page)
            logger.info("Successfully delivered.")

    def _send_messages_batch(self, messages):
        response = self.resource.meta.client.send_message_batch(
            QueueUrl=self.queue.url,
            Entries=[dict(
                Id=f"{item['country']}-{item['competitor']}-{item['category']}-{item['page_number']}",
                MessageBody=json.dumps(item)
            ) for item in messages]
        )
        successful = response.get("Successful", [])
        failed = response.get("Failed", [])

        logger.info(f"Number of successfully sent messages in batch: {len(successful)}")

        if len(failed) > 0:
            logger.error(f"Number of failed messages in batch: {len(failed)}")
            for f in failed:
                logger.error(f)


import requests
from bs4 import BeautifulSoup
from bs4 import element
import model

class TVShowSpider:
    def __init__(self, parser, dataStruct, url):
        self.parser = parser
        self.dataStruct = dataStruct
        self.url = url

    def doParse(self):
        self.parser.parse(self.dataStruct, self.url);

class TvShowParser:
    def parse(self, tvShow, url):
        tvShowDao = model.TvShowDao()
        flag = 0
        for tr in LoadCotent().get_content_from_internet(url).body.tbody.contents[1].next_siblings:
            if (type(tr) == element.Tag):
                index = 0
                for td in tr.contents:
                    if (type(td) == element.Tag):
                        if index == 0:
                            # tvShow.set_showTime(str(td.string))
                            tvShow.showTime = str(td.string)
                        elif index == 1:
                            tvShow.set_showPlatform(str(td.string))
                        elif index == 2:
                            tvShow.set_type(str(td.string))
                        elif index == 3:
                            tvShow.set_originName(str(td.string))
                        elif tvShow == 4:
                            tvShow.set_name(str(td.string))
                        index = index + 1
                        if type(td.find('a')) == element.Tag:
                            url = td.find('a').get('href')
                            print(url)
                            self.get_tvshow_detail(url, tvShow)
                tvShowDao.save_TvShow(tvShow)
                print("当前下载数:")
                print(flag)
                flag = flag + 1

    def get_tvshow_detail(self, url, tvShow):
        detailSoup = LoadCotent().get_content_from_internet(url)
        for entry in detailSoup.find_all(id='entry'):
            for entryChild in entry.children:
                # print(type(entryChild))
                if entryChild.name == 'p':
                    if type(entryChild) == element.Tag:
                        # print(entryChild.contents[2])
                        i = 0
                        s = ''
                        for pChild in entryChild.contents:
                            if type(pChild) == element.NavigableString:
                                s = s + pChild.string
                                i = i + 1
                                if i > 2:
                                    break
                        tvShow.set_introduction(s)

class LoadCotent:
    def get_content_from_internet(self, url):
        return BeautifulSoup(requests.get(url).text, 'html.parser')

if __name__ == '__main__':
    TVShowSpider(TvShowParser(), model.TvShow(), 'http://cn163.net/cwmeiju/').doParse()


import re

mailid = input("Enter Your Mail Id - ")

verify = re.findall("^.+@[a-z\.]+.[a-z]{2,4}$",mailid)

if len(verify)==0:
    print("Not Valid Email Id.")
else:
    print("Valid Email :)")


from collections import Counter

n, c = [int(d) for d in input().split()]
num = [int(d) for d in input().split()]

freq = Counter(num)
print(
    " ".join(
        [
            str(d)
            for d in sorted(
                num, key=lambda k: (freq[k], n - num.index(k)), reverse=True
            )
        ]
    )
)


'''
	@property把一个getter方法变成属性
	此时@property本身有创建了另一个装饰器@score.setter.
	负责把一个setter方法变成属性赋值.
'''

class Student(object):
	@property
	def score(self):
		print("get property..")
		return self._score

	@score.setter
	def score(self, value):
		print("setter..")
		self._score = value

s = Student()
s.score = 20 #setter... 实际转化为 s.score(20)
print(s.score) #get property.. 实际转化为s.score()
			   #20


import re
import math
import time

file = open('input.txt')
lines = [line.rstrip('\n') for line in file.readlines()]

timestamp = int(lines[0])
buses = [int(bus) if bus != 'x' else bus for bus in lines[1].split(',')]
filtered_buses = filter(lambda bus: bus != 'x', buses)

def find_soonest_bus(timestamp, buses):
    soonest_bus = None
    min_time_difference = None
    for bus in buses:
        if timestamp % bus == 0:
            return bus, 0

        bus_arrival = math.ceil(timestamp / bus) * bus
        if bus_arrival - timestamp < (min_time_difference or float('inf')):
            soonest_bus = bus
            min_time_difference = bus_arrival - timestamp

    return soonest_bus, min_time_difference

soonest_bus, time_difference = find_soonest_bus(timestamp, filtered_buses)
print(soonest_bus * time_difference)

def validate_sequence(timestamp, buses):
    for i in range(1, len(buses)):
        bus = buses[i]

        if bus == 'x':
            continue
        if (timestamp + i) % bus != 0:
            return False

    return True

def debug(buses, limit=3):
    print(f"======")
    print(buses)
    timestamp = buses[0]
    solutions = []
    while len(solutions) < limit:
        if validate_sequence(timestamp, buses):
            solutions.append(timestamp)
            print(timestamp)
            for i, bus in enumerate(buses):
                if bus == 'x':
                    continue
                print(f"\t+ {i}) / {bus} = {(timestamp + i) / bus}")
        timestamp += buses[0]

    print(f"difference: {solutions[1] - solutions[0]}")

# debug([7], 3)
# debug([7, 13], 3)
# debug([7, 13, 'x', 'x'], 3)
# debug([7, 13, 'x', 'x', 59], 3)
# debug([7, 13, 'x', 'x', 59, 'x'], 3)
# debug([7, 13, 'x', 'x', 59, 'x', 31], 3)
# debug([7, 13, 'x', 'x', 59, 'x', 31, 19], 3)

def lcm(a, b):
    return abs(a*b) // math.gcd(a, b)

def find_timestamp_for_sequence(buses):
    timestamp = buses[0]
    difference = buses[0]

    for i in range(1, len(buses) + 1):
        subarray = buses[:i]
        latest_bus = subarray[-1]

        if latest_bus == 'x':
            continue

        while not validate_sequence(timestamp, subarray):
            timestamp += difference

        difference = lcm(difference, latest_bus)

    return timestamp

print(find_timestamp_for_sequence(buses))


# -*- coding: utf-8 -*-
"""平安银行 定期理财产品 JRCP_LCCP_PAYH_GW_ALL"""
from database._mongodb import MongoClient

def data_shuffle(data):
    if "PDF_NAME_" in data:
        if isinstance(data["PDF_NAME_"], (list, tuple)):
            pdf_name = data["PDF_NAME_"]
            for each in pdf_name:
                if "说明书" in each:
                    pdf_index = data["PDF_NAME_"].index(each)
                    data["PDF_"] = data["PDF_"][pdf_index]
                    data["PDF_NAME_"] = data["PDF_NAME_"][pdf_index]
                    break
    return data

if __name__ == '__main__':
    main_mongo = MongoClient(entity_code="JRCP_LCCP_PAYH_GW_ALL", mongo_collection="JRCP_LCCP")
    data_list = main_mongo.main()
    for data in data_list:
        re_data = data_shuffle(data)
        print(re_data)


'''
1048. Longest String Chain
https://leetcode.com/problems/longest-string-chain/
'''
from typing import List, Optional

from test_tool import assert_value

class Solution:
    def longestStrChain(self, words: List[str]) -> int:
        words = sorted(words, key=len)
        cache = {}
        res = 0
        for word in words:
            cache[word] = 0
            for i in range(len(word)):
                key = word[:i] + word[i+1:]
                if key not in cache:
                    continue
                cache[word] = max(cache[word], cache[key])
            cache[word] += 1
            res = max(res, cache[word])
        return res

assert_value(4, Solution().longestStrChain, words=["a", "b", "ba", "bca", "bda", "bdca"])
assert_value(5, Solution().longestStrChain, words=["xbc", "pcxbcf", "xb", "cxbc", "pcxbc"])
assert_value(1, Solution().longestStrChain, words=["abcd", "dbqca"])
#


"""

SpectralSynthesis.py

Author: Felix Bilodeau-Chagnon
Affiliation: McGill University
Created on: Thursday 9 July 2020 17:03:55 EDT

Description: Contains a class which calculates the dust emissions
for each galaxy in a GalaxyEnsemble object.

"""

import numpy as np
from scipy.integrate import simps
from ares.physics.Constants import c, h, k_B, g_per_msun, cm_per_kpc
from ares.util.ParameterFile import ParameterFile

class DustPopulation:
    def __init__(self, galaxyEnsemble, fmin = 1e14, fmax = 1e17, Nfreqs = 500,\
        zmin = 4, zmax = 10, Nz = 7):
        """
        (GalaxyEnsemble, **kwargs) -> DustPopulation

        Creates a DustPopulation instance for the given GalaxyEnsemble
        instance.

        PARAMETERS:

        galaxyEnsemble: GalaxyEnsemble instance

            the GalaxyEnsemble instance for which the dust emissions must be calculated

        fmin: number

            minimum stellar emission frequency sampled in Hertz (default = 1e14 Hz)

        fmax: number

            maximum stellar emission frequency sampled in Hertz (default = 1e17 Hz)

        Nfreqs: integer

            number of frequencies between fmin and fmax to be sampled (default = 500)

        zmin: number

            minimum redshift where the dust emissions will be calculated (default = 4)

        zmax: number

            maximum redshift where the dust emissions will be calculated (default = 10)

        Nz: integer

            number of redshift between zmin and zmax where emissions will be calculated
            (default = 7)
        """
        self._pop = galaxyEnsemble

        self._fmin = fmin
        self._fmax = fmax
        self._Nfreqs = Nfreqs

        self._zmin = zmin
        self._zmax = zmax
        self._Nz = Nz

        self._Ngalaxies = galaxyEnsemble.histories['SFR'].shape[0]

        # Swap the order if they were put in wrong
        if self._fmin > self._fmax:
            self._fmin, self._fmax = self._fmax, self._fmin

        if self._zmin > self._zmax:
            self._zmin, self._zmax = self._zmax, self._zmin

        self._frequencies = np.linspace(self._fmin, self._fmax, self._Nfreqs)
        self._z = np.linspace(self._zmin, self._zmax, self._Nz)

    # Getters for the arguments and simply-derived quantites
    @property
    def pop(self):
        return self._pop

    @property
    def pf(self):
        return self._pop.pf

    @property
    def histories(self):
        return self._pop.histories

    @property
    def fmin(self):
        return self._fmin

    @property
    def fmax(self):
        return self._fmax

    @property
    def Nfreqs(self):
        return self._Nfreqs

    @property
    def frequencies(self):
        return self._frequencies

    @property
    def zmin(self):
        return self._zmin

    @property
    def zmax(self):
        return self._zmax

    @property
    def Nz(self):
        return self._Nz

    @property
    def z(self):
        return self._z

    @property
    def Ngalaxies(self):
        return self._Ngalaxies

    # Properties which must be calculated
    @property
    def L_nu(self):
        """
        (void) -> 3darray

        Calculates and / or returns the specific luminosity L_nu in
        ergs / s / Hz for each galaxy at all frequencies and redshifts
        provided.

        Note: This is a very expensive calculation! Make sure you really need
        to see all those redshifts. Lower redshifts tend to take longer to
        calculate.

        first axis: galaxy index
        second axis: data at given frequency index (ergs / s / Hz)
        third axis: data at given redshift index
        """
        if not hasattr(self, '_L_nu'):
            self._L_nu = np.zeros((self.Ngalaxies, self.Nfreqs, self.Nz))
            waves = c / self.frequencies * 1e8
            for i in range(len(self.z)):
                self._L_nu[:,:,i] = self.pop.synth.Spectrum(waves, \
                    zobs = self.z[i], sfh = self.histories['SFR'], tarr = self.histories['t'])

        return self._L_nu

    @property
    def R_dust(self):
        """
        (void) -> 2darray

        Calculates and / or returns the dust radius for each galaxy at each redshift
        in kpc.

        first axis: galaxy index
        second axis: redshift index
        """
        if not hasattr(self, '_R_dust'):
            Mh = np.zeros((self.Ngalaxies, self.Nz))

            for i in range(self.Nz):
                Mh[:,i] = self.pop.get_field(self.z[i], 'Mh')

            self._R_dust = self.pop.halos.VirialRadius(self.z[:], Mh[:,:]) * 0.018

        return self._R_dust

    @property
    def M_dust(self):
        """
        (void) -> 2darray

        Fetches and / or returns the dust mass in each galaxy at each redshift in solar masses.

        first axis: galaxy index
        second axis: redshift index
        """
        if not hasattr(self, '_M_dust'):
            self._M_dust = np.zeros((self.Ngalaxies, self.Nz))

            for i in range(self.Nz):
                self._M_dust[:,i] = self.pop.get_field(self.z[i], 'Md')

        return self._M_dust

    @property
    def kappa_nu(self):
        """
        (void) -> 3darray

        Returns and / or calculates the dust opacity given the frequencies. The data
        returned is in cm^2 / g.

        first axis: galaxy index
        second axis: frequency index
        third axis: redshift index
        """
        if not hasattr(self, '_kappa_nu'):
            self._kappa_nu = np.zeros((self.Ngalaxies, self.Nfreqs, self.Nz))
            self._kappa_nu += (0.1 * (self.frequencies / 1e12)**2)[None, :, None]

        return self._kappa_nu

    @property
    def tau_nu(self):
        """
        (void) -> 3darray

        Returns and / or calculates the optical depth of the dust. This
        quantity is dimensionless.

        first axis: galaxy index
        second axis: frequency index
        third axis: redshift index
        """
        if not hasattr(self, '_tau_nu'):
            self._tau_nu = 3 * (self.M_dust[:, None, :] * g_per_msun) * self.kappa_nu \
                / (4 * np.pi * (self.R_dust[:, None, :] * cm_per_kpc)**2)

        return self._tau_nu

    @property
    def T_cmb(self):
        """
        (void) -> 2darray

        Returns and / or calculates the Cosmic Microwave Background temperature
        for each galaxy at each redshift.

        first axis: galaxy index
        second axis: redshift index
        """
        if not hasattr(self, '_T_cmb'):
            self._T_cmb = np.zeros((self.Ngalaxies, self.Nz))
            self._T_cmb += self.pop.cosm.TCMB(self.z[None, :])

        return self._T_cmb

    @property
    def T_dust(self):
        """
        (void) -> 2darray

        Calculates and / or returns the dust temperature for each galaxy at
        each redshift.

        first axis: galaxy index
        second axis: redshift index

        If the temperature returns nan, then this means that there was no dust
        in the galaxy in the first place (maybe galaxy was not born yet, or the halo
        has no galaxy at all)
        """
        if not hasattr(self, '_T_dust'):
            # Calculate total power absorbed per dust mass
            tmp_stellar = self.L_nu * (1 - np.exp(-self.tau_nu)) \
                / self.tau_nu * self.pop.histories['fcov'] * self.kappa_nu \
                / (self.R_dust[:,None,:] * cm_per_kpc)**2

            tmp_cmb = 8 * np.pi * h / c**2 * self.kappa_nu * (self.frequencies[None, :, None])**3 \
                / (np.exp(h * self.frequencies[None,:,None] / k_B / self.T_cmb[:,None,:]) - 1)

            tmp_power = simps(tmp_stellar + tmp_cmb, self.frequencies, axis = 1)
            # This prefactor is based on analytically performing the integral
            tmp_prefactor = 64e-25 / 63 * np.pi**7 * k_B**6 / c**2 / h**5
            tmp_T_dust = (tmp_power / tmp_prefactor)**(1/6)
            NaNs = np.isnan(tmp_T_dust)
            tmp_T_dust[NaNs] = 0.
            self._T_dust = tmp_T_dust

        return self._T_dust

    def dust_sed(self, fmin, fmax, Nfreqs):
        """
        (num, num, int) -> 1darray, 3darray

        Returns and / or calculates the dust SED in ergs / s / Hz for each galaxy
        at each redshift.

        RETURNS

        frequencies, SED: 1darray, 3darray

        first axis: galaxy index
        second axis: frequency index
        third axis: redshift index

        PARAMETERS

        fmin: number
            minimum frequency of the band

        fmax: number
            maximum frequency of the band

        Nfreqs: integer
            number of frequencies between fmin and fmax to be calculated
        """
        freqs = np.linspace(fmin, fmax, Nfreqs)
        kappa_nu = np.zeros((self.Ngalaxies, Nfreqs, self.Nz))
        kappa_nu += 0.1 * (freqs[None,:,None] / 1e12)**2
        SED = 8 * np.pi * h / c**2 * freqs[None,:,None]**3 * kappa_nu \
            / (np.exp(h * freqs[None,:,None] / k_B / self.T_dust[:,None,:]) - 1) \
            * self.M_dust[:,None,:] * g_per_msun
        return freqs, SED


from scipy.ndimage import distance_transform_edt as distance
from skimage import segmentation as skimage_seg
import torch
import torch.nn as nn
import numpy as np

"""
original code: https://github.com/LIVIAETS/boundary-loss
refarence from: https://github.com/JunMa11/SegWithDistMap
"""

class BoundaryLoss(nn.Module):
    """
    compute boundary loss for binary segmentation
    input: outputs_soft: sigmoid results,  shape=(b,c,x,y)
           gt_sdf: sdf of ground truth (can be original or normalized sdf); shape=(b,c,x,y)
    output: boundary_loss; sclar
    """

    def __init__(self, class_idx=1):
        super(BoundaryLoss, self).__init__()
        # Self.idc is used to filter out some classes of the target mask. Use fancy indexing
        self.class_idx = class_idx

    def __call__(self, outputs_soft, label_batch):
        # print(outputs_soft.shape)
        gt_sdf_npy = compute_sdf1_1(label_batch.cpu().numpy(), outputs_soft.shape)
        gt_sdf = torch.from_numpy(gt_sdf_npy).float().cuda(outputs_soft.device.index)
        pc = outputs_soft.type(torch.float32)
        dc = gt_sdf.type(torch.float32)

        multipled = torch.einsum("bcwh, bcwh->bcwh", pc, dc)
        loss = multipled.mean(dim=(1,2,3))

        return loss

def compute_sdf1_1(img_gt, out_shape):
    """
    compute the normalized signed distance map of binary mask
    input: segmentation, shape = (batch_size, c, x, y)
    output: the Signed Distance Map (SDM)
    sdf(x) = 0; x in segmentation boundary
             -inf|x-y|; x in segmentation
             +inf|x-y|; x out of segmentation
    normalize sdf to [-1, 1]
    """
    img_gt = img_gt.astype(np.uint8)
    normalized_sdf = np.zeros(out_shape)

    for b in range(out_shape[0]): # batch size
            # ignore background
        posmask = img_gt[b].astype(np.bool)
        for c in range(out_shape[1]):
            if posmask.any():
                negmask = ~posmask
                posdis = distance(posmask)
                negdis = distance(negmask)
                boundary = skimage_seg.find_boundaries(posmask, mode='inner').astype(np.uint8)
                sdf = (negdis-np.min(negdis))/(np.max(negdis)-np.min(negdis)) - (posdis-np.min(posdis))/(np.max(posdis)-np.min(posdis))
                sdf[boundary==1] = 0
                normalized_sdf[b][c] = sdf

    return normalized_sdf

def compute_sdf(img_gt, out_shape):
    """
    compute the signed distance map of binary mask
    input: segmentation, shape = (batch_size, x, y, z)
    output: the Signed Distance Map (SDM)
    sdf(x) = 0; x in segmentation boundary
             -inf|x-y|; x in segmentation
             +inf|x-y|; x out of segmentation
    """
    img_gt = img_gt.astype(np.uint8)
    gt_sdf = np.zeros(out_shape)

    for b in range(out_shape[0]): # batch size
        posmask = img_gt[b].astype(np.bool)
        for c in range(out_shape[1]):
            if posmask.any():
                negmask = ~posmask
                posdis = distance(posmask)
                negdis = distance(negmask)
                boundary = skimage_seg.find_boundaries(posmask, mode='inner').astype(np.uint8)
                sdf = negdis - posdis
                sdf[boundary==1] = 0
                gt_sdf[b][c] = sdf

    return gt_sdf