import csv
import json
import multiprocessing
import os
import random
import traceback
from datetime import date
from multiprocessing.pool import ApplyResult
from time import sleep, time

from dotenv import find_dotenv, load_dotenv
from openai.error import (AuthenticationError, InvalidRequestError,
                          RateLimitError)
from pymongo import MongoClient
from pymongo.database import Database

from helpers import remove_non_numbers_regex
from logger import Logger
from stages import (categorize, deep_research, extra_research, impactul_news,
                    prediction, summarize_article)

# find and load .env file
load_dotenv(find_dotenv())

csv.field_size_limit(50000000)

class Analyzer:
    """Article analyzer using Langchain and GPT"""
    session: MongoClient
    db: Database
    collections: dict
    categories: list[str]
    apikeys: list[str]
    logger: Logger

    def __init__(self) -> None:
        self.logger = Logger()
        self.session = MongoClient(os.environ["MONGODB_URL"])
        self.db = self.session["news-test"]
        self.article_db = self.session["test"]
        self.collections = {
            0: "All categories",
            1: "lawandcrime",
            2: "web3",
            3: "entertainment",
            4: "sport",
            5: "artandfashion",
            6: "bizandfinance",
            7: "politics",
            8: "scienceandtech",
            9: "lifestyleandhealth",
            10: "gaming"
        }
        self.categories = [
            "Law and Crime",
            "Crypto/Web3",
            "Entertainment",
            "Sports",
            "Art and Fashion",
            "Business and Finance",
            "Politics",
            "Science and Technology",
            "Lifestyle and Health",
            "Gaming",
        ]
        with open('keys/keys.txt', 'r', encoding='utf-8') as keys_file:
            # Maximum 50 processes
            self.apikeys = [line.strip() for line in keys_file.readlines()[:50]]

    def log_invalid_key(self, apikey):
        # remove invallid apikey from valid list
        if apikey in self.apikeys:
            self.apikeys.remove(apikey)
            with open('keys/keys.txt', 'w', encoding='utf-8') as keys_file:
                for key in self.apikeys:
                    keys_file.write(key + '\n')

            # add apikey to invalid key file
            with open('keys/invalid_keys.txt', 'a', encoding='utf-8') as invalid_file:
                invalid_file.write(apikey + '\n')

    def stage_1(self, csv_filename: str, curDate: str):
        os.remove(csv_filename) if os.path.exists(csv_filename) else None

        start_t = time()
        # to test
        collection_name = [self.collections[i] for i in range(1, 11)]
        articles = []
        article_count = 0

        for idx, category in enumerate(collection_name):
            collection = self.article_db[category][curDate]
            rcategory = self.categories[idx]
            documents = collection.find()
            cate_article_count = 0
            for document in documents:
                article_count += 1
                cate_article_count += 1
                # print(f"{article_count} : {rcategory}: {document['siteName']}, {document['link']}")
                articles.append([document['article'], document['siteName'], document['link'], rcategory])
            self.logger.log(f'Stage 1 - {category} {curDate} {cate_article_count} articles')
        end_t = time()

        self.logger.log(f'Stage 1 - {len(articles)} articles uploaded in {end_t - start_t} seconds, start processing...')

        start_t = time()
        article_count = 0

        with open(csv_filename, 'a', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow([
                'baseData',
                'Title',
                'Category',
                'Summary',
                'Importance 1 day',
                'Reasoning for 1 day score',
                'Importance 1 week',
                'Reasoning for 1 week score',
                'Importance 1 month',
                'Reasoning for 1 month score',
                'site_name',
                'link'
            ])
            csvfile.close()
        sumarized_count = 0
        total = len(articles)
        while len(articles) != 0:
            pool = multiprocessing.Pool(processes=min(len(self.apikeys), 50))
            results: list[ApplyResult] = []

            for i, (article, site_name, link, rCategory) in enumerate(articles):
                api_key = self.apikeys[i % len(self.apikeys)]  # Use a different API key for each process
                if len(self.apikeys) > len(articles):
                    api_key = random.choice(self.apikeys)
                results.append(pool.apply_async(stage_1_thread_handler, (api_key, article, site_name, link, rCategory)))
            articles = []
            with open(csv_filename, 'a', newline='', encoding='utf-8') as csvfile:
                writer = csv.writer(csvfile)
                for result in results:
                    article_data = result.get()
                    if article_data[1] == 'APIKey_Error':
                        api_key = article_data[0]
                        self.log_invalid_key(api_key)
                    elif article_data[1] == 'Error':
                        print(f"Statge 1 - Error was occurred while get summary\n: {article_data[0]}")
                    else:
                        writer.writerow(article_data[0:-1])
                        self.logger.log(f"Statge 1 - {sumarized_count}/{total} : {article_data[-1]}")
                        sumarized_count += 1
                        continue
                    articles.append([article_data[2], article_data[3], article_data[4]])
            pool.close()
            pool.join()
        end_t = time()
        self.logger.log(f'Stage 1 - {sumarized_count} articles were summarized in {end_t - start_t} seconds')

    def stage_1_save_db(self, csv_filename, curDate: str):
        self.logger.log(f'Stage 1 - loading summaries from {csv_filename}')
        start_t = time()
        data_list = []
        with open(csv_filename, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                if not row:
                    continue
                try:
                    baseData = json.loads(row[0])
                except Exception as e:
                    self.logger.log(f'Stage 1 - Failed to save analyzed article', e)
                    print("Failed to save summarized article:\ncontent======", row[0])
                    print("error ===========", e)
                    continue
                article = baseData['article']
                title = row[1]
                category = row[2]
                summary = row[3]
                day_score = row[4]
                day_reason = row[5]
                week_score = row[6]
                week_reason = row[7]
                month_score = row[8]
                month_reason = row[9]
                site_name = row[10]
                link = row[11]

                score = {
                    "day": {"score": day_score, "reason": day_reason},
                    "week": {"score": week_score, "reason": week_reason},
                    "month": {"score": month_score, "reason": month_reason}
                }
                data_list.append(
                    {"article": article,
                    "title": title,
                    "category": category,
                    "summary": summary,
                    "score": score,
                    "site_name": site_name,
                    "link": link}
                )
        end_t = time()
        self.logger.log(f'Stage 1 - loaded {len(data_list)} summaries in {end_t - start_t} seconds')
        self.logger.log(f'Stage 1 - saving summaries in db...')
        new_collection = self.db['analyzed_articles'][curDate]
        new_collection.drop()
        result = new_collection.insert_many(data_list)
        print("Data inserted successfully. Inserted IDs:", result.inserted_ids)
        end_t = time()
        self.logger.log(f'Stage 1 - Saved {len(result.inserted_ids)} summaries in db...')

    def stage_2(self, stage1_csv: str, csv_filename: str, timeframe: str):
        os.remove(csv_filename) if os.path.exists(csv_filename) else None
        self.logger.log(f'Stage 2 - loading articles from {stage1_csv}')
        titles = []
        categories = set()
        total = 0
        start_t = time()
        with open(stage1_csv, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)

            for row in csv_reader:
                total += 1
                try:
                    if row[1] == '': continue
                    if timeframe == 'day' and row[4] == '': continue
                    if timeframe == 'week' and row[6] == '': continue
                    if timeframe == 'month' and row[8] == '': continue
                    score = 0
                    if timeframe == 'day':
                        score = int(remove_non_numbers_regex(row[4]))
                    elif timeframe == 'week':
                        score = int(remove_non_numbers_regex(row[6]))
                    elif timeframe == 'month':
                        score = int(remove_non_numbers_regex(row[8]))

                    article = json.loads(row[0])['article']
                    title = row[1]
                    category = row[2]  # Assuming the category is in the 4th column
                    summary = row[3]
                    categories.add(category)  # Add category to the set of unique categories

                    if any(item['title'] == title for item in titles):
                        print(title)
                        continue
                    titles.append({'article': article, 'title': title, 'score': score, 'category': category, 'summary': summary})
                except IndexError as err:
                    self.logger.log(f'Stage 2 - Error while loading articles: {err}')
                    pass
        end_t = time()
        self.logger.log(f'Stage 2 - articles were loaded from {stage1_csv} in {end_t - start_t} second')

        for category in categories:
            if category not in self.categories:
                continue
            category_titles = [title for title in titles if title['category'] == category]
            sorted_titles = sorted(category_titles, key=lambda x: x['score'], reverse=True)

            primaries = []
            for title in sorted_titles[0:20]:
                if title['title'] != '' and title['title'] not in primaries:
                    primaries.append(title['title'])
            secondaries = []
            for title in sorted_titles[20:270]:
                if title['title'] != '' and title['title'] not in primaries and title['title'] not in secondaries:
                    secondaries.append(title['title'])

            result = []
            try:
                result = self.stage_2_category(primaries=primaries, secondaries=secondaries)
                print(result[0])
                self.logger.log(f"Stage 2: {category} for {timeframe}-timeframe:\n {result[1]}")
            except Exception as er:
                error = str(traceback.print_exc())
                self.logger.log(f"Stage 2: Error while categorizing: {er} in {error}")
            try:
                data = json.loads(result[0])
                if data:
                    self.logger.log(f"Stage 2: {len(data)}")
                    with open(csv_filename, "a", encoding='utf-8') as file:
                        writer = csv.writer(file)
                        writer.writerow([category, primaries, secondaries, data])
                else:
                    print('Error')
            except json.decoder.JSONDecodeError as err:
                self.logger.log(f"Stage 2: JSONDecode Error: {err} in {result}")

    def stage_2_category(self, primaries, secondaries):
        apikey = random.choice(self.apikeys)
        try:
            result = categorize(apikey=apikey, primaries=primaries, secondaries=secondaries)
            return result
        except InvalidRequestError as er:
            print(er)
        except AuthenticationError as er:
            self.log_invalid_key(apikey)
            self.logger.log(f"Stage 2: Invalid apikey: {apikey}")
            return self.stage_2_category(primaries, secondaries)
        except RateLimitError as er:
            print(f"args: {er.args}\nparam: {er.code}, error: {er.error}, header: {er.headers}")
            if er.error['type'] == 'insufficient_quota':
                self.logger.log(f"Stage 2: Invalid apikey: {apikey}")
                self.log_invalid_key(apikey)
                return self.stage_2_category(primaries, secondaries)
            # elif er.error['type'] == 'requests' and er.error['code'] == 'rate_limit_exceeded':
            elif er.error['code'] == 'rate_limit_exceeded':
                # check if remaining requests are not 0
                return self.stage_2_category(primaries, secondaries)
                # if er.headers['x-ratelimit-remaining-requests'] == 0:
                # else:
                #     sleep(20)
                #     return stage_1_thread_handler( apikey, article, site_name, link,)

    def stage_2_save_db(self, csv_filename: str, collection: str, curDate: str):
        data_list = []
        with open(csv_filename, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            for row in csv_reader:
                if not row:
                    continue
                category = row[0]
                data = row[3]
                dictionary = eval(data)
                data_list.append({"category": category, "data": dictionary})

        new_collection = self.db[collection][curDate]
        new_collection.drop()

        result = new_collection.insert_many(data_list)
        self.logger.log(f"Stage 2 - data saved from {csv_filename} into {collection} collection")

    def stage_3(self, category_csv, summary_csv, csv_filename):
        os.remove(csv_filename) if os.path.exists(csv_filename) else None
        with open(csv_filename, 'a', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow([
                'category',
                'topic',
                'research',
                'articles'
            ])
        toprompts = []
        summaries = []
        self.logger.log(f"Stage 3 - Preparing article and topic datas")

        with open(summary_csv, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                summaries.append({
                    "category": row[2],
                    "title": row[1],
                    "summary": row[3],
                    "content": json.loads(row[0])['article'],
                })
        total = 0
        with open(category_csv, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            for row in csv_reader:
                if not row:
                    continue
                category = row[0]
                topics =  eval(row[3])
                print(len(topics))
                if len(topics) < 20:
                    print(category, topics)
                for topic in topics:
                    total += 1
                    primary = topic["Primary"]
                    secondary = topic["Secondary"]
                    articles = []
                    for title in secondary:
                        summary = ""
                        content = ""
                        for ele in summaries:
                            if ele["title"] == title:
                                summary = ele["summary"]
                                content = ele["content"]
                                break
                        articles.append({
                            "title": title,
                            "summary": summary,
                            "content": content,
                        })
                    if len(articles) == 0: continue
                    toprompt = {
                        "topic": primary,
                        "category": category,
                        "articles": articles,
                    }
                    toprompts.append(toprompt)
        self.logger.log(f"Stage 3 - Prepared {len(toprompts)} article and topic data")
        self.logger.log(f"Stage 3 - Start extra research...")

        researched_count = 0
        total = len(toprompts)
        start_t = time()
        while len(toprompts) != 0:
            pool = multiprocessing.Pool(processes=min(len(self.apikeys), 50))
            results: list[ApplyResult] = []

            for i, toprompt in enumerate(toprompts):
                topic = toprompt["topic"]
                category = toprompt["category"]
                articles = toprompt["articles"]
                api_key = self.apikeys[i % len(self.apikeys)]  # Use a different API key for each process
                if len(self.apikeys) > len(toprompts):
                    api_key = random.choice(self.apikeys)
                results.append(pool.apply_async(stage_3_thread_handler, (api_key, category, topic, articles,)))
            toprompts = []
            with open(csv_filename, 'a', newline='', encoding='utf-8') as csvfile:
                writer = csv.writer(csvfile)
                for result in results:
                    article_data = result.get()
                    if article_data[1] == 'APIKey_Error':
                        api_key = article_data[0]
                        self.log_invalid_key(api_key)
                    elif article_data[1] == 'Error':
                        print(f"Statge 3 - Error was occurred in extra research\n: {article_data[0]}")
                    elif article_data[1] == 'UnexpectedError':
                        print(f"Statge 3 - UnexpectedError was occurred in extra research\n: {article_data[0]}")
                        continue
                    else:
                        writer.writerow(article_data[0:-1])
                        researched_count += 1
                        self.logger.log(f"Statge 3 - {researched_count}/{total} : {article_data[-1]}")
                        continue
                    toprompt = {
                        "topic": article_data[2],
                        "category": article_data[3],
                        "articles": article_data[4],
                    }
                    toprompts.append(toprompt)
            pool.close()
            pool.join()
        end_t = time()
        self.logger.log(f'Stage 3 - {researched_count} articles were extra researched in {end_t - start_t} seconds')

    def stage_3_save_db(self, csv_filename: str, collection: str, curDate: str):
        data_list = []
        categories = set()
        researches = []
        with open(csv_filename, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                if not row:
                    continue
                categories.add(row[0])
                researches.append({
                    "category": row[0],
                    "topic": row[1],
                    "research": eval(row[2]),
                })
        for category in categories:
            data_list.append({
                "category": category,
                "data": [{
                    "topic": item["topic"],
                    "research": item["research"]
                } for item in researches if item["category"] == category]
            })
        new_collection = self.db[collection][curDate]
        new_collection.drop()
        result = new_collection.insert_many(data_list)
        self.logger.log(f"Stage 3 - data saved from {csv_filename} into {collection} collection")

    def stage_4(self, stage3_csv: str, csv_filename: str):
        os.remove(csv_filename) if os.path.exists(csv_filename) else None
        with open(csv_filename, 'a', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow([
                'category',
                'topic',
                'background',
                'deep_research'
                'articles'
            ])

        data = []
        categories = set()
        start_t = time()
        self.logger.log(f"Stage 4 - Loading data from {stage3_csv}...")
        with open(stage3_csv, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                if not row:
                    continue
                categories.add(row[0])
                data.append({
                    "category": row[0],
                    "topic": row[1],
                    "research": eval(row[2]),
                    "articles": eval(row[3])
                })
        end_t = time()
        self.logger.log(f"Stage 4 - loaded {len(data)} data from {stage3_csv} in {end_t - start_t} seconds")

        total = len(data)
        start_t = time()
        researched = 0
        while len(data) != 0:
            pool = multiprocessing.Pool(processes=min(len(self.apikeys), 50))
            results: list[ApplyResult] = []

            for i, item in enumerate(data):
                topic = item["topic"]
                category = item["category"]
                research = item["research"]
                articles = item["articles"]
                api_key = self.apikeys[i % len(self.apikeys)]  # Use a different API key for each process
                if len(self.apikeys) > len(data):
                    api_key = random.choice(self.apikeys)
                results.append(pool.apply_async(stage_4_thread_handler, (api_key, category, topic, research, articles,)))
            data = []
            with open(csv_filename, 'a', newline='', encoding='utf-8') as csvfile:
                writer = csv.writer(csvfile)
                for result in results:
                    article_data = result.get()
                    if article_data[1] == 'APIKey_Error':
                        api_key = article_data[0]
                        self.log_invalid_key(api_key)
                    elif article_data[1] == 'Error':
                        print(f"Statge 4 - Error was occurred in deep research\n: {article_data[0]}")
                        continue
                    else:
                        writer.writerow([article_data[0], article_data[1], article_data[2], article_data[3], str(article_data[4])])
                        researched += 1
                        self.logger.log(f"Statge 4 - {researched}/{total} : {article_data[-1]}")
                        continue
                    data.append({
                        "category": article_data[2],
                        "topic": article_data[3],
                        "research": article_data[4],
                        "articles": article_data[5]
                    })
            pool.close()
            pool.join()
        end_t = time()
        self.logger.log(f'Stage 4 - {researched} articles were extra researched in {end_t - start_t} seconds')

    def stage_4_save_db(self, csv_filename: str, collection: str, curDate: str):
        data_list = []
        categories = set()
        dresearches = []
        with open(csv_filename, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                if not row:
                    continue
                categories.add(row[0])
                dresearches.append({
                    "category": row[0],
                    "topic": row[1],
                    "deep_research": eval(row[3]),
                })
        for category in categories:
            print(row[3])
            data_list.append({
                "category": category,
                "data": [{
                    "topic": item["topic"],
                    "deep_research": item["deep_research"]
                } for item in dresearches if item["category"] == category]
            })
        new_collection = self.db[collection][curDate]
        new_collection.drop()
        result = new_collection.insert_many(data_list)
        self.logger.log(f"Stage 4 - data saved from {csv_filename} into {collection} collection")

    def stage_5(self, stage1_csv, csv_filename, timeframe):
        self.logger.log(f'Stage 5 - loading articles from {stage1_csv}')
        articles = []
        categories = set()
        total = 0
        start_t = time()
        with open(stage1_csv, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)

            for row in csv_reader:
                total += 1
                try:
                    if row[1] == '': continue
                    if timeframe == 'day' and row[4] == '': continue
                    if timeframe == 'week' and row[6] == '': continue
                    if timeframe == 'month' and row[8] == '': continue
                    score = 0
                    if timeframe == 'day':
                        score = int(remove_non_numbers_regex(row[4]))
                    elif timeframe == 'week':
                        score = int(remove_non_numbers_regex(row[6]))
                    elif timeframe == 'month':
                        score = int(remove_non_numbers_regex(row[8]))

                    # article = row[0]
                    title = row[1]
                    category = row[2]  # Assuming the category is in the 4th column
                    summary = row[3]
                    categories.add(category)  # Add category to the set of unique categories

                    if any(item['title'] == title for item in articles):
                        continue
                    articles.append({'title': title, 'score': score, 'category': category, 'summary': summary})
                except IndexError as err:
                    self.logger.log(f'Stage 2 - Error while loading articles: {err}')
                    pass
        end_t = time()
        self.logger.log(f'Stage 5 - articles were loaded from {stage1_csv} in {end_t - start_t} second')
        top30 = []
        start_t = time()
        for category in categories:
            if category not in self.categories:
                continue
            category_articles = [article for article in articles if article['category'] == category]
            sorted_articles = sorted(category_articles, key=lambda x: x['score'], reverse=True)
            for tmp in sorted_articles[0:3]:
                top30.append(tmp)
        end_t = time()
        self.logger.log(f'Stage 5 - articles were sorted in {end_t - start_t} second')
        start_t = time()
        result = []
        try:
            result = self.stage_5_impactful_news(top30)
        except Exception as er:
            error = str(traceback.print_exc())
            self.logger.log(f"Stage 5: Error : {er} in {error}")
        self.logger.log(f'Stage 5 - {result[1]}')
        with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(["no", "title", "explanation"])
            print(result[0])
            tops: list = eval(result[0])
            print(tops)
            for i, top in enumerate(tops):
                writer.writerow([i+1, top['title'], top['explanation']])

        end_t = time()
        self.logger.log(f'Stage 5 - got the result in {end_t - start_t} second')

    def stage_5_impactful_news(self, articles):
        apikey = random.choice(self.apikeys)
        try:
            result = impactul_news(apikey=apikey, articles=articles)
            try:
                eval(result[0])
            except NameError as er:
                return self.stage_5_impactful_news(articles)
            except SyntaxError as er:
                return self.stage_5_impactful_news(articles)
            return result
        except InvalidRequestError as er:
            result = self.stage_5_impactful_news(articles)
            return result
        except AuthenticationError as er:
            self.log_invalid_key(apikey)
            self.logger.log(f"Stage 5: Invalid apikey: {apikey}")
            return self.stage_5_impactful_news(articles)
        except RateLimitError as er:
            print(f"args: {er.args}\nparam: {er.code}, error: {er.error}, header: {er.headers}")
            if er.error['type'] == 'insufficient_quota':
                self.logger.log(f"Stage 5: Invalid apikey: {apikey}")
                self.log_invalid_key(apikey)
                return self.stage_5_impactful_news(articles)
            elif er.error['code'] == 'rate_limit_exceeded':
                return self.stage_5_impactful_news(articles)

    def stage_5_save_db(self, csv_filename: str, collection: str, curDate: str):
        data_list = []
        with open(csv_filename, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                if not row:
                    continue
                title = row[1]
                explanation = row[2]
                data_list.append({"title": title, "explanation": explanation})

        new_collection = self.db[collection][curDate]
        new_collection.drop()
        result = new_collection.insert_many(data_list)
        self.logger.log(f"Stage 5 - data saved from {csv_filename} into {collection} collection")

    def stage_6(self, stage4_csv: str, csv_filename: str, timeframe: str):
        self.logger.log(f'Stage 6 - loading topics from {stage4_csv}')
        topics = []
        data_list = []
        categories = set()
        start_t = time()
        with open(stage4_csv, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                if not row:
                    continue
                categories.add(row[0])
                deep_research = eval(row[3])
                topics.append({
                    "category": row[0],
                    "topic": row[1],
                    "prediction": f"Description: {deep_research['1 day timeframe']['Most likely']['Description']}\nExplanation: {deep_research['1 day timeframe']['Most likely']['Explanation']}",
                })
        for category in categories:
            if category not in self.categories:
                continue
            data_list.append({
                "category": category,
                "data": [{
                    "topic": topic["topic"],
                    "prediction": topic["prediction"]
                } for topic in topics if topic["category"] == category]
            })
        data_list.append({
            "category": "at_glance",
            "data": [{
                "topic": topic["topic"],
                "prediction": topic["prediction"]
            } for topic in topics]
        })
        end_t = time()
        self.logger.log(f'Stage 6 - topics were loaded from {stage4_csv} in {end_t - start_t} second')
        start_t = time()
        results = []
        try:
            for i, data in enumerate(data_list):
                result = self.stage_6_prediction(data, timeframe)
                results.append([result[0], eval(result[1][0])])
                self.logger.log(f'Stage 6 - {i+1}/{len(data_list)} - {result[1][1]}')
        except Exception as er:
            error = str(traceback.print_exc())
            self.logger.log(f"Stage 6: Error : {er} in {error}")
        with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(["category", "prediction"])
            for result in results:
                writer.writerow(result)

        end_t = time()
        self.logger.log(f'Stage 6 - got the result in {end_t - start_t} second')

    def stage_6_prediction(self, topics, timeframe):
        apikey = random.choice(self.apikeys)
        try:
            result = prediction(apikey, topics["data"], topics["category"], timeframe)
            return [topics["category"], result]
        except InvalidRequestError as er:
            result = self.stage_6_prediction(topics, timeframe)
            return result
        except AuthenticationError as er:
            self.log_invalid_key(apikey)
            self.logger.log(f"Stage 6: Invalid apikey: {apikey}")
            return self.stage_6_prediction(topics, timeframe)
        except RateLimitError as er:
            print(f"args: {er.args}\nparam: {er.code}, error: {er.error}, header: {er.headers}")
            if er.error['type'] == 'insufficient_quota':
                self.logger.log(f"Stage 6: Invalid apikey: {apikey}")
                self.log_invalid_key(apikey)
                return self.stage_6_prediction(topics, timeframe)
            elif er.error['code'] == 'rate_limit_exceeded':
                return self.stage_6_prediction(topics, timeframe)

    def stage_6_save_db(self, csv_filename: str, collection: str, curDate: str):
        data_list = []
        with open(csv_filename, 'r', encoding='utf-8') as file:
            csv_reader = csv.reader(file)
            next(csv_reader)
            for row in csv_reader:
                if not row:
                    continue
                data_list.append({"category": row[0], "prediction": eval(row[1])})

        new_collection = self.db[collection][curDate]
        new_collection.drop()
        result = new_collection.insert_many(data_list)
        self.logger.log(f"Stage 6 - data saved from {csv_filename} into {collection} collection")

def stage_1_thread_handler(
        apikey: str,
        article: str,
        site_name: str,
        link: str,
        rCategory: str,
    ):
    items = [
        'Title:',
        'Category:',
        'Summary:',
        'Importance 1 day:',
        'Reasoning for 1 day score:',
        'Importance 1 week:',
        'Reasoning for 1 week score:',
        'Importance 1 month:',
        'Reasoning for 1 month score:',
    ]

    item_dict = {
        'Title:': None,
        'Category:': rCategory,
        'Summary:': None,
        'Importance 1 day:': None,
        'Reasoning for 1 day score:': None,
        'Importance 1 week:': None,
        'Reasoning for 1 week score:': None,
        'Importance 1 month:': None,
        'Reasoning for 1 month score:': None
    }

    try:
        summary = summarize_article(apikey, article)
        for item in items:
            for line in summary[0].split('\n'):
                if line.startswith(item):
                    content = line[len(item):].strip()
                    item_dict[item] = content
                    break
        baseData = {
            'article': article,
            'result': summary[0]
        }
        return [
            json.dumps(baseData),
            item_dict['Title:'],
            rCategory,
            item_dict['Summary:'],
            item_dict['Importance 1 day:'],
            item_dict['Reasoning for 1 day score:'],
            item_dict['Importance 1 week:'],
            item_dict['Reasoning for 1 week score:'],
            item_dict['Importance 1 month:'],
            item_dict['Reasoning for 1 month score:'],
            site_name,
            link,
            summary[1]
        ]
    except InvalidRequestError as er:
        return [er, 'Error', article, site_name, link, rCategory]
    except RateLimitError as er:
        print(f"args: {er.args}\nparam: {er.code}, error: {er.error}, header: {er.headers}")
        if er.error['type'] == 'insufficient_quota':
            return [apikey, 'APIKey_Error', article, site_name, link, rCategory]
        # elif er.error['type'] == 'requests' and er.error['code'] == 'rate_limit_exceeded':
        elif er.error['code'] == 'rate_limit_exceeded':
            # check if remaining requests are not 0
            return [er, 'Error', article, site_name, link, rCategory]
            # if er.headers['x-ratelimit-remaining-requests'] == 0:
            # else:
            #     sleep(20)
            #     return stage_1_thread_handler( apikey, article, site_name, link,)
    except AuthenticationError as er:
        return [apikey, 'APIKey_Error', article, site_name, link, rCategory]
    except Exception as er:
        # if 'Limit: 200 / day' in str(er):
        #     sleep(450)
        #     return stage_1_thread_handler( apikey, article, site_name, link,)
        # if 'Limit: 3 / min' in str(er):
        #     sleep(20)
        #     return stage_1_thread_handler( apikey, article, site_name, link,)
        # if 'Limit: 3 / min' in str(er) and 'Rate limit reached' in str(er):
        #     return stage_1_thread_handler(str, article, site_name, link)
        return [er, 'Error', article, site_name, link, rCategory]

def stage_3_thread_handler(
        apikey: str,
        category: str,
        topic: str,
        articles: dict,
    ):
    try:
        contents = [article['content'] for article in articles]
        summary = extra_research(apikey, contents)
        return [category, topic, json.loads(summary[0].replace("\n", " ")), articles,  summary[1]]
    except InvalidRequestError as er:
        return [er, 'Error', category, topic, articles]
    except RateLimitError as er:
        print(f"args: {er.args}\nparam: {er.code}, error: {er.error}, header: {er.headers}")
        if er.error['type'] == 'insufficient_quota':
            return [apikey, 'APIKey_Error', category, topic, articles]
        elif er.error['code'] == 'rate_limit_exceeded':
            return [er, 'Error', category, topic, articles]
    except AuthenticationError as er:
        return [apikey, 'APIKey_Error', category, topic, articles]
    except Exception as er:
        error = str(traceback.print_exc())
        print(error)
        return [er, 'UnexpectedError', category, topic, articles]

def stage_4_thread_handler(
        apikey: str,
        category: str,
        topic: str,
        research: dict,
        articles: list,
    ):
    try:
        summary = deep_research(apikey, articles, background=research)
        eval(summary[0])
        return [category, topic, research, summary[0], articles,  summary[1]]
    except InvalidRequestError as er:
        return [er, 'Error', category, topic, research, articles]
    except RateLimitError as er:
        print(f"args: {er.args}\nparam: {er.code}, error: {er.error}, header: {er.headers}")
        if er.error['type'] == 'insufficient_quota':
            return [apikey, 'APIKey_Error', category, topic, research, articles]
        elif er.error['code'] == 'rate_limit_exceeded':
            return [er, 'Error', category, topic, research, articles]
    except AuthenticationError as er:
        return [apikey, 'APIKey_Error', category, topic, research, articles]
    except Exception as er:
        error = str(traceback.print_exc())
        print(error)
        return [er, 'Error', category, topic, research, articles]


from transformers import BertTokenizer, BertForSequenceClassification, BertModel
import torch
from torch import nn

class BertHumorNetwork(nn.Module):
    def __init__(self):
        super(BertHumorNetwork, self).__init__()
        model_name = "bert-base-uncased"
        self.bert = BertModel.from_pretrained(model_name, num_labels=2)

        self.linear1 = nn.Linear(768, 256)
        self.relu = nn.LeakyReLU(0.3)
        self.linear2 = nn.Linear(256, 1)

        self.sigmoid = nn.Sigmoid()

    def forward(self, ids, mask):
        output = self.bert(ids, mask)
        # print("AAA")
        # print(output[0].shape)

        # sequence_output has the following shape: (batch_size, sequence_length, 768)
        linear1_output = self.linear1(output[0][:, 0, :].view(-1, 768))   # extract the 1st token's embeddings

        relu_output = self.relu(linear1_output)

        linear2_output = self.linear2(relu_output)

        sigmoid_output = 3 * self.sigmoid(linear2_output)

        sigmoid_output = sigmoid_output.squeeze()

        return sigmoid_output.float()

class MixedBertHumorNetwork(nn.Module):
    def __init__(self):
        super(MixedBertHumorNetwork, self).__init__()
        model_name = "bert-base-uncased"
        self.bert = BertModel.from_pretrained(model_name, num_labels=2)

        self.bertlinear1 = nn.Linear(768, 256)

        self.dropout = nn.Dropout()

        self.relu = nn.LeakyReLU(0.3)

        self.baseline_linear1 = nn.Linear(12, 12)

        self.linear2 = nn.Linear(256 + 12, 1)

        self.sigmoid = nn.Sigmoid()

        for param in self.bert.parameters():
            param.requires_grad = False

        self.bertlinear1.requires_grad = True

        self.baseline_linear1.requires_grad = True

        self.linear2.requires_grad = True

    def forward(self, ids, mask, baseline_vec):
        baseline_vec = baseline_vec / torch.norm(baseline_vec)
        output = self.bert(ids, mask)
        # print("AAA")
        # print(output[0].shape)

        # sequence_output has the following shape: (batch_size, sequence_length, 768)
        bert_linear1_output = self.bertlinear1(output[0][:, 0, :].view(-1, 768))   # extract the 1st token's embeddings

        bert_linear1_output = self.dropout(bert_linear1_output)

        relu_bert_output = self.relu(bert_linear1_output)

        baseline_linear_output = self.baseline_linear1(baseline_vec)

        baseline_linear_output = self.dropout(baseline_linear_output)

        relu_baseline_output = self.relu(baseline_linear_output)

        cat_vec = torch.cat((relu_baseline_output, relu_bert_output), 1)

        linear2_output = self.linear2(cat_vec)

        sigmoid_output = 3 * self.sigmoid(linear2_output)

        sigmoid_output = sigmoid_output.squeeze()

        return sigmoid_output.float()


import itertools
import multiprocessing as mp
import os
import shutil
from functools import partial

import numpy as np
from astropy.table import Table

import pypetal.pyzdcf.utils as utils
from pypetal.pyzdcf.plotting import plot_pyzdcf_results
from pypetal.utils import defaults

#For multiprocessing
def mp_map(func, args, threads):
    n_inputs = len(args)

    if (threads > 1) & (n_inputs > 1):
        pool = mp.get_context('fork').Pool(threads)
        res = pool.starmap(func, args)

        pool.close()
        pool.join()

    else:
        res = list(itertools.starmap(func, args))

    return res

def pyzdcf_tot(cont_fname, line_fnames, line_names, output_dir,
               general_kwargs, kwargs):

    if line_fnames is str:
        line_fnames = [line_fnames]
        line_names = [line_names]

    #--------------------------------------------------
    #Read general kwargs

    verbose = general_kwargs['verbose']
    plot = general_kwargs['plot']
    time_unit = general_kwargs['time_unit']
    lc_unit = general_kwargs['lc_unit']
    lag_bounds = general_kwargs['lag_bounds']
    threads = general_kwargs['threads']

    #--------------------------------------------------
    #Read kwargs

    nsim, minpts, uniform_sampling, omit_zero_lags, \
        sparse, prefix, run_plike, plike_dir = defaults.set_pyzdcf(kwargs,
                                                                   np.hstack([ [cont_fname], line_fnames ])
                                                                   )

    #-------------------------------------------
    if (run_plike) & (plike_dir is None):
        print('Error: plike_dir must be specified if run_plike=True')
        print('Skipping PLIKE')
        run_plike = False

    input_dir = os.path.dirname( os.path.realpath(cont_fname) )

    for i in range(len(line_fnames)):
        if input_dir != os.path.dirname( os.path.realpath(line_fnames[i]) ):
            print('ERROR: All light curve files must be in the same directory')
            return {}

    if verbose:

        txt_str = """
Running pyZDCF
----------------------
nsim: {}
minpts: {}
uniform_sampling: {}
omit_zero_lags: {}
sparse: {}
prefix: {}
run_plike: {}
plike_dir: {}
----------------------
        """.format( nsim, minpts, uniform_sampling, omit_zero_lags,
                    sparse, prefix, run_plike, plike_dir)

        print(txt_str)

    input_dir += r'/'
    cont_fname_short = os.path.basename(cont_fname)
    line_fnames_short = [ os.path.basename(x) for x in line_fnames ]

    pyzdcf_func = partial( utils.get_zdcf, num_MC=nsim, minpts=minpts,
                           uniform_sampling=uniform_sampling, omit_zero_lags=omit_zero_lags,
                           sparse=sparse, sep=',', verbose=False)

    arg1 = np.full( len(line_fnames), input_dir )
    arg2 = np.full( len(line_fnames), cont_fname_short )
    arg3 = line_fnames_short
    arg4 = [ output_dir + x + r'/pyzdcf/' for x in line_names[1:] ]
    arg5 = [ x + '_' + prefix for x in line_names[1:] ]

    args = list(zip(arg1, arg2, arg3, arg4, arg5))
    res_tot = mp_map(pyzdcf_func, args, threads)

    plike_tot = []
    for i in range(len(line_fnames)):

        x1, y1, yerr1 = np.loadtxt( cont_fname, delimiter=',', unpack=True, usecols=[0,1,2] )
        x2, y2, yerr2 = np.loadtxt( line_fnames[i], delimiter=',', unpack=True, usecols=[0,1,2] )

        plike_dict = None
        if run_plike:

            utils.run_plike( arg4[i] + arg5[i] + '.dcf', lag_bounds[i], plike_dir,
                            verbose=verbose)

            plike_fname = plike_dir + 'plike.out'
            output_fname = output_dir + line_names[i+1] + r'/pyzdcf/' + line_names[i+1] + '_plike.out'
            assert os.path.exists( os.path.dirname(output_fname) )

            shutil.move( plike_fname, output_fname )
            plike_dat = Table.read( output_fname,
                                    format='ascii',
                                    names=['num', 'lag', 'r', '-dr', '+dr', 'likelihood'])

            file = open(output_fname, 'r')
            output_str = list(file)[-3:]

            ml_lag = float( output_str[1].split()[7] )
            ml_lag_err_hi = np.abs( float( output_str[1].split()[8] )  )
            ml_lag_err_lo = np.abs( float( output_str[1].split()[9] )  )

            plike_dict = {
                'output': plike_dat,
                'ML_lag': ml_lag,
                'ML_lag_err_hi': ml_lag_err_hi,
                'ML_lag_err_lo': ml_lag_err_lo
            }

            plike_tot.append( plike_dict )

        plot_pyzdcf_results(x1, y1, yerr1, x2, y2, yerr2,
                                     res_tot[i], plike_dict,
                                     time_unit=time_unit, lc_unit=[lc_unit[0], lc_unit[i+1]],
                                     lc_names=[line_names[0], line_names[i+1]],
                                     fname=output_dir + line_names[i+1] + r'/pyzdcf/' + line_names[i+1] + '_zdcf.pdf',
                                     show=plot)

    return res_tot, plike_tot


from kafka import KafkaProducer
import os

#9092:32452/TCP
producer = KafkaProducer(bootstrap_servers=os.environ.get('KAFKA_HOST', 'node1:32452,node2:32452,node3:32452'))
print("------->>")
future = producer.send('test', key=b'my_key', value=b'my_value', partition=0)
result = future.get(timeout=100)
print(result)


from selenium import webdriver
import time
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.by import By

with open('job_scraping_multipe_pages.csv', 'w') as file:
    file.write("Job_title; Company; Salary; Location; Job_description \n")

driver=webdriver.Chrome()

# driver.get('https://www.jobsite.co.uk/')

driver.get('https://www.jobsite.co.uk/jobs/data-engineer/in-united-kingdom?radius=30') # The particular url I wish to scrap

driver.maximize_window()
time.sleep(5)

# //*[@id="ccmgt_explicit_accept"]/span

cookie= driver.find_element(By.XPATH, '//*[@id="ccmgt_explicit_accept"]/span') # Locate the button to accept cookies
try:
    cookie.click()
except:
    pass

time.sleep(5)
# job_title=driver.find_element(By.ID, 'keywords')
# job_title.click()
# job_title.send_keys('Software Engineer')
# time.sleep(1)

# location=driver.find_element(By.ID, 'location')
# location.click()
# location.send_keys('Manchester')
# time.sleep(1)

# dropdown=driver.find_element(By.ID, 'Radius')
# radius=Select(dropdown)
# radius.select_by_visible_text('30 miles')
# time.sleep(1)

# search=driver.find_element(By.ID, 'search-button')
# search.click()
# time.sleep(50)

for k in range(5):
    titles=driver.find_elements(By.XPATH, '//div[@class="resultlist-dpthza"]')
    company=driver.find_elements(By.XPATH, '//div[@class="resultlist-1mjyynf"]')
    salary=driver.find_elements(By.XPATH, '//div[@class="resultlist-noiqwp"]/span')
    location=driver.find_elements(By.XPATH, '//div[@class="resultlist-5dwl9c"]')
    job_details=driver.find_elements(By.XPATH, '//div[@class="resultlist-1n63zbq"]')

    with open('job_scraping_multipe_pages.csv', 'a') as file:
        for i in range(len(titles)):
            file.write(titles[i].text + ";" + company[i].text + ";" + salary[i].text + ";" + location[i].text + ";"+
                      job_details[i].text + "\n")

        next=driver.find_element(By.XPATH, '//a[@aria-label="Next"]')
        next.click()
        time.sleep(7)
    file.close()
driver.close()


# -*- coding: utf-8 -*-
"""
Created on Sat Jul  2 11:29:33 2022

@author: Seshu Kumar Damarla
"""

"""
Linear Discriminant Analysis for multi-class classification problem
"""

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import gc

gc.collect()

data = pd.read_csv('data.csv',header=None)

xydata = np.array(data)
"""" test observation """
test_x=np.array([[6,130,8]])

xdata=xydata[:,1:]
N=xydata.shape[0]    # no. of examples

# noralization
xmax = np.amax(xdata, axis = 0, keepdims=True)
xmin = np.amin(xdata, axis = 0, keepdims=True)

xdata = (xdata - xmin) / (xmax-xmin)
print(xdata)

dd=xydata[:,0] == 1
c1xdata=xdata[dd==True,:]
c2xdata=xdata[dd==False,:]

C=np.unique(xydata[:,0])
C=C.astype(int)
C=np.array([C])

mean_class1 = np.mean(c1xdata, axis=0, keepdims=True, dtype=np.float)
mean_class2 = np.mean(c2xdata, axis=0, keepdims=True, dtype=np.float)

#print(mean_class1)
#print(np.transpose(mean_class1))

cov_class1 = np.cov(c1xdata,rowvar=False)
cov_class2 = np.cov(c2xdata,rowvar=False)

S=(1/(N-C.shape[1]))*(cov_class1 + cov_class2)
#print(S)
#print(np.linalg.inv(S))

prior=np.zeros([1,C.shape[1]])
prior[:,0] = c1xdata.shape[0] / N
prior[:,1] = c2xdata.shape[0] / N

#print(np.log(prior[:,0]))
# discriminate function

#delta_class1 = test_x * (np.linalg.inv(S)) * np.transpose(mean_class1) -(1/2) *(mean_class1) * (np.linalg.inv(S)) * np.transpose(mean_class1) + np.log(prior[:,0])
#delta_class2 = test_x * (np.linalg.inv(S)) * np.transpose(mean_class2) -(1/2) *(mean_class2) * (np.linalg.inv(S)) * np.transpose(mean_class2) + np.log(prior[:,1])

test_x = (test_x -xmean) / xstd

delta_class1 = np.dot(np.dot(test_x,np.linalg.inv(S)),np.transpose(mean_class1)) - (1/2) * np.dot(np.dot(mean_class1,np.linalg.inv(S)),np.transpose(mean_class1)) + np.log(prior[:,0])
delta_class2 = np.dot(np.dot(test_x,np.linalg.inv(S)),np.transpose(mean_class2)) - (1/2) * np.dot(np.dot(mean_class2,np.linalg.inv(S)),np.transpose(mean_class2)) + np.log(prior[:,0])

print(delta_class1)
print(delta_class2)

if delta_class1 > delta_class2:
    test_class = 1
else:
    test_class = 0

print(test_class)


class Solution(object):
    def searchMatrix(self, matrix, target):
        """
        :type matrix: List[List[int]]
        :type target: int
        :rtype: bool
        """
        if not matrix or not matrix[0]:
            return False

        rows, cols = len(matrix), len(matrix[0])
        left, right = 0, rows * cols - 1

        while left <= right:
            mid = (left + right) // 2
            mid_val = matrix[mid // cols][mid % cols]

            if mid_val == target:
                return True
            elif mid_val < target:
                left = mid + 1
            else:
                right = mid - 1

        return False


def count_bases(seq):
    """Counting the number of bases in the string"""
    count_a = seq.count("A")
    count_t = seq.count("T")
    count_c = seq.count("C")
    count_g = seq.count("G")

    seq_dict = {"A": count_a, "T":count_t, "C": count_c, "G": count_g}
    return seq_dict

def calc_perc(dict):
    for k, v in dict.items():
        if v > 0:
            perc = (round(100.0 * v / tl, 1))
        else:
            perc = 0

        print("Base", k)
        print("Counter: ", v)
        print("Percentage: ", perc)

s1 = input("Enter the sequence 1: ")
s2 = input("Enter the sequence 2: ")
sequences = [s1, s2]
bases_dicts = []

for i in range(len(sequences)):
    bases_dicts.append(count_bases(sequences[i]))
    tl = len(sequences[i])
    print("This sequence", i+1 ,"is {} bases in length".format(tl))
    calc_perc(bases_dicts[i])


from unittest import TestCase, mock

from api.translation_v2.exceptions import UnsupportedLanguageError
from api.translation_v2.functions import translate_using_convergent_definition
from api.translation_v2.types import ConvergentTranslation

class TestTranslateUsingConvergentDefinition(TestCase):
    @mock.patch('api.translation_v2.functions.definitions.database_based.convergent_translations')
    def test_translate_using_convergent_definition_1(self, convergent_translations):
        # Test case 1: Successful translation from English to Spanish
        convergent_translations.get_convergent_translation.return_value = [{'suggested_definition': 'translation 1'}]
        result = translate_using_convergent_definition(part_of_speech='ana', definition_line='test definition',
                                                       source_language='en', target_language='mg')
        convergent_translations.get_convergent_translation.assert_called_once()

    @mock.patch('api.translation_v2.functions.definitions.database_based.convergent_translations')
    def test_translate_using_convergent_definition_2(self, convergent_translations):
        # Test case 2: Successful translation from French to Portuguese
        convergent_translations.get_convergent_translation.return_value = [{'suggested_definition': 'translation 2'}]
        result = translate_using_convergent_definition(part_of_speech='mat', definition_line='test definition2',
                                                       source_language='fr', target_language='mg')
        convergent_translations.get_convergent_translation.assert_called_once()
        self.assertEqual(result, 'translation 2')
        self.assertIsInstance(result, ConvergentTranslation)

    @mock.patch('api.translation_v2.functions.definitions.convergent_translations')
    def test_translate_using_convergent_definition_3(self, mock_get_convergent_translation):
        # Test case 3: Unsupported source language
        with self.assertRaises(UnsupportedLanguageError):
            translate_using_convergent_definition('mat', 'test definition', 'de', 'es')


#!/usr/bin/python
#
#

with open('day2-input.txt', 'r') as f:
    lines = [ l.strip() for l in list(f) ]
#
#lines = '''abcdef
#bababc
#abbcde
#abcccd
#aabcdd
#abcdee
#ababab'''
#lines = lines.splitlines()
# example res.: 12
lines_ = '''abcde
fghij
klmno
pqrst
fguij
axcye
wvxyz'''
lines_ = lines_.splitlines()
# example res.: fgij
#print(lines)

n_two_letters = 0
n_three_letters = 0

for line in lines:
    strcheck = {}
    for c in line:
        if c not in strcheck.keys():
            strcheck[c] = 1
        else:
            strcheck[c] += 1
    match_two = False
    match_three = False
    #print(line)
    for v in strcheck.values():
        if not match_two and v == 2:
            print("match two")
            n_two_letters += 1
            match_two = True
        if not match_three and v == 3:
            print("match three")
            n_three_letters += 1
            match_three = True
        if match_two and match_three:
            break

print("res:", n_two_letters*n_three_letters)

lines_match = []
found = False
for line in lines:
    for line_ in lines:
        if line != line_:
            diff_c = 0
            for i, c in enumerate(line):
                #print("comp {} {}".format(c, line_[i]))
                if c != line_[i]:
                    diff_c += 1
                if diff_c > 1: break
            #print(diff_c)
            if diff_c == 1:
                lines_match.append(line)
                lines_match.append(line_)
                found = True
                break
    if found: break
print(lines_match)
for n, c in enumerate(lines_match[0]):
    if c != lines_match[1][n]:
        x = n
        print(x)
        break
res = lines_match[0][:x] + lines_match[0][x+1:]
print("res2:", res)


#!/usr/bin/env python

"""
Example on how to train a probabilistic autoencoder using pytorch-genotypes.

./train_genotype_autoencoder.py \
    --n-variants 1664852 \
    --chunk-size 5000 \
    --enc-hidden 10 128 32 \
    --enc-activations LeakyReLU \
    --dec-hidden 32 128 10 \
    --dec-activations LeakyReLU \
    --lr 1e-3 \
    --weight-decay 1e-4 \
    --backend 1kg_common_norel.pkl \
    --test-proportion 0.1

"""

import argparse

import pytorch_lightning as pl
import torch.nn as nn
from torch.utils.data import DataLoader, random_split

from pytorch_genotypes.models import GenotypeProbabilisticAutoencoder
from pytorch_genotypes.models.utils import build_mlp
from pytorch_genotypes.modules import (
    ChunkPartiallyConnected,
    ChunkDropout
)
from pytorch_genotypes.dataset import BACKENDS, GeneticDataset

def train_genotype_autoencoder(args, _backend=None):
    print("Initializing weights")

    # Parse encoder arguments and initialize.
    enc_layers = [
        nn.BatchNorm1d(args.n_variants),
        ChunkPartiallyConnected(
            args.n_variants,
            args.enc_hidden[0],
            chunk_size=5000
        )
    ]

    hidden_layer_sizes = args.enc_hidden[1:]
    output_layer_size = hidden_layer_sizes.pop()

    enc_layers.extend(build_mlp(
        enc_layers[-1].get_effective_output_size(),
        hidden_layer_sizes,
        output_layer_size,
        add_batchnorm=args.enc_add_batchnorm,
    ))

    enc_layers = nn.Sequential(*enc_layers)

    encoder = nn.Sequential(*enc_layers)
    decoder = nn.Linear(7, args.n_variants, bias=False)

    print("Initializing backend")

    # Initialize backend.
    if _backend is None:
        backend = BACKENDS["NumpyBackend"].load(args.backend)
    else:
        backend = _backend

    print("Making datasets")

    # Make it a dataset.
    if args.test_proportion > 0:
        n_test = round(len(backend) * args.test_proportion)
        n_train = len(backend) - n_test

        train_dataset, test_dataset = random_split(
            GeneticDataset(backend, genotype_standardization=False),
            [n_train, n_test]
        )

    else:
        train_dataset = GeneticDataset(backend, genotype_standardization=False)
        test_dataset = None

    print("Initialize final model and start training.")

    # Initialize the autoencoder.
    autoencoder = GenotypeProbabilisticAutoencoder(
        lr=args.lr,
        weight_decay=args.weight_decay,
        encoder=encoder,
        decoder=decoder
    )

    print(autoencoder)

    # Train the model.
    trainer = pl.Trainer.from_argparse_args(args)

    print("Starting fit.")
    trainer.fit(
        autoencoder,
        DataLoader(train_dataset, batch_size=65)
    )
    del train_dataset

    print("Saving checkpoint...")
    trainer.save_checkpoint("_autoencoder.ckpt")

    if test_dataset:
        trainer.test(
            autoencoder,
            DataLoader(test_dataset, batch_size=len(test_dataset) // 5)
        )

def parse_args():
    parser = argparse.ArgumentParser()

    # Add architecture parameters.
    parser.add_argument("--n-variants", required=True, type=int)
    parser.add_argument("--chunk-size", required=True, type=int, default=2000)

    enc = parser.add_argument_group(title="Encoder architecture.")
    enc.add_argument("--enc-hidden", nargs="+", required=True, type=int)
    enc.add_argument("--enc-activations", nargs="+", required=True, type=str)
    enc.add_argument("--enc-add-batchnorm", action="store_true")

    dec = parser.add_argument_group(title="Decoder architecture.")
    dec.add_argument("--dec-hidden", nargs="+", required=True, type=int)
    dec.add_argument("--dec-activations", nargs="+", required=True, type=str)
    dec.add_argument("--dec-add-batchnorm", action="store_true")

    # Add training hyperparameters.
    trainer = parser.add_argument_group(title="Trainer parameters.")
    trainer.add_argument("--lr", help="Learning rate", type=float,
                         required=True)
    trainer.add_argument("--weight-decay", help="Weight decay", type=float,
                         required=True)

    trainer.add_argument("--test-proportion", type=float, default=0.1,
                         help="Proportion of samples to use as test set.")

    # Add genotype backend.
    geno = parser.add_argument_group(title="Genotype parameters.")
    geno.add_argument(
        "--backend",
        type=str,
        required=True,
        help="Path to pickle containing NumpyBackend.\n"
             "For now, we only allow this backend to allow easy sample "
             "splitting which is hard to implement with the ZarrBackend for "
             "performance reasons."
    )

    # Add trainer parameters.
    # pl.Trainer.add_argparse_args(parser)
    trainer = parser.add_argument_group(title="pl.Trainer")
    trainer.add_argument("--accelerator", default=None)
    trainer.add_argument("--devices", default=None)
    trainer.add_argument("--log-every-n-steps", default=1, type=int)

    return parser.parse_args()

if __name__ == "__main__":
    args = parse_args()
    train_genotype_autoencoder(args)


#!/usr/bin/python

import os
import shutil

inDirBase = '/home/disk/funnel/impacts-website/archive/research'
outDirBase = '/home/disk/funnel/impacts-website/archive_ncar/surface'
category_new = 'surface'
platforms = {'stonybrook':'Pluvio',
             'manhattan':'Parsivel',
             'stonybrookmobile':'Parsivel',
             'ualbany':'Parsivel'}
SBU_products = {'pluvio':'Stonybrook_NY'}
manhattan_products = {'parsivel':'Manhattan_NY'}
SBUM_products = {'parsivel':'Stonybrook_NY_Mobile'}
UAlbany_products = {'parsivel':'UAlbany'}

for platform in platforms.keys():

    # get appropriate product dictionary
    if platform == 'stonybrook':
        products = SBU_products
    elif platform == 'manhattan':
        products = manhattan_products
    elif platform == 'stonybrookmobile':
        products = SBUM_products
    else:
        products = UAlbany_products

    # make output product dir
    if not os.path.isdir(outDirBase+'/'+platforms[platform]):
        os.mkdir(outDirBase+'/'+platforms[platform])

    # go through dates & files
    for date in os.listdir(inDirBase+'/'+platform):
        if not os.path.isdir(outDirBase+'/'+platforms[platform]+'/'+date):
            os.mkdir(outDirBase+'/'+platforms[platform]+'/'+date)
        for file in os.listdir(inDirBase+'/'+platform+'/'+date):
            (basename,ext) = os.path.splitext(file)
            if file.endswith(products.keys()[0]+ext):
                print('file = '+file)
                (category_orig,platform_orig,datetime,product_orig) = basename.split('.')
                file_new = category_new+'.'+platforms[platform]+'.'+datetime+'.'+products[product_orig]+ext
                print('file_new = '+file_new)
                shutil.copy(inDirBase+'/'+platform+'/'+date+'/'+file,
                            outDirBase+'/'+platforms[platform]+'/'+date+'/'+file_new)


from pathlib import Path

from torchvision.datasets import CIFAR10, CIFAR100, GTSRB, SVHN

DATA_ROOT: Path = Path('data')

def cal_file_sha256(file_path: str | Path) -> str:
    import hashlib

    BLOCKSIZE = 65536
    sha256_hash = hashlib.sha256()

    with open(file_path, 'rb') as f:
        block = f.read(BLOCKSIZE)
        while block:
            sha256_hash.update(block)
            block = f.read(BLOCKSIZE)

    return sha256_hash.hexdigest()

def download_cifar10() -> dict:
    CIFAR10(root=DATA_ROOT, download=True, train=True)
    CIFAR10(root=DATA_ROOT, download=True, train=False)

    path = DATA_ROOT / CIFAR10.filename

    return dict(name='cifar10',
                path=path.as_posix(),
                sha256=cal_file_sha256(path))

def download_cifar100() -> dict:
    CIFAR100(root=DATA_ROOT, download=True, train=True)
    CIFAR100(root=DATA_ROOT, download=True, train=False)

    path = DATA_ROOT / CIFAR100.filename

    return dict(name='cifar100',
                path=path.as_posix(),
                sha256=cal_file_sha256(path))

def download_gtsrb() -> dict:
    GTSRB(root=DATA_ROOT, download=True, split='train')
    GTSRB(root=DATA_ROOT, download=True, split='test')

    train_path = DATA_ROOT / 'gtsrb' / 'GTSRB-Training_fixed.zip'
    test_images_path = DATA_ROOT / 'gtsrb' / 'GTSRB_Final_Test_Images.zip'
    test_gt_path = DATA_ROOT / 'gtsrb' / 'GTSRB_Final_Test_GT.zip'

    return dict(name='gtsrb',
                train=dict(path=train_path.as_posix(),
                           sha256=cal_file_sha256(train_path)),
                test=[
                    dict(path=test_images_path.as_posix(),
                         sha256=cal_file_sha256(test_images_path)),
                    dict(path=test_gt_path.as_posix(),
                         sha256=cal_file_sha256(test_gt_path))
                ])

def download_svhn() -> dict:
    SVHN(root=DATA_ROOT / 'svhn', download=True, split='train')
    SVHN(root=DATA_ROOT / 'svhn', download=True, split='test')
    SVHN(root=DATA_ROOT / 'svhn', download=True, split='extra')

    train_path = DATA_ROOT / 'svhn' / 'train_32x32.mat'
    test_images_path = DATA_ROOT / 'svhn' / 'test_32x32.mat'
    test_gt_path = DATA_ROOT / 'svhn' / 'extra_32x32.mat'

    return dict(name='svhn',
                train=dict(path=train_path.as_posix(),
                           sha256=cal_file_sha256(train_path)),
                test=dict(path=test_images_path.as_posix(),
                          sha256=cal_file_sha256(test_images_path)),
                extra=dict(path=test_gt_path.as_posix(),
                           sha256=cal_file_sha256(test_gt_path)))

if __name__ == '__main__':
    import json

    meta_dict = []
    meta_dict.append(download_cifar10())
    meta_dict.append(download_cifar100())
    meta_dict.append(download_gtsrb())
    meta_dict.append(download_svhn())

    with open(DATA_ROOT / 'meta.json', 'w', encoding='utf8') as f:
        f.write(json.dumps(meta_dict))


from datetime import datetime, timedelta
import json
import traceback

import boto3
from django.conf import settings
from django.contrib.gis.db.models.functions import Distance
from django.contrib.gis.geos import Point
from django.utils import timezone

from routing.models import (
    DriveTimeNode, DriveTimePolygon, DriveTimeQuery, Ways, WaysVerticesPgr,
)
from database.functions import DriveTime

def new_drive_time_query(request_data):
    start_time = datetime.now()
    # TODO: Double check for the drive-time-query to account for multiple identical requests
    # in the queue
    print(f'[{start_time}]: {request_data}')
    lon = request_data.get('lon', None)
    lat = request_data.get('lat', None)
    osm_id = request_data.get('osm_id', None)
    drive_time_hours = request_data.get('drive_time_hours', 0.25)
    inspection_years = request_data.get('inspection_years', 2)
    place_id = request_data['place_id']

    try:
        polygon_pending = False
        existing_drive_time_query = DriveTimeQuery.objects.filter(
            place_id=place_id
        ).filter(
            drive_time_hours=drive_time_hours
        ).order_by(
            '-created_time'
        )[:1].get()

        polygon_pending = existing_drive_time_query.polygon_pending

        drive_time_polygon = DriveTimePolygon.objects.filter(
            drive_time_query=existing_drive_time_query
        ).order_by(
            '-created_time'
        )[:1].get()
    except DriveTimeQuery.DoesNotExist as exc:
        print(f'[{datetime.now()}] Exception: {exc}')
        print(f'[{datetime.now()}] Generating new results')
    except DriveTimePolygon.DoesNotExist as exc:
        print(f'[{datetime.now()}] DriveTimeQuery {existing_drive_time_query.id}  polygon_pending: {polygon_pending}')
        stale_timedelta = timedelta(minutes=15)
        polygon_calculation_age = timezone.now()-existing_drive_time_query.edited_time
        print(f'[{datetime.now()}] DTQ last edited {existing_drive_time_query.edited_time}')
        if polygon_pending and polygon_calculation_age < stale_timedelta:
            print(f'[{datetime.now()}] Early exit. Polygon is in the queue, pending creation.')
            return True
        if polygon_calculation_age > stale_timedelta:
            print(f'[{datetime.now()}] Polygon has not finished in {str(stale_timedelta)}. Trying again')
            polygon_pending = False
            existing_drive_time_query.polygon_pending = polygon_pending
            existing_drive_time_query.save()
            print(f'[{datetime.now()}] {existing_drive_time_query} polygon_pending set to False')
        print(f'[{datetime.now()}] No polygon in the queue. Processing query.')
    else:
        print(f'[{datetime.now()}] Early exit. {place_id} with drive_time_hours {drive_time_hours} exists.')
        return True

    # Query the Ways table with the osm_id returned by Nominatim API
    # If it wasn't a way object, it won't exist. Instead use the lat/lon from
    # the nominatim response to build a buffer polygon and capture the
    # nearest Ways object. If the lat/lon is too far from a road
    # or outside of the routable area, there will be a 500 error
    # TODO: Use osm_class from nominatim to more efficiently handle osm_ids
    # from OSM types relation and node, and don't return 500 errors. This is
    # the least of our performace concerns, so not worth it yet
    try:
        no_way = False
        print(f'[{datetime.now()}] finding way')
        way = Ways.objects.get(osm_id=osm_id)
    except (Ways.DoesNotExist, Ways.MultipleObjectsReturned) as exc:
        print(f'[{datetime.now()}]  not found')
        if not lat or not lon:
            raise exc
        nominatim_point = Point(float(lon), float(lat), srid=4326)
        print(f'[{datetime.now()}] nominatim point: {nominatim_point}')
        try:
            way = Ways.objects.filter(
                the_geom__intersects=nominatim_point.buffer(0.01)
            ).annotate(
                distance=Distance('the_geom', nominatim_point)
            ).order_by(
                'distance'
            )[:1].get()
            print(f'[{datetime.now()}]  generated way from distance: {way}')
        except:
            no_way = True
            pass

    # Remove nominatim fields that are not modeled
    allowed_fields = [field.name for field in DriveTimeQuery._meta.fields]
    model_data = {key: value for key, value in request_data.items() if key in allowed_fields}

    if no_way:
        ways_vertices_pgr = None
    else:
        ways_vertices_pgr = WaysVerticesPgr.objects.get(
            id=way.source.pk
        )
        print(f'[{datetime.now()}] ways_vertices_pgr: {ways_vertices_pgr}')

    drive_time_query = DriveTimeQuery.objects.create(
        **model_data,
        ways_vertices_pgr_source=ways_vertices_pgr
    )
    print(f'[{datetime.now()}] drive_time_query: {drive_time_query}')
    drive_time_query.save()

    if no_way:
        return True

    print(f'[{datetime.now()}] calling db func')
    drive_time = DriveTime(
        ways_vertices_pgr.id,
        drive_time_hours
    )
    print(f'[{datetime.now()}]  drive_time: {drive_time}')
    drive_time.execute_sql()
    DriveTimeNode.objects.bulk_create(
        drive_time.to_models(drive_time_query=drive_time_query)
    )

    drive_time_query.polygon_pending = True
    drive_time_query.save()

    print(
        f'[{datetime.now()}] Set DriveTimeQuery.pending_' +
        f'polygon to: {drive_time_query.polygon_pending}'
    )

    print(f'[{datetime.now()}] Function Name: {settings.LAMBDA_NAME}')
    client = boto3.client('lambda')
    response = client.invoke(
        FunctionName=settings.LAMBDA_NAME,
        InvocationType='Event',
        LogType='None',
        ClientContext='d-queue',
        Payload=json.dumps({'drive_time_query': drive_time_query.id}),
        Qualifier='$LATEST'
    )

    status_code = response['StatusCode']
    request_id = response['ResponseMetadata']['RequestId']
    print(f'[{datetime.now()}] Lambda Status Code: {status_code}')
    print(f'[{datetime.now()}] Request ID: {request_id}')
    print(f'[{datetime.now()}] completed in {datetime.now()-start_time}')

    return True


# -*- coding: utf-8 -*-
"""
Created on Sat Jul  4 19:03:55 2020

@author: ravit
"""

#!/usr/bin/python

import matplotlib.pyplot as plt
from prep_terrain_data import makeTerrainData
from class_vis import prettyPicture
import numpy as np

features_train, labels_train, features_test, labels_test = makeTerrainData()

### the training data (features_train, labels_train) have both "fast" and "slow"
### points mixed together--separate them so we can give them different colors
### in the scatterplot and identify them visually
grade_fast = [features_train[ii][0] for ii in range(0, len(features_train)) if labels_train[ii]==0]
bumpy_fast = [features_train[ii][1] for ii in range(0, len(features_train)) if labels_train[ii]==0]
grade_slow = [features_train[ii][0] for ii in range(0, len(features_train)) if labels_train[ii]==1]
bumpy_slow = [features_train[ii][1] for ii in range(0, len(features_train)) if labels_train[ii]==1]

#### initial visualization
plt.xlim(0.0, 1.0)
plt.ylim(0.0, 1.0)
plt.scatter(bumpy_fast, grade_fast, color = "b", label="fast")
plt.scatter(grade_slow, bumpy_slow, color = "r", label="slow")
plt.legend()
plt.xlabel("bumpiness")
plt.ylabel("grade")
plt.show()
################################################################################

### your code here!  name your classifier object clf if you want the
### visualization code (prettyPicture) to show you the decision boundary
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import confusion_matrix, accuracy_score

clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), n_estimators = 50)
clf.fit(features_train, labels_train)

pred = clf.predict(features_test)
print(accuracy_score(labels_test, pred))
print(confusion_matrix(labels_test, pred))

l_rate = np.linspace(start = 0.1, stop= 1, num = 10, endpoint= True)
scores_list = []
error = []
for l in l_rate:
    clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1), n_estimators = 50, learning_rate= l)
    clf.fit(features_train, labels_train)
    pred = clf.predict(features_test)
    scores_list.append(accuracy_score(labels_test, pred))
    error.append(np.mean( pred != labels_test))

plt.figure(figsize=(12, 6))
plt.plot(l_rate, scores_list, color='black', linestyle='dashed', marker='o',
         markerfacecolor='grey', markersize=10)
plt.xlabel('Values of learning_rate')
plt.ylabel('Testing Accuracy')
plt.show()

clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=2), n_estimators = 50, learning_rate=0.4)
clf.fit(features_train, labels_train)

pred = clf.predict(features_test)
print(confusion_matrix(labels_test, pred))
print(accuracy_score(labels_test, pred))

try:
    prettyPicture(clf, features_test, labels_test)
except NameError:
    pass


def increasingSubsequence(A):
    D = [ [] for i in range(len(A)) ]
    longestOverallSubsequence = []
    for i in range(len(A)):
        import pdb; pdb.set_trace()
        longestEarlierSubsequence = []
        for k in range(i):
            if len(D[k]) > len(longestEarlierSubsequence) and A[k] < A[i]:
                longestEarlierSubsequence = D[k]
        D[i] = longestEarlierSubsequence + [A[i]]
        if len(D[i]) > len(longestOverallSubsequence):
            longestOverallSubsequence = D[i]
    return longestOverallSubsequence

def numberincreasingsubsequence(A):
    D = [ 0 for i in range(len(A)) ]
    for i in range(len(A)):
        longestEarlierSubsequence = 0
        for k in range(i):
            if D[k] > longestEarlierSubsequence and A[k] < A[i]:
                longestEarlierSubsequence = D[k]
        D[i] = longestEarlierSubsequence + 1
    print(D)

    for i in range(len(A)-1, -1, -1):
    	print(A[i])
    return max(D)

if __name__ == '__main__':
	print(numberincreasingsubsequence([6, 3, 2, 5, 6, 4, 8]))


# -*- coding=utf-8 -*-
from common import ProjectCommon as tool
from common import Properties

properties = Properties.parse(tool.getPath() + "\\resources\\Settings.properties")

# 过滤器初始化大小
initialCapacity = int(properties.get("initialCapacity"))

# 过滤器错误率
errorRate = float(properties.get("errorRate"))

# excel表格路径
excelPlace = properties.get("excelPlace")

# sheet名称
sheetNames = properties.get("sheetNames")

# excel统一列索引
excelCellIndex = properties.get("excelCellIndex").split(',')

# 新建excel表格路径
excelCreatePath = properties.get("excelCreatePath")

# 新建excel表格Sheet名称(不重复的)
excelCreateSheetNameForNotExist = properties.get("excelCreateSheetNameForNotExist")

# 新建excel表格Sheet名称(重复的)
excelCreateSheetNameForExist = properties.get("excelCreateSheetNameForExist")

# excel表格基准定义
excelBenchmark = properties.get("excelBenchmark")

# excel文档的索引开始行
excelIndexStartRow = int(properties.get("excelIndexStartRow"))

# 判断索引1(用于判断存入为重复或不重复)
firstIndex = properties.get("firstIndex")

# 判断索引1(用于判断不重复的集合中，索引1为空并且筛选当前索引为不重复)
secondIndex = properties.get("secondIndex")


print('-='*15)
print('{:^30}'.format('Lista de compras'))
print('-='*15)
r = 'S'
pme = str()
s = pm = 0
me = 1000000000
while r == 'S':
    p = str(input('Digite o nome do produto: '))
    pp = float(input('Digite o preço do produto: R$'))
    r = str(input('Gostaria de continuar?\n[ S } sim\n[ N } não\n')).upper().strip()
    while r not in 'SN':
        print('\033[30;41mOpção incorreta\033[m')
        r = str(input('Gostaria de continuar?\n[ S } sim\n[ N } não\n')).upper().strip()
    s += pp
    if pp > 1000:
        pm += 1
    if pp < me:
        me = pp
        pme = p
print(f'Foram gastos R${s:.2f}\n{pm:.0f} produto(s) mais caros que R$1000.00.\nO produto mais barato foi {pme}\nO programa foi encerrado.')


class Core:
    def __init__(self):
        self._types = {
            'A': 100,
            'B': 200,
            'C': 300
        }

    def get_salary(self, type_class):
        return self._types.get(type_class, 0)

class AccountingInterface:
    def __init__(self, data):
        self.core = Core()
        self.database = data

    def get_salary(self, name):
        type_work = self.database.get(name)
        salary = self.core.get_salary(type_work)
        return salary

db = {'Ivan': 'A', 'Olya': 'B', "Pashko": 'C'}
interface = AccountingInterface(db)
res = interface.get_salary('Olya')
print('Salary Olya:', res)


import os
import sys
import pickle
import numpy as np
import xml.etree.ElementTree as ET
import random
import svgwrite
from IPython.display import SVG, display
import tensorflow as tf

def get_bounds( data, factor ):
  min_x = 0
  max_x = 0
  min_y = 0
  max_y = 0

  abs_x = 0
  abs_y = 0
  for i in range( len( data ) ):
    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor
    abs_x += x
    abs_y += y
    min_x = min( min_x, abs_x )
    min_y = min( min_y, abs_y )
    max_x = max( max_x, abs_x )
    max_y = max( max_y, abs_y )

  return ( min_x, max_x, min_y, max_y )

# version where each path is entire stroke ( smaller svg size, but have to keep same color )
def draw_strokes( data, factor = 10, svg_filename = 'sample.svg' ):
  min_x, max_x, min_y, max_y = get_bounds( data, factor )
  dims = ( 50 + max_x - min_x, 50 + max_y - min_y )

  dwg = svgwrite.Drawing( svg_filename, size = dims )
  dwg.add( dwg.rect( insert = ( 0, 0 ), size = dims, fill = 'white' ) )

  lift_pen = 1

  abs_x = 25 - min_x
  abs_y = 25 - min_y
  p = "M%s, %s " % ( abs_x, abs_y )

  command = "m"

  for i in range( len( data ) ):
    if ( lift_pen == 1 ):
      command = "m"
    elif ( command != "l" ):
      command = "l"
    else:
      command = ""
    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor
    lift_pen = data[ i, 2 ]
    p += command+str( x )+", "+str( y )+" "

  the_color = "black"
  stroke_width = 1

  dwg.add( dwg.path( p ).stroke( the_color, stroke_width ).fill( "none" ) )

  dwg.save( )
  display( SVG( dwg.tostring( ) ) )

def draw_strokes_eos_weighted( stroke, param, factor = 10, svg_filename = 'sample[ A_eos.svg' ):
  c_data_eos = np.zeros( ( len( stroke ), 3 ) )
  for i in range( len( param ) ):
    c_data_eos[ i, : ] = ( 1-param[ i ][ 6 ][ 0 ] )*225 # make color gray scale, darker = more likely to eos
  draw_strokes_custom_color( stroke, factor = factor, svg_filename = svg_filename, color_data = c_data_eos, stroke_width = 3 )

def draw_strokes_random_color( stroke, factor = 10, svg_filename = 'sample_random_color.svg', per_stroke_mode = True ):
  c_data = np.array( np.random.rand( len( stroke ), 3 )*240, dtype = np.uint8 )
  if per_stroke_mode:
    switch_color = False
    for i in range( len( stroke ) ):
      if switch_color == False and i > 0:
        c_data[ i ] = c_data[ i-1 ]
      if stroke[ i, 2 ] < 1: # same strike
        switch_color = False
      else:
        switch_color = True
  draw_strokes_custom_color( stroke, factor = factor, svg_filename = svg_filename, color_data = c_data, stroke_width = 2 )

def draw_strokes_custom_color( data, factor = 10, svg_filename = 'test.svg', color_data = None, stroke_width = 1 ):
  min_x, max_x, min_y, max_y = get_bounds( data, factor )
  dims = ( 50 + max_x - min_x, 50 + max_y - min_y )

  dwg = svgwrite.Drawing( svg_filename, size = dims )
  dwg.add( dwg.rect( insert = ( 0, 0 ), size = dims, fill = 'white' ) )

  lift_pen = 1
  abs_x = 25 - min_x
  abs_y = 25 - min_y

  for i in range( len( data ) ):

    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor

    prev_x = abs_x
    prev_y = abs_y

    abs_x += x
    abs_y += y

    if ( lift_pen == 1 ):
      p = "M "+str( abs_x )+", "+str( abs_y )+" "
    else:
      p = "M +"+str( prev_x )+", "+str( prev_y )+" L "+str( abs_x )+", "+str( abs_y )+" "

    lift_pen = data[ i, 2 ]

    the_color = "black"

    if ( color_data is not None ):
      the_color = "rgb( "+str( int( color_data[ i, 0 ] ) )+", "+str( int( color_data[ i, 1 ] ) )+", "+str( int( color_data[ i, 2 ] ) )+" )"

    dwg.add( dwg.path( p ).stroke( the_color, stroke_width ).fill( the_color ) )
  dwg.save( )
  display( SVG( dwg.tostring( ) ) )

def draw_strokes_pdf( data, param, factor = 10, svg_filename = 'sample_pdf.svg' ):
  min_x, max_x, min_y, max_y = get_bounds( data, factor )
  dims = ( 50 + max_x - min_x, 50 + max_y - min_y )

  dwg = svgwrite.Drawing( svg_filename, size = dims )
  dwg.add( dwg.rect( insert = ( 0, 0 ), size = dims, fill = 'white' ) )

  abs_x = 25 - min_x
  abs_y = 25 - min_y

  num_mixture = len( param[ 0 ][ 0 ] )

  for i in range( len( data ) ):

    x = float( data[ i, 0 ] )/factor
    y = float( data[ i, 1 ] )/factor

    for k in range( num_mixture ):
      pi = param[ i ][ 0 ][ k ]
      if pi > 0.01: # optimisation, ignore pi's less than 1% chance
        mu1 = param[ i ][ 1 ][ k ]
        mu2 = param[ i ][ 2 ][ k ]
        s1 = param[ i ][ 3 ][ k ]
        s2 = param[ i ][ 4 ][ k ]
        sigma = np.sqrt( s1*s2 )
        dwg.add( dwg.circle( center = ( abs_x+mu1*factor, abs_y+mu2*factor ), r = int( sigma*factor ) ).fill( 'red', opacity = pi/( sigma*sigma*factor ) ) )

    prev_x = abs_x
    prev_y = abs_y

    abs_x += x
    abs_y += y

  dwg.save( )
  display( SVG( dwg.tostring( ) ) )

class DataLoader( ):

  def getRandValue( self ):
    value = self.randvalues[ self.randvaluepointer ]
    self.randvaluepointer += 1
    if ( self.randvaluepointer >= self.nrrandvalues ):
        self.randvaluepointer = 0
    return value

  def createRandValues( self ):
    self.nrrandvalues = 1000
    self.randvalues = np.zeros( ( self.nrrandvalues ), dtype = np.float32 )
    for i in range( self.nrrandvalues ):
        value = random.random( )
        self.randvalues[ i ] = value
    self.randvaluepointer = 0

  def getClassLabels( self ):
    if self.train:
      fn = self.data_dir + "trainlabels.txt"
    else:
      fn = self.data_dir + "testlabels.txt"
    classlabels = np.loadtxt( fn )
    classlabels = classlabels[ :self.nrinputfiles ]
    return classlabels

  def findAvailableExamples( self, args ):
    self.availableExamples = [ ]
    findexamples = True
    if findexamples:
      for i in range( len( self.classlabels ) ):
        if ( self.classlabels[ i ] < args.curnrdigits ):
          self.availableExamples.append( i )
    self.availableExamples = np.array( self.availableExamples )

  def __init__( self, datadir, args, totnrfiles, curnrexamples, seqlength = 0, train = 1, file_label = "", print_input = 0, rangemin = 0, rangelen = 0 ):

    random.seed( 100*args.runnr )
    np.random.seed( 100*args.runnr )
    tf.set_random_seed( 100*args.runnr )
    self.args = args

    self.data_dir = datadir
    self.train = train
    if self.train:
      self.traintest = "train"
    else:
      self.traintest = "test"
    self.rangemin = rangemin
    self.rangelen = rangelen
    self.nrinputfiles = totnrfiles

    self.curnrexamples = curnrexamples
    self.nrseq_per_batch = args.nrseq_per_batch
    self.file_label = file_label
    self.print_input = print_input
    self.nrinputvars_data = self.getInputVectorLength( args )
    self.max_seq_length = args.max_seq_length

    self.nrsequenceinputs = 4 #dx dy eos eod
    self.nrauxinputvars = args.nrClassOutputVars #either [ 0..9 dx dy eos eod ] or [ dx dy eos ]

    strokedatafile = os.path.join( self.data_dir, "strokes_"+self.traintest+"ing_data"+ file_label+args.explabel+ ".cpkl" )
    raw_data_dir = self.data_dir+"/lineStrokes"

    print ( "creating data cpkl file from source data" )
    self.preprocess( args, raw_data_dir, strokedatafile )

    if ( seqlength > 0 ): #provided
      self.seq_length = seqlength
    else:
      self.seq_length = min( self.max_seq_length, args.maxdigitlength_nrpoints )

    self.load_preprocessed( args, strokedatafile )

    self.classlabels = self.getClassLabels( )
    self.findAvailableExamples( args )

    self.nrbatches_per_epoch = max( 1, int( self.curnrexamples / self.nrseq_per_batch ) )
    print ( "curnrexamples", self.curnrexamples, "seq_length", self.seq_length, " --> nrbatches_per_epoch: ", self.nrbatches_per_epoch )

    print ( "loaded data" )
    self.reset_batch_pointer( args )

  def constructInputFileName( self, args, file_label, imgnr ):
    filename = self.data_dir + self.traintest + 'img' + file_label + '-' + str( imgnr ) + '-targetdata.txt' #currently, we expect 14 inputs
    return filename

  def getInputVectorLength( self, args ):
    result = [ ]

    filename = self.constructInputFileName( args, self.file_label, imgnr = 0 )

    with open( filename ) as f:
      points = [ ]
      line = f.readline( )
      print ( "read sample line from inputdata file: ", line )
      nrs = [ float( x ) for x in line.split( ) ]
      length = len( nrs )
      print ( "Determined nrinputvars based on data: ", length )
    self.nrinputvars_data = length
    return length

  def preprocess( self, args, data_dir, strokedatafile ):
    filelist = [ ]

    if len( args.fileselection )>0:
      fileselection = ' '.join( args.fileselection )
      if len( fileselection )>0:
        fileselection = [ int( s ) for s in fileselection.split( ', ' ) ]

    for imgnr in range( 0, self.nrinputfiles ):
        if len( args.fileselection )>0:
          fname = self.constructInputFileName( args, self.file_label, fileselection[ imgnr ] )
        else:
          fname = self.constructInputFileName( args, self.file_label, imgnr )
        filelist.append( fname )

    def getStrokes( filename, nrauxinputvars ): #returns array of arrays with points
      result_points = [ ]
      result_auxinputs = [ ]
      nrsequencevars = 4
      dxmin = 1e100
      dxmax = -1e100
      dymin = 1e100
      dymax = -1e100
      nrauxinputs_data = 10

      with open( filename ) as f:
        points = [ ]
        auxinputs = [ ]
        for line in f: # read rest of lines
          nrs = [ float( x ) for x in line.split( ) ]
          auxinputvalues = nrs[ 0:nrauxinputvars ]
          point = nrs[ nrauxinputs_data:nrauxinputs_data+nrsequencevars ] #currently: x, y, end-of-stroke
          points.append( point )
          auxinputs.append( auxinputvalues )
        result_points.append( points )
        result_auxinputs.append( auxinputs )
        pointarray = np.array( points )
        digitlength_nrpoints = len( points )
        dxmin = pointarray[ :, 0 ].min( )
        dxmax = pointarray[ :, 0 ].max( )
        dymin = pointarray[ :, 1 ].min( )
        dymax = pointarray[ :, 1 ].max( )
        ranges = [ dxmin, dxmax, dymin, dymax ]
      return result_auxinputs, result_points, ranges, digitlength_nrpoints

    # converts a list of arrays into a 2d numpy int16 array
    def convert_stroke_to_array( stroke ):

      n_point = 0
      for i in range( len( stroke ) ):
        n_point += len( stroke[ i ] )

      prev_x = 0
      prev_y = 0
      counter = 0
      nrsequencevars = 4
      stroke_data = np.zeros( ( n_point, nrsequencevars ), dtype = np.int16 )

      for j in range( len( stroke ) ):
        for k in range( len( stroke[ j ] ) ):
          for s in range( nrsequencevars ):
            stroke_data[ counter, s ] = int( stroke[ j ][ k ][ s ] )
          counter += 1
      return stroke_data

    # converts a list of arrays into a 2d numpy int16 array
    def convert_auxinputs_to_array( auxinputs, nrauxinputvars ):

      n_point = 0
      for i in range( len( auxinputs ) ):
        n_point += len( auxinputs[ i ] )
      auxinputdata = np.zeros( ( n_point, nrauxinputvars ), dtype = np.int16 )

      prev_x = 0
      prev_y = 0
      counter = 0

      for j in range( len( auxinputs ) ):
        for k in range( len( auxinputs[ j ] ) ):
          for a in range( nrauxinputvars ):
            auxinputdata[ counter, a ] = int( auxinputs[ j ][ k ][ a ] )
          counter += 1
      return auxinputdata

    # preprocess body: build stroke array
    strokearray = [ ]
    auxinputarray = [ ]
    rangelist = [ ]
    self.seqlengthlist = [ ]
    if self.train:
      args.maxdigitlength_nrpoints = 0
    digitlengthsum = 0
    for i in range( len( filelist ) ):
      print ( 'dataloader', self.traintest, 'processing '+filelist[ i ] )
      [ auxinputs, strokeinputs, ranges, digitlength_nrpoints ] = getStrokes( filelist[ i ], self.nrauxinputvars )
      strokearray.append( convert_stroke_to_array( strokeinputs ) )
      auxinputarray.append( convert_auxinputs_to_array( auxinputs, self.nrauxinputvars ) )
      rangelist.append( ranges )
      self.seqlengthlist.append( digitlength_nrpoints )
      if self.train:
        args.maxdigitlength_nrpoints = max( args.maxdigitlength_nrpoints, digitlength_nrpoints )
        digitlengthsum += digitlength_nrpoints

    rangearray = np.array( rangelist )
    ranges = [ rangearray[ :, 0 ].min( ), rangearray[ :, 1 ].max( ), rangearray[ :, 2 ].min( ), rangearray[ :, 3 ].max( ) ]
    print ( "found overall ranges", ranges )
    self.avgseqlength = digitlengthsum / len( filelist )
    print( "dataloader: found avg seq length: ", self.avgseqlength )
    print ( "found maxdigitlength_nrpoints", args.maxdigitlength_nrpoints )

    f = open( strokedatafile, "wb" )
    pickle.dump( strokearray, f )
    pickle.dump( auxinputarray, f )
    pickle.dump( ranges, f )
    pickle.dump( self.seqlengthlist, f )
    f.close( )

  def load_preprocessed( self, args, strokedatafile ):
    f = open( strokedatafile, "rb" )
    self.strokedataraw = pickle.load( f )
    self.auxdataraw = pickle.load( f )
    self.ranges = pickle.load( f )
    self.seqlengthlist = pickle.load( f )
    f.close( )

    print ( "loaded ranges", self.ranges )
    print ( "rangemin", self.rangemin, "rangelen", self.rangelen )

    self.strokedata = [ ] #contains one array per file
    self.auxdata = [ ]
    counter = 0

    for data_el in self.strokedataraw:
        data = np.array( np.zeros( ( self.seq_length, self.nrsequenceinputs ), dtype = np.float32 ) )
        len_data = len( data )
        nrpoints = min( self.seq_length, len( data_el ) )
        data[ :nrpoints, ] = data_el[ :nrpoints ]
        if ( len( data_el ) > self.seq_length ) and ( self.seq_length >= args.max_seq_length ):
          data[ self.seq_length-1, 2:4 ] = np.ones( ( 1, 2 ), dtype = np.float32 ) #add eos and eod for sequences exceeding length
        data[ nrpoints:, 0:4 ] = np.zeros( ( len_data - nrpoints, 4 ), dtype = np.float32 ) #pad remainder with zero rows
        data[ :, 0:2 ] -= self.rangemin
        data[ :, 0:2 ] /= self.rangelen
        self.strokedata.append( data )

        counter += 1
    for data_el in self.auxdataraw:
        data = np.array( np.zeros( ( self.seq_length, self.nrauxinputvars ), dtype = np.float32 ) )
        nrpoints = min( self.seq_length, len( data_el ) )
        data[ :nrpoints, ] = data_el[ :nrpoints ]
        data[ nrpoints:self.seq_length, ] = data[ nrpoints-1, ]
        self.auxdata.append( data )
    print ( "#sequences found in data: ", counter )

  def next_batch( self, args, curseqlength ):

    # returns a batch of the training data of nrseq_per_batch * seq_length points
    x_batch = [ ]
    y_batch = [ ]
    seqlen = self.seq_length
    sequence_index = [ ]
    use_points_stopcrit = False

    nrpoints_per_batch = 0
    if hasattr( args, 'nrpoints_per_batch' ):
      nrpoints_per_batch = args.nrpoints_per_batch
    if nrpoints_per_batch > 0:
      use_points_stopcrit = True
    batch_nrpoints = 0
    batch_sequencenr = 0
    done = False
    while not done:
      sequence_index.append( self.pointer )
      strokes = np.copy( self.strokedata[ self.pointer ] )
      auxvalues = np.copy( self.auxdata[ self.pointer ] )

      if args.useStrokeOutputVars:
        ytab = np.copy( np.hstack( [ auxvalues[ 1:seqlen ], strokes[ 1:seqlen ] ] ) )
      else:
        ytab = np.copy( np.hstack( [ auxvalues[ 1:seqlen ] ] ) )

      if args.discard_classvar_inputs:
        auxvalues[ : ] = 0

      if args.useClassInputVars:
        xtab = np.hstack( [ auxvalues[ :seqlen-1 ], strokes[ :seqlen-1 ] ] )
      else:
        xtab = strokes[ :seqlen-1 ]

      actual_seq_length = self.seqlengthlist[ self.pointer ]

      firsttrainstep = 0
      if hasattr( args, 'firsttrainstep' ):
        firsttrainstep = args.firsttrainstep
      firsttrainstep = min ( firsttrainstep, actual_seq_length - 1 )
      if firsttrainstep > 0: #remove earlier part from _target_ data, so that it will not be used in loss.
        ytab[ :firsttrainstep, : ] = 0

      #only keep points up to current seq_length - 1; e.g. if sequence has 3 points, use 2 pairs of ( x, y ): k = 1.n-1 for x and k = 2..n for y
      firstafter = min( actual_seq_length - 1, curseqlength ) #zero out part after sequence

      xtab[ firstafter:, : ] = 0
      ytab[ firstafter:, : ] = 0

      nrusedpoints = firstafter - firsttrainstep

      if args.discard_inputs:
        xtab[ : ] = 0

      x_batch.append( np.copy( xtab ) )
      y_batch.append( np.copy( ytab ) )

      self.next_batch_pointer( args )

      batch_sequencenr += 1
      nrseq_per_batch = self.nrseq_per_batch
      if ( not self.train ) and hasattr( args, 'nrseq_per_batch_test' ):
        nrseq_per_batch = args.nrseq_per_batch_test

      if use_points_stopcrit:
        batch_nrpoints += nrusedpoints
        done = batch_nrpoints >= nrpoints_per_batch
      else:
        done = batch_sequencenr >= nrseq_per_batch

    return x_batch, y_batch, sequence_index

  def selectExamples( self, nrdigits ):
    sample = np.random.permutation( len( self.availableExamples ) )
    return self.availableExamples[ sample ]

  def next_batch_pointer( self, args ):
    self.index += 1
    if ( self.index >= len( self.example_permutation ) ):
      self.reset_batch_pointer( args )
    self.pointer = self.example_permutation[ self.index ]

  def reset_batch_pointer( self, args ):
    self.index = 0

    if ( args.incremental_nr_digits and self.train ):
      self.example_permutation = self.selectExamples( args.curnrdigits )
    else:
      if self.train:
        self.example_permutation = np.random.permutation( int( self.curnrexamples ) )
      else:
        self.example_permutation = np.arange( 0, int( self.curnrexamples ) )
    self.pointer = self.example_permutation[ self.index ]


import base.job
from plugins.common.RVT_disk import getSourceImage

""" Modules to mount or umount all partitions in a source."""

class Mount(base.job.BaseModule):
    """ Mount all partitions in a disk and then run from_module.

    Configuration:
        - **partitions** (str): comma separated list of partitions to mount (ex: p03,p05,v1p05). All non vss partitions by default
        - **recovery_keys** (str): comma separated list of recovery keys for encrypted partition
        - **ntfs_args** (str): specific options for mounting an NTFS partition.
        - **fat32_args** (str): specific options for mounting a FAT32 partition.
        - **ext4_args** (str): specific options for mounting an EXT4 partition.
        - **hfs_args** (str): specific options for mounting an HFS partition.
        - **vss** (bool): mount regular (False) or Volume Shadow Snapshots (True) partitions
    """
    def run(self, path=None):
        disk = getSourceImage(self.myconfig)
        disk.mount(partitions=self.myconfig('partitions'), vss=self.myconfig('vss'), unzip_path=self.myconfig('unzip_path'))
        if self.from_module:
            for data in self.from_module.run(path):
                yield data

class UMount(base.job.BaseModule):
    """ Run from_module and then umount all partitions in a disk.

    """
    def run(self, path=None):
        if self.from_module:
            for data in self.from_module.run(path):
                yield data
        disk = getSourceImage(self.myconfig)
        disk.umount()
        return []


import os
import google.oauth2.credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from oauth2client.tools import argparser, run_flow

# API credentials
CLIENT_SECRETS_FILE = "{replaice with key value}"
SCOPES = ["https://www.googleapis.com/auth/youtube.force-ssl"]
API_SERVICE_NAME = "youtube"
API_VERSION = "v3"

def get_authenticated_service():
    flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRETS_FILE, SCOPES)
    credentials = flow.run_local_server()
    return build(API_SERVICE_NAME, API_VERSION, credentials=credentials)

def delete_video(youtube, video_id):
    youtube.videos().delete(id=video_id).execute()
    print("Video with ID '%s' deleted successfully." % video_id)

if __name__ == "__main__":
    argparser.add_argument("--id", required=True, help="Youtube video ID")
    args = argparser.parse_args()

    youtube = get_authenticated_service()
    video_id = args.id
    delete_video(youtube, video_id)