import os
import logging
import requests
import re

from sys import exit

URL_DUCK_DNS = 'https://www.duckdns.org/update?domains=${sub_domains}.duckdns.org&token=${token}&ip=${ip}'
PAGE_MY_IP ='https://www.cual-es-mi-ip.net/'
try:
    PATH_LOGS=os.environ["UPDATE_DUCK_DNS_LOGS"]
except KeyError:
    print('log path not found creating in default folder')

try:
    logging.basicConfig(
        filename=PATH_LOGS+'/updateDuckDNS.log',
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        datefmt='%d/%m/%Y %I:%M:%S %p')
except:
    logging.basicConfig(
        filename='/tmp/updateDuckDNS.log',
        level=logging.DEBUG,
        format='%(asctime)s - %(levelname)s - %(message)s',
        datefmt='%d/%m/%Y %I:%M:%S %p')

logging.info('Started')

try:
    TOKEN = os.environ["UPDATE_DUCK_DNS_TOKEN"]
    SUB_DOMAINS = os.environ["UPDATE_DUCK_DNS_SUB_DOMAIN"]
    URL_DUCK_DNS = URL_DUCK_DNS.replace('${sub_domains}', SUB_DOMAINS)
    URL_DUCK_DNS = URL_DUCK_DNS.replace('${token}', TOKEN)
except KeyError:
    logging.warning('there are no environment variables')
    exit()

page_link ='https://domains.google.com/checkip'
page_response = requests.get(page_link, timeout=5)
my_ip = page_response.text

logging.info("My IP: " + my_ip)

try:
    PATH_INSTALL=os.environ["PATH_INSTALL_SCRIPT_PYTHON_DUCK_DNS"]
except Exception:
    logging.warning('not found enviroment PATH_INSTALL_SCRIPT_PYTHON')

replace_ip = True
try:
    if(os.path.isfile(PATH_INSTALL+"/my_ip.txt") == True):
        file = open(PATH_INSTALL+"/my_ip.txt", "r+")
        ip_file = file.read()
        logging.info('my_ip.txt: ' + ip_file.strip())
        if(ip_file.strip() != my_ip.strip()):
            ip_file = re.sub(ip_file.strip(), my_ip.strip(), ip_file)
            file.seek(0)
            file.write(ip_file)
            file.close()
            logging.info('update ip')
        else:
            replace_ip = False
            logging.info('not update ip')
    else:
        logging.info('create file my_ip.txt')
        file = open(PATH_INSTALL+"/my_ip.txt","w")
        file.write(my_ip)
        file.close()

except Exception as ex:
    logging.warning(ex)
    logging.warning('not found file my_ip.txt')
    exit()

if (replace_ip):
    URL_DUCK_DNS = URL_DUCK_DNS.replace('${ip}', my_ip)
    logging.info("DUCK_URL: " + URL_DUCK_DNS)

    page_response = requests.get(URL_DUCK_DNS, timeout=5)

    logging.info("Response: " + page_response.text)

logging.info('Finished')


import os, sys
from os.path import dirname, abspath

if getattr(sys, 'frozen', False):
    filedir = os.path.dirname(sys.executable)
elif __file__:
    filedir = os.path.dirname(os.path.abspath(__file__))

if getattr(sys, 'frozen', False):
    basedir = os.path.dirname(os.path.dirname(sys.executable))
elif __file__:
    basedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

sys.path.insert(1, basedir)
import config
import pandas as pd
import general.df_preprocessing as dfpre
import humanfriendly
from datetime import datetime, timedelta
from hurry.filesize import size
SEP = "\t"

def apply(dataframe):
    dataframe = dfpre.apply(dataframe)

    event_num = dataframe.shape[0]
    account_num = len(dataframe['ACCOUNT'].unique())
    jobs_num = len(dataframe['JOBID'].unique())

    min_time = str(dataframe['CURRENT_TIMESTAMP'].min()).split('.')[0] if '.' in str(dataframe['CURRENT_TIMESTAMP'].min()) else str(dataframe['CURRENT_TIMESTAMP'].min())
    max_time = str(dataframe['CURRENT_TIMESTAMP'].max()).split('.')[0] if '.' in str(dataframe['CURRENT_TIMESTAMP'].max()) else str(dataframe['CURRENT_TIMESTAMP'].max())

    slurmy_accounts = list()
    slurmy_jobs = list()
    print("start finding slurmys", datetime.now())
    for index, row in dataframe.iterrows():
        if '(unfulfilled)' in str(row['DEPENDENCY']) and row['DEPENDENCY'] != '(null)':
            if row['ACCOUNT'] not in slurmy_accounts:
                slurmy_accounts.append(row['ACCOUNT'])

            if row['JOBID'] not in slurmy_jobs:
                slurmy_jobs.append(row['JOBID'])
        else:
            pass

    slurmy_accounts_num = len(slurmy_accounts)
    slurmy_jobs_num = len(slurmy_jobs)
    print("start finding cpus", datetime.now())
    cpu_sum = 0
    for cpu_val in dataframe['CPUS'].tolist():
        s_cpu_val = str(cpu_val)
        if len(s_cpu_val)>0 and s_cpu_val.isnumeric():
            cpu_sum = cpu_sum + int(s_cpu_val)
        else:
            pass

    average_cpu_usage = cpu_sum / len(dataframe['CPUS'].tolist())
    print("start finding RAMs", datetime.now())
    RAM_list = list()
    for RAM in dataframe['MIN_MEMORY'].to_list():
        if RAM != 'None' and any(char.isdigit() for char in RAM) and RAM not in dataframe['PARTITION'].unique():
            num_bytes = humanfriendly.parse_size(RAM)
            RAM_list.append(num_bytes)

    average_RAM_usage = sum(RAM_list) / len(RAM_list)

    res = {
        "startTime": str(min_time),
        "endTime": str(max_time),
        "eventsNumber": str(event_num),
        "jobNumber": str(jobs_num),
        "accountNumber": str(account_num),
        "SLURMaccountsPercentage": str(round(slurmy_accounts_num/account_num, 2)),
        "SLURMaccounts": str(slurmy_accounts),
        "SLURMaccountNumber": str(slurmy_accounts_num) + " out of " + str(account_num),
        "SLURMjobssPercentage": str(round(slurmy_jobs_num/jobs_num, 2)),
        "SLURMjobNumber": str(slurmy_jobs_num) + " out of " + str(jobs_num),
        "CPUusageAvg": str(round(average_cpu_usage, 2)),
        "RAMusageAvg": str(size(round(average_RAM_usage, 2)))
    }

    return res

def account_batches(df, start_time_obj, duration, batches_list):
    batches_dict = {}
    duration_obj = datetime.strptime(duration, '%H:%M:%S')
    next_time = start_time_obj + timedelta(hours=duration_obj.hour, minutes=duration_obj.minute, seconds=duration_obj.second)

    df['CURRENT_TIMESTAMP'] = pd.to_datetime(df.CURRENT_TIMESTAMP, format='%Y-%m-%d %H:%M:%S')
    new_df = df[(start_time_obj<df['CURRENT_TIMESTAMP']) & (df['CURRENT_TIMESTAMP']<next_time)]
    for account in new_df.ACCOUNT.unique():
        account_df = new_df[new_df['ACCOUNT']==account]
        batches_dict[account] = len(account_df.JOBID.unique())

    if len(batches_dict) > 0:
        batches_list.append(batches_dict)
    return next_time, batches_list

def num_of_intervals_an_account_did_batching(batches_list, threshold, num_of_intervals_account_did_batches):
    for batches in batches_list:
        for account in batches:
            if batches[account] >= threshold:
                if account in num_of_intervals_account_did_batches:
                    num_of_intervals_account_did_batches[account] += 1
                else:
                    num_of_intervals_account_did_batches[account] = 1

    return num_of_intervals_account_did_batches

def get_batches(df, duration, threshold):
    df = dfpre.apply(df)
    start_time = df['CURRENT_TIMESTAMP'].min()
    start_time_obj = datetime.strptime(start_time.split('.')[0], '%Y-%m-%d %H:%M:%S')
    end_time = df['CURRENT_TIMESTAMP'].max()
    end_time_obj = datetime.strptime(end_time.split('.')[0], '%Y-%m-%d %H:%M:%S')

    batches_list = []
    while start_time_obj < end_time_obj:
        next_time, new_batches_list = account_batches(df, start_time_obj, duration, batches_list)
        batches_list = new_batches_list
        start_time_obj = next_time

    num_of_intervals_account_did_batches = num_of_intervals_an_account_did_batching(batches_list, threshold, {})

    return num_of_intervals_account_did_batches

if __name__ == "__main__":
    dataframe = pd.read_csv(config.logger.local_log, sep=SEP, encoding="iso-8859-1", error_bad_lines=False)
    print(apply(dataframe))


def egcd(a, b):
	if a == 0:
		return(b, 0, 1)
	else:
        	g, y, x = egcd(b % a, a)
        	return (g, x - (b // a) * y, y)

def fieldInv(g, p):
	#want yg = 1 (mod p) => yg = 1 + xp
	gcd, x, y = egcd(g,p)
	if gcd != 1:
		return None
	else:
		return x % p

#print fieldInv(22,29)


from django.shortcuts import render

from api.serializers import BookSerializer
from api.models import Books
from rest_framework.viewsets import ModelViewSet
from rest_framework.response import Response
from rest_framework.decorators import action
from api.serializers import UserSerializer
from django.contrib.auth.models import User
from rest_framework import authentication,permissions

class BooksView(ModelViewSet):
    # authentication_classes=[authentication.TokenAuthentication]
    permission_classes=[permissions.IsAuthenticated]

    serializer_class=BookSerializer
    model=Books
    queryset=Books.objects.all()

    @action(methods=["GET"],detail=False)
    def genre(self,request,*args,**kwargs):
        qs=Books.objects.all().values_list("genre",flat=True).distinct()
        return Response(data=qs)

    def get_queryset(self):
        qs=Books.objects.all()

        if "genre" in self.request.query_params:
            cat=self.request.query_params.get("genre")
            qs=qs.filter(genre=cat)
            return qs

        if "price_gt" in self.request.query_params:
            price=self.request.query_params.get("price_gt")
            qs=qs.filter(price__gt=price)
            return qs

        return qs

class UserView(ModelViewSet):
    serializer_class=UserSerializer
    queryset=User.objects.all()

    # def create(self, request, *args, **kwargs):
    #     serializer=UserSerializer(data=request.data)
    #     if serializer.is_valid():
    #         usr=User.objects.create_user(**serializer.validated_data)
    #         serializer=UserSerializer(usr,many=False)
    #         return Response(data=serializer.data)
    #     else:
    #         return Response(data=serializer.errors)


"""
Only reports
No file changes
Should break build on any issues.
Expect few issues
All issues should be addressable immediately.
"""
import os

from navio_tasks.cli_commands import config_pythonpath, execute_with_environment
from navio_tasks.settings import PYTHON

# Legacy code. Don't use nose.
def do_django_tests() -> None:
    """
    manage.py tests
    """

    # check_command_exists("")
    if os.path.exists("manage.py"):
        do_django_tests_regardless()

def do_django_tests_regardless() -> None:
    """
    Extracted function to make easier to test this task
    """
    command = f"{PYTHON} manage.py test -v 2"
    my_env = config_pythonpath()
    execute_with_environment(command, env=my_env)


from django import template
from eulfedora.util import RequestFailed, PermissionDenied

register = template.Library()

class CatchFedoraErrorsNode(template.Node):
    """Template render node for catching fedora access errors and providing
    fallback content."""
    def __init__(self, fedora_access, permission_denied=None,
                 fedora_failed=None):
        self.fedora_access = fedora_access
        self.permission_denied = permission_denied
        self.fedora_failed = fedora_failed

    def render(self, context):
        try:
            return self.fedora_access.render(context)
        except PermissionDenied:
            if self.permission_denied is not None:
                return self.permission_denied.render(context)
            elif self.fedora_failed is not None:
                return self.fedora_failed.render(context)
            else:
                return ''
        except RequestFailed:
            if self.fedora_failed is not None:
                return self.fedora_failed.render(context)
            else:
                return ''

@register.tag(name='fedora_access')
def do_catch_fedora_errors(parser, token):
    """Catches fedora errors between ``{% fedora_access %}`` and
    ``{% end_fedora_access %}``. Template designers may specify
    optional ``{% permission_denied %}`` and ``{% fedora_failed %}``
    sections with fallback content in case of permission or other errors
    while rendering the main block.

    Note that when Django's ``TEMPLATE_DEBUG`` setting is on, it precludes
    all error handling and displays the Django exception screen for all
    errors, including fedora errors, even if you use this template tag. Turn
    off ``TEMPLATE_DEBUG`` if that debug screen is getting in the way of the
    use of {% fedora_access %}.
    """

    END_TAGS = ('end_fedora_access',
                'permission_denied', 'fedora_failed')

    blocks = {}
    blocks['fedora_access'] = parser.parse(END_TAGS)
    token = parser.next_token()
    while token.contents != 'end_fedora_access':
        # need to convert token.contents manually to a str. django gives us
        # a unicode. we use it below in **blocks, but python 2.6.2 and
        # earlier can't use **kwargs with unicode keys. (2.6.5 is ok with
        # it; not sure about intervening versions.) in any case, direct
        # conversion to string is safe here (i.e., no encoding needed)
        # because the parser guarantees it's one of our END_TAGS, which are
        # all ascii.
        current_block = str(token.contents)
        if current_block in blocks:
            raise template.TemplateSyntaxError(
                current_block + ' may appear only once in a fedora_access block')

        blocks[current_block] = parser.parse(END_TAGS)
        token = parser.next_token()

    return CatchFedoraErrorsNode(**blocks)


# -*- coding: utf-8 -*-
import itertools

from cabbage.utils import ExponentialBackoff, FibonaccianBackoff

class TestExponentialBackoff:

    def test_double(self):
        backoff = ExponentialBackoff(1, 2, 2048)
        assert list(itertools.islice(backoff, 15)) == [
            1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 2048, 2048, 2048
        ]

class TestFibonaccianBackoff:

    def test_maxprecision(self):
        backoff = FibonaccianBackoff(1000)
        assert list(itertools.islice(backoff, 15)) == [
            1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610
        ]

    def test_1min(self):
        backoff = FibonaccianBackoff(60)
        assert list(itertools.islice(backoff, 15)) == [
            1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 60, 60, 60, 60, 60
        ]


def bfs():
    q = [(N, 0)]
    visited = [0] * (1000000)
    visited[N] = 1

    while q:
        t, m = q.pop(0)
        if t == K :
            return m
        for i, x in [(t*2, True), (t-1, False), (t+1, False)]:
            if not x:
                if 0 <= i < 100002 and not visited[i]:
                    visited[i] = 1
                    q.append((i, m+1))
            else:
                if 0 <= i < 100002 and not visited[i]:
                    visited[i] = 1
                    q.append((i, m))

N, K= map(int,input().split())
print(bfs())


from collections import deque
import sys

def getminrisk(get, width, height):
    processedrisk = [sys.maxsize] * width*height
    toprocess = { (0,0,0) }
    processed = set()
    while toprocess:
        posx, posy, risk = sorted(toprocess, key=lambda p: p[2])[0]
        toprocess.remove((posx, posy, risk))
        processed.add((posx, posy))
        if posx == width-1 and posy == height-1:
            return risk
        for movex, movey in ((-1,0), (1,0), (0,-1), (0,1)):
            nextx, nexty = posx+movex, posy+movey
            if nextx < 0 or nexty < 0 or nextx >= width or nexty >= height:
                continue
            if (nextx, nexty) in processed:
                continue
            nextrisk = risk + get(nextx, nexty)
            if nextrisk < processedrisk[nexty*width+nextx]:
                processedrisk[nexty*width+nextx] = nextrisk
                toprocess.add((nextx, nexty, nextrisk))

with open("15.txt", "r") as f:
    data = f.readlines()
    width, height = len(data[0].strip()), len(data)
    riskmap = [int(x) for l in data for x in l.strip()]

def get(x, y):
    return riskmap[y*width+x]
def get5(x, y):
    xi, yi = x // width, y // width
    truex, truey = x % width, y % height
    return (riskmap[truey*width+truex]+xi+yi-1)%9+1

print(getminrisk(get, width, height))
print(getminrisk(get5, width*5, height*5))


import PySimpleGUI as sg
import cv2
from ImageEditing import ChangeBrightness,ChangeSaturation,ImageSharpen,ImageFlip
from ImageFilters import OilPainting, Water , BlackAndWhite,GrayScale
'''
Requirements
    - PySimpleGUI
    - OpenCV (specifically pip install opencv-contrib-python, if error persists uninstall both opencv-contrib-python  and opencv-python(if installed) and "pip install --no-cache-dir opencv-contrib-python")

'''

def main():

    # sg.theme("Reddit")
    sg.theme("DarkBrown2")
    column = [
        sg.Column([
            [
            sg.Radio("Oil Painting", "Radio", size=(10, 1), key="-OIL-",font=("Cascadia Code SemiBold", 12)),
            sg.Slider(
                (1, 50),
                # add 50 ticks to the slider
                tick_interval=0.1,
                orientation="h",
                size=(20, 15),
                key="-OIL SLIDER SIZE-",
            ),
            sg.Slider(
                (1, 100),
                tick_interval=0.1,
                orientation="h",
                size=(20, 15),
                key="-OIL SLIDER LEVEL-",
            )
            ],
            [
                sg.Radio("Water Color", "Radio", size=(10, 1), key="-WATER-",font=("Cascadia Code SemiBold", 12)),
                sg.Slider(
                    (1, 400),
                    orientation="h",
                    size=(20, 15),
                    key="-WATER SLIDER SIZE-",
                ),
                sg.Slider(
                    (1, 100),
                    orientation="h",
                    size=(20, 15),
                    key="-WATER SLIDER LEVEL-",
                )
            ],
            [
                sg.Radio("Brightness", "Radio", size=(10, 1), key="-BRIGHTNESS-",font=("Cascadia Code SemiBold", 12)),
                sg.Slider(
                    (0, 100),
                    orientation="h",
                    size=(40, 15),
                    key="-BRIGHTNESS SLIDER-",
                ),
            ],
            [
                sg.Radio("Saturation", "Radio", size=(10, 1), key="-SATURATION-",font=("Cascadia Code SemiBold", 12)),
                sg.Slider(
                    (0, 100),
                    orientation="h",
                    size=(20, 15),
                    key="-SATURATION SLIDER-",
                )
            ],
            [
                sg.Radio("Blur", "Radio", size=(10, 1), key="-BLUR-",font=("Cascadia Code SemiBold", 12)),
                sg.Slider(
                    (1, 100),
                    orientation="h",
                    size=(40, 15),
                    key="-BLUR SLIDER-",
                ),
            ],
            [
                sg.Radio("Hue", "Radio", size=(10, 1), key="-HUE-",font=("Cascadia Code SemiBold", 12)),
                sg.Slider(
                    (0, 225),
                    orientation="h",
                    size=(40, 15),
                    key="-HUE SLIDER-",
                ),
            ],
            [
                sg.Radio("enhance", "Radio", size=(10, 1), key="-ENHANCE-",font=("Cascadia Code SemiBold", 12)),
                sg.Slider(
                    (1, 255),
                    orientation="h",
                    size=(40, 15),
                    key="-ENHANCE SLIDER-",
                ),
            ],
            [
                sg.Radio("GrayScale", "Radio", size=(10, 1), key="-GRAYSCALE-", font=("Cascadia Code SemiBold", 12)),
            ],

            [sg.Button("Exit", size=(10, 1), font=("Cascadia Code SemiBold", 12))],
    ],scrollable=True,vertical_scroll_only=True)
]

    # Define the window layout
    layout = [
        [
        sg.Column(
            [
                [sg.Text("Image Editing", font=("Cascadia Code SemiBold", 25))],
                [sg.Image(filename="", key="-IMAGE-", size=(800, 800))]],element_justification='center')
            ]
            ,
            column
        ]

    # Create the window and show it without the plot
    window = sg.Window("OpenCV Integration", layout, location=(400, 0),grab_anywhere=True,resizable=True,size=(600, 1000) )

    cap = cv2.VideoCapture(0)

    while True:
        event, values = window.read(timeout=20)

        if event == "Exit" or event == sg.WIN_CLOSED:
            break

        ret, frame = cap.read()

        if values["-BRIGHTNESS-"]:
            frame = ChangeBrightness(frame, values["-BRIGHTNESS SLIDER-"])
        elif values["-SATURATION-"]:
            frame = ChangeSaturation(frame, values["-SATURATION SLIDER-"])
        elif values["-BLUR-"]:
            frame = ImageSharpen(frame, values["-BLUR SLIDER-"])
        elif values["-HUE-"]:
            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
            frame[:, :, 0] += int(values["-HUE SLIDER-"])
            frame = cv2.cvtColor(frame, cv2.COLOR_HSV2BGR)
        elif values["-ENHANCE-"]:
            enh_val = values["-ENHANCE SLIDER-"] / 40
            clahe = cv2.createCLAHE(clipLimit=enh_val, tileGridSize=(8, 8))
            lab = cv2.cvtColor(frame, cv2.COLOR_BGR2LAB)
            lab[:, :, 0] = clahe.apply(lab[:, :, 0])
            frame = cv2.cvtColor(lab, cv2.COLOR_LAB2BGR)
        elif values["-OIL-"]:
            frame = OilPainting(
                frame,
                values["-OIL SLIDER SIZE-"],
                values["-OIL SLIDER LEVEL-"],
            )
        elif values["-WATER-"]:
            frame = Water(
                frame,
                NeighbourhoodSize = values["-WATER SLIDER SIZE-"],
                Range = values["-WATER SLIDER LEVEL-"]
            )
        elif values["-GRAYSCALE-"]:
            frame = GrayScale(frame)
        imgbytes = cv2.imencode(".png", frame)[1].tobytes()
        window["-IMAGE-"].update(data=imgbytes)

    window.close()

if __name__ == "__main__":
    main()


import getopt, sys
import csv
from utils import*

threshold=0
output="output.csv"
input_=""

# list of command line arguments
argumentList = sys.argv[1:]

#short options
options ="hi:t:o:"

#long options
long_options=["help", "input=", "threshold=","output="]

try:
	arguments, values = getopt.getopt(argumentList, options, long_options)

	if len(arguments)==1 and arguments[0][0] in ("-h", "--help"):
		helpMessageForRemovingFunctions()
	else:
		for i in range(len(arguments)):
			if arguments[i][0] in ("-h", "--help"):
				helpMessageForRemovingFunctions()
			elif arguments[i][0] in ("-i", "--input"):
				input_=str(arguments[i][1])
			elif arguments[i][0] in ("-t", "--threshold"):
				threshold = arguments[i][1]
			elif arguments[i][0] in ("-o", "--output"):
				output=str(arguments[i][1])

		outputFile=open(output,'w',newline='')
		writer=csv.writer(outputFile)

		data=readDataset(input_) #for DataFrame Structure to use functions in utils.py
		num_of_samples=getNumberOfSamples(data)

		columns=getColumns(data)
		titles=[]
		removed_cols=[]
		for c in columns:
			if computePercentageOfMissingValuesInColumn(data,c) < float(threshold):
				titles.append(c)
		writer.writerow(titles) #write columns name
		for i in range(num_of_samples):
			row=[]
			for c in titles:
				row.append(data[c][i])
			writer.writerow(row)
		for c in columns:
			if c not in titles:
				removed_cols.append(c)
		print('Removing done, checking in ', output)
		print('Columns are removed: ', removed_cols)
		outputFile.close()
except:
	showNotice()


# coding: UTF-8

import smbus
from debug import ERROR, WARN, INFO, DEBUG, TRACE

"""

Grove I2C Mini Motor Driver Class

DataSeet:https://www.seeedstudio.com/Grove-I2C-Mini-Motor-Driver.html

"""

class MiniMotorDriver:
    # @brief コンスタラクタ
    # @detail 初期化処理を行う
    def __init__(self, addr):
        TRACE('MiniMotorDriver generated')
        self._i2c = smbus.SMBus(1)
        self._addr = addr

    def drive(self, speed):
        reg_value = 0x80
        self._i2c.write_byte_data(self._addr, 0x01, 0x80)
        reg_value = abs(int(speed))
        if reg_value > 63:
            reg_value = 63
        reg_value = reg_value << 2
        if speed < 0:
            reg_value |= 0x01
        else:
            reg_value |= 0x02
        self._i2c.write_byte_data(self._addr, 0x00, reg_value)

    def stop(self):
        self._i2c.write_byte_data(self._addr, 0x00, 0x00)

    def brake(self):
        self._i2c.write_byte_data(self._addr, 0x00, 0x03)


from tkinter import *
from tkinter import ttk
from tkcalendar import DateEntry
import ghasedakpack
import datetime
def send_sms():
    # create api key
        sms = ghasedakpack.Ghasedak("39a79a573ba0e90ee353b76acb178b8d33ee0626ed96cd7cb9ec984cf2612ac6")
        date_today=datetime.date.today()
        # create formate date
        date_today = str(int(date_today.strftime("%d")))+"/"+str(int(date_today.strftime("%m")))+"/"+date_today.strftime("%Y")

        date_user=edate.get()
        if date_user==date_today:

            sms.send({ 'message':etext.get(),  'receptor' :eadress.get() })
            # زمان‌بندی فراخوانی تابع send_sms() هر یک دقیقه
        window.after(60000, send_sms)

window=Tk()
window.configure(bg="green")
window.geometry("500x600")
window.title("sms panel")
window.resizable(width=False,height=False)
Label(window,text="Sms Panel", fg="black",bg="yellow",cursor="arrow",width=71,height=2).place(x='5',y='1')
adress=Label(window,bg="green" ,text="  number phone    :  ")
adress.place(x='80',y='100')
eadress=Entry(window)
eadress.place(x='200',y='100')

text=Label(window,bg="green",text=" Enter your Message :  ")
text.place(x='80',y='180')
etext=Entry(window)
etext.place(x='200',y='180')

text=Label(window,text=" Enter your date :  ",bg="green")
text.place(x='80',y='260')
edate = DateEntry(window)
edate.place(x='200',y='260')

Button(window,text="send",command=send_sms,width=5,height=2).place(x='240',y='350')
window.after(60000, send_sms)  # زمان‌بندی اولیه فراخوانی تابع send_sms() هر یک دقی
window.mainloop()


#!/usr/bin/env python
# coding: utf-8

import pandas
import numpy
import sys
import os
import glob
import pybedtools
import argparse

def getOpts():
    parser = argparse.ArgumentParser(description='make-rna-cluster-sample-count-matrix')
    parser.add_argument('--counts', required=True, nargs='+', help ="""input gene by nuclei counts matrix. Column name syntax eg: <modality>.499.NM.1.<barcode> """)
    parser.add_argument('--cluster-info', required=True, type=str, help ="""Tab separated map of nuclei to cluster and sample. Must have columns modality, batch, sample, library, SNG.1ST, cluster_name, barcode""")
    parser.add_argument('--cluster-col', required=True, help ="""Name of the cluster column. Will output files per cluster""")
    parser.add_argument('--gene-info', help ="""If making qtltools input format file, provide gene info tsv with chrom, unique gene_name, start, end, strand""")
    parser.add_argument('--cols', required=True, nargs='+', help ="""Space separated column names from --cluster-info file to be used""")
    parser.add_argument('--batch-col', required=True, help ="""Name of the batch column. This will be used to confirm that nuclei for one sample come from only one batch""")
    parser.add_argument('--sample-col', required=True, help ="""Name of the sample column.""")
    parser.add_argument('--min-samples', type=int, default=1, help ="""No. of samples in which a gene should have a nonzero value to keep that gene""")
    parser.add_argument('--min-fusion-samples', type=int, default=1, help ="""No. of samples in which a gene should have a nonzero value to keep that gene for the eQTL subset file""")
    parser.add_argument('--separate-fusion-for-eqtl', action="store_true", help ="""Separate FUSION samples and organize for sample names to match vcf. Format output per qtltools requirements.""")
    parser.add_argument('--output',  help ="""Output prefix.""")
    args = parser.parse_args()
    return args

def subset_genes(d, min_samples):
    assert "clist" not in d.columns
    d['clist'] = d[[c for c in d.columns if c!="gene_name"]].astype(bool).sum(axis=1)
    d = d[d['clist'] >= min_samples]
    assert not d.empty
    d.drop('clist', axis=1, inplace=True)
    return d

if __name__ == '__main__':

    args = getOpts()

    cols = args.cols
    cluster_col = args.cluster_col
    sample_col = args.sample_col
    batch_col = args.batch_col

    metadata = pandas.read_csv(args.cluster_info, sep='\t', usecols=cols)
    metadata = metadata[metadata['modality']=="rna"]

    clusters = set(metadata[cluster_col].tolist())

    if args.separate_fusion_for_eqtl:
        chroms = [f"chr{i}" for i in list(range(1, 23)) ]
        geneinfo = pandas.read_csv(args.gene_info, sep='\t', usecols=['chrom', 'start', 'end', 'gene_name', 'strand']).drop_duplicates()
        geneinfo = geneinfo[geneinfo['chrom'].isin(chroms)]
        geneinfo.rename(columns = {'gene_name': "gid",
                                   'chrom': "#Chr"}, inplace=True)
        geneinfo['pid'] = geneinfo['gid']
        geneinfo['length'] = geneinfo.apply(lambda x: abs(x['end'] - x['start']), axis=1)
        geneinfo = geneinfo[['#Chr', 'start', 'end', 'pid', 'gid', 'strand', 'length']]

    maindict = {}
    for clust in clusters:
        maindict[clust] = []

    # get sum of counts for sample in each batch file
    for f in args.counts:
        mtx = pandas.read_csv(f, sep='\t')
        mtx.rename(columns = {'Unnamed: 0':'gene_name'}, inplace=True)
        meta = metadata[metadata['index'].isin(mtx.columns)]

        for clust, cgrp in meta.groupby(cluster_col):
            o = mtx[['gene_name']]
            for samp, grp in cgrp.groupby(sample_col):
                assert len(grp[batch_col].drop_duplicates().tolist()) == 1 # one sample should come from only 1 batch
                nuclei = grp['index'].tolist()
                o[samp] = mtx[nuclei].sum(axis=1)

            o.set_index('gene_name', inplace=True)
            maindict[clust].append(o)

    # concat samples across batches for each cluster
    # Subset genes to min nonzero exp in at least x samples
    for clust in maindict.keys():
        df = pandas.concat(maindict[clust], axis=1)
        df.fillna(0, inplace=True)
        df = df.astype(int)
        df = subset_genes(df, args.min_samples)

        samples = df.columns
        df.reset_index(inplace=True)
        print(df.head())
        df.rename(columns={'index':"gene_name"}, inplace=True)
        print(f"cluster {clust} had {len(samples)} samples and {len(df.index)} genes")
        df.to_csv(f"{args.output}.allsamples.{clust}.tsv", sep='\t', na_rep="NA", index=False)

        if args.separate_fusion_for_eqtl:

            fusion = metadata[(metadata['cohort']=="FUSION") & (metadata[cluster_col]==clust)]['sample'].drop_duplicates().tolist()
            fusion_samples = [s for s in fusion if s in df.columns]
            df = df[['gene_name'] + fusion_samples]
            if len(fusion_samples) == len(set([s.split('--')[0] for s in fusion_samples] )):
                rename = {s: s.split('--')[0] for s in fusion_samples}
                df.rename(columns = rename, inplace=True)

            df = subset_genes(df, args.min_fusion_samples)
            df.rename(columns = {'gene_name': "gid"}, inplace=True)
            df = pandas.merge(geneinfo, df, how="inner", on=['gid'])

            df.sort_values(['#Chr','start','end'], inplace=True)

            print(f"cluster {clust} had {len(samples)} FUSION samples and {len(df.index)} genes")
            df.to_csv(f"{args.output}.fusion.{clust}.tsv", sep='\t', na_rep="NA", index=False)


### write a script that resizes all images in a directory to 100 x 100 ###
import cv2
import glob

list_of_jpgs = glob.glob("batch_image_resizing/sample_images/*.jpg")

def trim_file_path(raw_path):
    return raw_path[35:]

trimmed_paths = list(map(trim_file_path, list_of_jpgs))

for index, image in enumerate(list_of_jpgs):
    print(index, image)
    cur_image = cv2.imread(image, 0)
    new_image = cv2.resize(cur_image, [100, 100])
    cv2.imwrite("batch_image_resizing/sample_images/resized_" + trimmed_paths[index], new_image)


class NumArray(object):
    def __init__(self, nums):
        """
        :type nums: List[int]
        """
        self.f = [0] * len(nums)
        for i, x in enumerate(nums):
            self.f[i] = self.f[i-1] + x

    def sumRange(self, i, j):
        """
        :type i: int
        :type j: int
        :rtype: int
        """
        if i > 0:
            prefix = self.f[i-1]
        else:
            prefix = 0
        return self.f[j] - prefix

if __name__ == '__main__':
    left, right = 0, 3
    nums = [-2, 0, 3, -5, 2, -1]
    obj = NumArray(nums)
    result = obj.sumRange(left, right)
    print(result)


import gc
import sys

foo = []

# 2 references, 1 from the foo var and 1 from getrefcount
print(sys.getrefcount(foo))

def bar(a):
    # 4 references
    # from the foo var, function argument, getrefcount and Python's function stack
    print(sys.getrefcount(a))

bar(foo)
# 2 references, the function scope is destroyed
print(sys.getrefcount(foo))

# For example, when a list is deleted the reference count for all its items is decreased.


"""
4 - Fizz Buzz - Se o parâmetro da função for divisível por 3, retorne fizz,
se o parâmetro da função for divisível por 5, retorne buzz. Se o parâmetro da
função for divisível por 5 e por 3, retorne FizzBuzz, caso contrário, retorne o
número enviado.
"""

def divisivel(parametro):
    if parametro%3 == 0:
        if parametro%5 == 0:
            return 'FizzBuzz'
        return 'Fizz'
    elif parametro%5 == 0:
        return 'Buzz'

    return parametro

valor = divisivel(25)
print(valor)

Módulo 2 - Python Intermediário (Programação Procedural)/Aula 34 - Exercícios Propostos/aula34_exercicio4.py


import numpy as np
import matplotlib.pyplot as plt
import ising as ising
from sklearn import linear_model
from sklearn.model_selection import train_test_split

#------------------#
# DATA PREPARATION #
#------------------#

# Ising states parameters
nsample = 1000   # number of data samples
nsites = 20       # number of ising sites
rndseed = 12      # random number generator seed

# generate random ising configurations
ising_states = ising.generate_states(nsample, nsites, rand_seed=rndseed)
# generate ising interaction matrix
ising_intmat = ising.interaction_matrix(ising_states)
# calculate total energy for ising states (y)
energy = ising.total_energy(ising_states)

# prepare data for model training
# features (X) : pair interactions for each configuration with size (nsample, nsite*nsite)
# labels (y) : total energy for each configuration (nsample, 1)
X = ising_intmat.reshape(1, -1, nsample).swapaxes(0, 2).squeeze()
y = energy

#--------------------------------#
# MODEL SELECTION AND EVALUATION #
#--------------------------------#

# split data
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=.33, random_state=rndseed)

# instantiate models
linreg_ols = linear_model.LinearRegression()    # ordinary least square regression
linreg_rdg = linear_model.Ridge()               # Ridge regression
linreg_las = linear_model.Lasso()               # Lasso regression

# train old model
linreg_ols.fit(X_train, y_train)
err_train_ols = linreg_ols.score(X_train, y_train)
err_test_ols = linreg_ols.score(X_test, y_test)
coef_ols = linreg_ols.coef_

# train ridge and lasso models
err_train_las = []
err_train_reg = []
err_test_las = []
err_test_reg = []
coef_las = []
coef_reg = []
regpara = np.logspace(-4, 5, 10)    # regularization parameter

for alpha in regpara:
    # ridge
    linreg_rdg.set_params(alpha=alpha)
    linreg_rdg.fit(X_train, y_train)
    # errors and coeffs
    err_train_reg.append(linreg_rdg.score(X_train, y_train))
    err_test_reg.append(linreg_rdg.score(X_test, y_test))
    coef_reg.append(linreg_rdg.coef_)

    # lasso
    linreg_las.set_params(alpha=alpha)
    linreg_las.fit(X_train, y_train)
    # errors and coeffs
    err_train_las.append(linreg_las.score(X_train, y_train))
    err_test_las.append(linreg_las.score(X_test, y_test))
    coef_las.append(linreg_las.coef_)

#-----------------------#
# ANALYZE MODEL OUTPUTS #
#-----------------------#

# plot train and test errors
plt.hlines(err_train_ols, min(regpara), max(regpara), colors='k', linestyles='dotted', label='Train OLS')
plt.hlines(err_test_ols, min(regpara), max(regpara), colors='k', linestyles='solid', label='Test OLS')
plt.semilogx(regpara, err_train_reg, '--ro', label="Train Ridge")
plt.semilogx(regpara, err_test_reg, 'ro', label="Test Ridge")
plt.semilogx(regpara, err_train_las, '--bo', label="Train Lasso")
plt.semilogx(regpara, err_test_las, 'bo', label="Test Lasso")
plt.xlim(min(regpara), max(regpara))
plt.xlabel("Regularization parameter")
plt.ylabel("Model score")
plt.legend()

# linear regression coefficients (interaction strength of ising pairs)
npara = regpara.size
fig, ax = plt.subplots(nrows=npara, ncols=3)

coef_reg = np.array(coef_reg).reshape(npara, nsites, nsites)
coef_las = np.array(coef_las).reshape(npara, nsites, nsites)
coef_ols = coef_ols.reshape(nsites, nsites)

for iax in range(npara):
    # Ridge
    ax[iax, 0].imshow(coef_reg[iax, :, :])
    # Lasso
    ax[iax, 1].imshow(coef_las[iax, :, :])
    # Ols
    ax[iax, 2].imshow(coef_ols)

plt.show()

# train and test dataset scores
# print("linreg_ols train set score:{:2.2f}".format(linreg_ols.score(X_train, y_train)))
# print("linreg_ols test set score:{:2.2f}".format(linreg_ols.score(X_test, y_test)))

# linear regression coefficients (interaction strength of ising pairs)
# coef_ols = linreg_ols.coef_
# coef_ols = coef_ols.reshape(nsites, nsites)

# imgplt = plt.imshow(coef_ols)
# imgplt.set_clim(-1, 1)
# plt.colorbar()
# plt.show()

# new sample
# new_state = ising.generate_states(1,nsites)
# new_energy = ising.total_energy(new_state)
# new_X = new_state.reshape(nsites, 1) * new_state
# new_X = new_X.reshape(1,-1)
# linreg_ols.predict(new_X)

# paper example
# X_train_exmpl = X[:400, :]
# y_train_exmpl = y[:400]
# X_test_exmpl = X[400:3*400//2, :]
# y_test_exmpl = y[400:3*400//2]

# linreg_ols_exmpl = linear_model.LinearRegression()
# linreg_ols_exmpl.fit(X_train_exmpl, y_train_exmpl)
# linreg_ols_exmpl.score(X_train_exmpl, y_train_exmpl)
# linreg_ols_exmpl.score(X_test_exmpl, y_test_exmpl)

# print scores
# print("model 1 train score: {:4.2f}".format(linreg_ols.score(X_train, y_train)))
# print("model 1 test score: {:4.2f}".format(linreg_ols.score(X_test, y_test)))
# print("model 2 train score: {:4.2f}".format(linreg_ols_exmpl.score(X_train_exmpl, y_train_exmpl)))
# print("model 2 test score: {:4.2f}".format(linreg_ols_exmpl.score(X_test_exmpl, y_test_exmpl)))
#


import json
import os
import base64
import string
import random
import boto3

QUEUE_URL = os.environ['QUEUE_URL']

def lambda_handler(event,context):
    content = "
Hello there "+event['headers']['x-forwarded-for']+" using "+event['headers']['user-agent']+"

... my favorite food is "+os.environ['MY_CONSTANT']+"
"
    print("Hello there", format(event['headers']['x-forwarded-for']), "using ", format(event['headers']['user-agent']), "... my favorite food is ", os.environ['MY_CONSTANT'])
    post_content = event['headers']['x-forwarded-for']+" /// "+event['headers']['user-agent']
    sqs = boto3.client('sqs')

    try:
        response = sqs.send_message(
            QueueUrl=QUEUE_URL,
            MessageBody=str(post_content),
            DelaySeconds=0
        )
        print (response)
    except Exception as e:
        raise IOError(e)

    return {
        'body': content,
        'headers': {
            'Content-Type': 'text/html'
        },

        'statusCode': 200
    }


#Setting up source
import os
import csv

budget_data = os.path.join("..", "Resources", "budget_data.csv")

# Setting Variables
total_months = 0
months_cycled = []

total_revenue = 0
revenue = []
past_revenue = 0

revenue_change = 0
revenue_change_list = []
average_revenue = 0

max_increase = ["", 0]
max_decrease = ["", 9999999]

#open the csv file
with open('budget_data.csv') as csvfile:
    csvreader = csv.reader(csvfile)

    #Loop through to find total months
    for row in csvreader:

        #totaling months
        total_months += 1

        #calculating Total $$$
        total_revenue = total_revenue + int(row["Profit/Losses"])

        #Calculating through cycled months
        revenue_change = float(row["Profit/Losses"])- past_revenue
        past_revenue = float(row["Profit/Losses"])
        revenue_change_list = revenue_change_list + [revenue_change]
        months_cycled = [months_cycled] + [row["Date"]]

        # Calculating max decreases & increases
        if revenue_change>max_increase[1]:
            max_increase[1]= revenue_change
            max_increase[0] = row['Date']

        if revenue_change<max_decrease[1]:
            max_decrease[1]= revenue_change
            max_decrease[0] = row['Date']
    average_revenue = sum(revenue_change_list)/len(revenue_change_list)

#produce written file
text = "analysis.txt"

#write changes to csv
with open(text, 'w') as file:
    file.write("Financial Analysis\n")
    file.write("Total Months: %d\n" % total_months)
    file.write("Total Revenue: $(%d)\n" % total_revenue)
    file.write("Average Revenue Change $%d\n" % average_revenue)
    file.write("Greatest Increase in Revenue: %s ($%s)\n" % (max_increase[0], max_increase[1]))
    file.write("Greatest Decrease in Revenue: %s ($%s)\n" % (max_decrease[0], max_decrease[1]))


from trlc import ast
from trlc.errors import Message_Handler, TRLC_Error
from trlc.vcg import VCG

class Linter:
    def __init__(self, mh, stab, verify_checks, debug_vcg, cvc5_binary):
        # lobster-exclude: Not safety relevant
        assert isinstance(mh, Message_Handler)
        assert isinstance(stab, ast.Symbol_Table)
        assert isinstance(verify_checks, bool)
        assert isinstance(debug_vcg, bool)
        assert isinstance(cvc5_binary, str) or cvc5_binary is None

        self.mh            = mh
        self.stab          = stab
        self.verify_checks = verify_checks
        self.debug_vcg     = debug_vcg
        self.cvc5_binary   = cvc5_binary

        self.abstract_extensions = {}
        self.checked_types       = set()

    def verify(self):
        # lobster-exclude: Not safety relevant
        ok = True
        for package in self.stab.values(ast.Package):
            for n_typ in package.symbols.values(ast.Type):
                try:
                    self.verify_type(n_typ)
                except TRLC_Error:
                    ok = False

        # Complain about abstract types without extensions
        for package in self.stab.values(ast.Package):
            for n_typ in package.symbols.values(ast.Record_Type):
                if n_typ.is_abstract and not self.abstract_extensions[n_typ]:
                    self.mh.check(
                        n_typ.location,
                        "abstract type %s does not have any extensions" %
                        n_typ.name,
                        "abstract_leaf_types")

        return ok

    def verify_type(self, n_typ):
        # lobster-exclude: Not safety relevant
        assert isinstance(n_typ, ast.Type)

        if n_typ in self.checked_types:
            return
        else:
            self.checked_types.add(n_typ)

        if isinstance(n_typ, ast.Record_Type):
            self.verify_record_type(n_typ)

        elif isinstance(n_typ, ast.Tuple_Type):
            self.verify_tuple_type(n_typ)

        elif isinstance(n_typ, ast.Array_Type):
            self.verify_array_type(n_typ)

    def verify_tuple_type(self, n_tuple_type):
        assert isinstance(n_tuple_type, ast.Tuple_Type)

        # Detect confusing separators
        # lobster-trace: LRM.Tuple_Based_Literal_Ambiguity
        previous_was_int     = False
        previous_was_bad_sep = False
        bad_separator        = None
        location             = None
        for n_item in n_tuple_type.iter_sequence():
            if previous_was_bad_sep:
                assert isinstance(n_item, ast.Composite_Component)
                if isinstance(n_item.n_typ, ast.Builtin_Integer):
                    explanation = [
                        "For example 0%s100 would be a base %u literal" %
                        (bad_separator,
                         {"b" : 2, "x" : 16}[bad_separator]),
                        "instead of the tuple segment 0 %s 100." %
                        bad_separator
                    ]
                else:
                    explanation = [
                        "For example 0%s%s would be a lexer error" %
                        (bad_separator,
                         n_item.n_typ.get_example_value()),
                        "instead of the tuple segment 0 %s %s." %
                        (bad_separator,
                         n_item.n_typ.get_example_value())
                    ]

                self.mh.check(
                    location,
                    "%s separator after integer component"
                    " creates ambiguities" % bad_separator,
                    "separator_based_literal_ambiguity",
                    "\n".join(explanation))

            elif isinstance(n_item, ast.Composite_Component) and \
               isinstance(n_item.n_typ, ast.Builtin_Integer):
                previous_was_int = True

            elif isinstance(n_item, ast.Separator) and \
                 previous_was_int and \
                 n_item.to_string() in ("x", "b"):
                previous_was_bad_sep = True
                bad_separator        = n_item.to_string()
                location             = n_item.location

            else:
                previous_was_int     = False
                previous_was_bad_sep = False

        # Walk over components
        for n_component in n_tuple_type.components.values():
            self.verify_type(n_component.n_typ)

        # Verify checks
        if self.verify_checks:
            vcg = VCG(mh          = self.mh,
                      n_ctyp      = n_tuple_type,
                      debug       = self.debug_vcg,
                      use_api     = self.cvc5_binary is None,
                      cvc5_binary = self.cvc5_binary)
            vcg.analyze()

    def verify_record_type(self, n_record_type):
        # lobster-exclude: Not safety relevant
        assert isinstance(n_record_type, ast.Record_Type)

        # Mark abstract extensions
        if n_record_type.is_abstract:
            if n_record_type not in self.abstract_extensions:
                self.abstract_extensions[n_record_type] = set()
        elif n_record_type.parent and n_record_type.parent.is_abstract:
            if n_record_type.parent not in self.abstract_extensions:
                self.abstract_extensions[n_record_type.parent] = set()
            self.abstract_extensions[n_record_type.parent].add(n_record_type)

        # Walk over components
        for n_component in n_record_type.components.values():
            self.verify_type(n_component.n_typ)

        # Verify checks
        if self.verify_checks:
            vcg = VCG(mh          = self.mh,
                      n_ctyp      = n_record_type,
                      debug       = self.debug_vcg,
                      use_api     = self.cvc5_binary is None,
                      cvc5_binary = self.cvc5_binary)
            vcg.analyze()

    def verify_array_type(self, n_typ):
        # lobster-exclude: Not safety relevant
        assert isinstance(n_typ, ast.Array_Type)

        if n_typ.upper_bound is None:
            pass
        elif n_typ.lower_bound > n_typ.upper_bound:
            self.mh.check(n_typ.loc_upper,
                          "upper bound must be at least %u" %
                          n_typ.lower_bound,
                          "impossible_array_types")
        elif n_typ.upper_bound == 0:
            self.mh.check(n_typ.loc_upper,
                          "this array makes no sense",
                          "impossible_array_types")
        elif n_typ.upper_bound == 1 and n_typ.lower_bound == 1:
            self.mh.check(n_typ.loc_upper,
                          "array of fixed size 1 "
                          "should not be an array",
                          "weird_array_types",
                          "An array with a fixed size of 1 should not\n"
                          "be an array at all.")
        elif n_typ.upper_bound == 1 and n_typ.lower_bound == 0:
            self.mh.check(n_typ.loc_upper,
                          "consider making this array an"
                          " optional %s" % n_typ.element_type.name,
                          "weird_array_types",
                          "An array with 0 to 1 components should just\n"
                          "be an optional %s instead." %
                          n_typ.element_type.name)


import random

import numpy

from entity.individual import Individual

class Population(list):
    def __init__(self, size=0, **kwargs):
        super(Population, self).__init__()

        for _ in range(size):
            no_eval = kwargs.pop("no_eval", True)
            self.append(Individual(no_eval=no_eval, **kwargs))

    def objectives(self):
        objs = numpy.vstack([individual.objectives for individual in self])
        avg_objs = numpy.nanmean(objs, axis=0)
        return "Minimize Objective(instruction_length={}, perplexity={}, reciprocal_metric={})".format(
            round(avg_objs[0], 2), round(avg_objs[1], 2), round(avg_objs[2], 2)
        )
        # return "Minimize Objective(instruction_length={}, reciprocal_metric={})".format(
        #     round(avg_objs[0], 2), round(avg_objs[1], 2)
        # )

    def init_from_list(self, list_of_individuals):
        for individual in list_of_individuals:
            self.append(individual)
        return self

    def __str__(self):
        return "\n".join([str(individual) for individual in self])

    def __repr__(self):
        return "\n".join([str(individual) for individual in self])

if __name__ == "__main__":
    p = Population()
    print(p[0].fitness + p[1].fitness)
    print(p)
    print(len(p))


# -*- coding: utf-8 -*-

import cv2 as cv

#读取图像
src=cv.imread('example.png')

#namedWindow函数，用于创建一个窗口
cv.namedWindow('input_image')

#指定窗口显示图像，‘input_image'：窗口名；src：要显示的图像
cv.imshow('input_image', src)

#翻转
h_flip = cv.flip(src, 1)#水平翻转
v_flip = cv.flip(src, 0)#垂直翻转
hv_flip = cv.flip(src, -1)#水平垂直翻转

#保存图像
cv.imwrite("v_flip.jpg", v_flip)
cv.imwrite("h_flip.jpg", h_flip)
cv.imwrite("hv_flip.jpg", hv_flip)

#绑定键盘，0指永远
cv.waitKey(0)

#删除所有窗口，可用窗口名做参数表示删除指定窗口
cv.destroyAllWindows()


# solution to Advent of Code 2018, day 7 part one and two
# https://adventofcode.com/2018/day/7
#
# Usage: sleigh.py [inputfile]

from collections import defaultdict
import fileinput
import re

def main():
    steplist = set([re.findall(r'\s([A-Z]).*([A-Z])', line).pop() for line in fileinput.input()])

    # graph is a dict of tuples
    # dict key is the name of the step ('A', 'B', 'C', etc)
    # tuple is (set(), set()) where the sets contain inbound
    # and outbound nodes relating to the dict key
    graph = defaultdict(tuplesets)
    INBOUND = 0
    OUTBOUND = 1
    for a, b in steplist:
        graph[a][OUTBOUND].add(b)
        graph[b][INBOUND].add(a)

    # solve part one
    open_nodes = set()
    visited = set()
    steps = ''
    [open_nodes.add(node) for node, related in graph.items() if not related[INBOUND]]

    while open_nodes:
        next_node = min([node for node in open_nodes if graph[node][INBOUND] <= visited])
        open_nodes |= graph[next_node][OUTBOUND]
        steps += next_node
        visited.add(next_node)
        open_nodes.remove(next_node)

    print('Step order:', steps)

    # solve part two
    workers = [None] * 5
    open_nodes = set()
    done = set()
    [open_nodes.add(node) for node, related in graph.items() if not related[INBOUND]]

    seconds = 0
    finished = False
    while not finished:
        finishing_workers = [i for i,worker in enumerate(workers) if not worker == None and worker[0] == seconds]
        for i in finishing_workers:
            finished_step = workers[i][1]
            done.add(finished_step)
            open_nodes |= graph[finished_step][OUTBOUND]
            workers[i] = None

        if len(done) == len(graph.keys()):
            finished = True
            continue

        available_workers = [i for i,worker in enumerate(workers) if worker == None]
        if not available_workers:
            seconds += 1
            continue
        for i in available_workers:
            potential_next = [node for node in open_nodes if graph[node][INBOUND] <= done]
            if not potential_next:
                continue
            next_node = min(potential_next)
            workers[i] = ((seconds + ord(next_node) - 4), next_node)
            open_nodes.remove(next_node)
        seconds += 1

    print('Seconds to completion with 5 workers:', seconds)

def tuplesets():
    return (set(), set())

if __name__ == '__main__':
    main()


def printPattern(n, shift = 0):
    if n == 1:
        print(shift * ' ' + '*')
    else:
        printPattern(n // 2, shift)
        print(shift * ' ' + n * '*')
        printPattern(n // 2, shift + n // 2)

def gcd(m, n):
    if n == 0:
        return m
    elif m == 0:
        return n
    elif m >= n:
        return gcd(n, m - n)
    elif n > m:
        return gcd(m, n - m)

def count(lst, target):
    if type(lst) != list:
        return lst
    else:
        counter = 0
        for num in lst:
            if num == target:
                counter += 1
            elif type(num) == list:
                counter += count(num, target)
        return counter

if __name__=='__main__':
    import doctest
    print( doctest.testfile( 'lab8TEST.py' ))


from django.urls import path

from apps.task2.views import ListFeedbackAPIView, CreateFeedbackAPIView, DetailServiceAPIView, DeleteFeedbackAPIView

urlpatterns = [
    path('list/feedbacks/', ListFeedbackAPIView.as_view(), name='list_feedback'),
    path('create/feedback/', CreateFeedbackAPIView.as_view(), name='create_feedback'),
    path('detail/service/<int:pk>/', DetailServiceAPIView.as_view(), name='detail_service'),
    path('delete/feedback/<int:pk>/', DeleteFeedbackAPIView.as_view(), name='delete_feedback')

]


import datetime
from datetime import datetime, timedelta, date
from django.shortcuts import render, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect
from django.template.loader import get_template
from django.core.mail import send_mail, EmailMultiAlternatives
from django.views import generic
from django.urls import reverse
from django.utils.safestring import mark_safe
import calendar
from django.contrib.auth.models import User
from django.shortcuts import render, redirect, get_object_or_404
from .models import *
from .utils import Calendar
from .forms import *
from .emails import *

def index(request):
    return HttpResponse('hello')

def send_email_attachment(request, title, description, start_time, end_time, send_to, message):
    user_email = request.user.email
    message = EmailMultiAlternatives(subject=title, body=description,from_email=user_email, to=[send_to])
    html_templ = get_template("cal/email.html").render({'title':title, 'description': description, "start_time":start_time, "end_time":end_time, "send_to":send_to, 'message':message,})
    message.attach_alternative(html_templ, "text/html")
    message.send()
    print("email sent")

def detail(request, event_id=None):
    context = {}
    print(event_id)
    print(request.user.email)
    template_name = 'cal/detail.html'

    if not request.user.is_authenticated:
        return HttpResponse('oops, 404')
    if event_id:
        event = get_object_or_404(Event, pk=event_id)
        length = (event.start_time - event.end_time)
        # length.strtime('%Y-%m-%d')
        event.save()
        context['event'] = event
        context['length'] = length
        if request.method == 'GET':
            form = EmailForm(request.POST)
        if request.method == 'POST':
            form = EmailForm(request.POST)
        # all_files = request.FILES.getlist('files')
            if form.is_valid():
                obj = form.save(commit=False)
                # title, description, start_time, end_time, send_to
                obj.title = event.title
                obj.description = event.description
                obj.start_time = event.start_time
                obj.end_time = event.end_time
                obj.send_to = request.POST.get('to')
                obj.message = request.POST.get('message')
        #     recipient_list = [form.cleaned_data.get("to")]
        #     files = form.cleaned_data.get("all_files")
        #     subject = form.cleaned_data.get("subject")
        #     message = form.cleaned_data.get("message")
        #     message += str(files)
        #     email_from = 'welcome.amongus@gmail.com'

        #     obj.to = recipient_list
        #     obj.subject = subject
        #     obj.message = message
        #     obj.files = all_files
                obj.save()
                # title, description, start_time, end_time, send_to, message
                send_email_attachment(request, event.title, event.description, event.start_time, event.end_time, obj.send_to, obj.message)
            # htmly = get_template('cal/email.html')
            # send_email_attachment()
            # html_content = htmly.render({'order':order, "email":email, 'mattress': mattress})
            # send_mail(subject, message, email_from, recipient_list, fail_silently=False)
                print('sent')
            else:
                form = EmailForm()
            context['form'] = form
        context['form'] = form
        #     return HttpResponse('404')
            # form = EmailForm()
    # if request.method == 'GET':
    #     return render(request, template_name, context)
    # context = {'form': form}

    # event = get_object_or_404(Event, pk=event_id)
    return render(request, template_name, context)

class CalendarView(generic.ListView):
    model = Event
    template_name = 'cal/calendar.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        d = get_date(self.request.GET.get('month', None))
        cal = Calendar(d.year, d.month)
        html_cal = cal.formatmonth(withyear=True)
        today = datetime.now().date()
        context['today'] = today
        context['calendar'] = mark_safe(html_cal)
        context['prev_month'] = prev_month(d)
        context['next_month'] = next_month(d)
        return context

def get_date(req_month):
    if req_month:
        year, month = (int(x) for x in req_month.split('-'))
        return date(year, month, day=1)
    return datetime.today()

def prev_month(d):
    first = d.replace(day=1)
    prev_month = first - timedelta(days=1)
    month = 'month=' + str(prev_month.year) + '-' + str(prev_month.month)
    return month

def next_month(d):
    days_in_month = calendar.monthrange(d.year, d.month)[1]
    last = d.replace(day=days_in_month)
    next_month = last + timedelta(days=1)
    month = 'month=' + str(next_month.year) + '-' + str(next_month.month)
    return month

def event(request, event_id=None):
    instance = Event()
    if event_id:
        instance = get_object_or_404(Event, pk=event_id)
    else:
        instance = Event()

    form = EventForm(request.POST or None, instance=instance)
    if request.POST and form.is_valid():
        form.save()
        # print(f'{event.get_html_url}')
        return HttpResponseRedirect(reverse('cal:calendar'))
    return render(request, 'cal/event.html', {'form': form})


def find_high_boom(n, ground, booms):
    values = [[0 for _ in range(n)] for __ in range(n)]
    directions = [(0, 1), (0, -1), (1, 0), (-1, 0), (0, 0)]

    for y, x in booms:
        for dy, dx in directions:
            ny, nx = y - 1 + dy, x - 1 + dx
            if 0 <= ny < n and 0 <= nx < n and ground[ny][nx] != "#":
                values[ny][nx] += 2 if ground[ny][nx] == "@" else 1

    ans = 0
    for v in values:
        ans = max(ans, max(v))

    return ans

if __name__ == "__main__":
    N, K = map(int, input().split())
    GROUND = [list(input().split()) for _ in range(N)]
    BOOMS = [tuple(map(int, input().split())) for _ in range(K)]
    print(find_high_boom(N, GROUND, BOOMS))


import random

def create_maze(rows, columns):   #takes 2 parameters rows and columns
    maze = [[0]*columns for _ in range(rows)] #creates a 2D list with specified no. of rows and cols, each cell in the maze is initially 0
    return maze #returns the maze created

def set_start_goal(maze, rows, columns):
    start_node = (random.randint(0, rows-1), random.randint(0, 1)) #randomly selects a cell in the first two columns to be the start cell.
    goal_node = (random.randint(0, rows-1), random.randint(columns-2, columns-1)) #randomly selects a cell in the last two columns to be the goal cell.
    maze[start_node[0]][start_node[1]] = 7 #sets the start cell in maze to 7
    maze[goal_node[0]][goal_node[1]] = 8 # sets the goal cell in the maze to 8
    return start_node, goal_node

def set_barriers(maze, rows, columns, num_barriers):
    for _ in range(num_barriers):
        while True: #for each iteration while loop runs until it finds an empty cell in the maze.
            barrier = (random.randint(0, rows-1), random.randint(0, columns-1)) # randomly selects a cell in the maze.
            if maze[barrier[0]][barrier[1]] == 0: #indicating cell is empty
                maze[barrier[0]][barrier[1]] = 1 # indicating cell is a barrier so sets the value of that cell to one.
                break #breaks out the while loop

def print_maze(maze):
    for row in maze:
        print(row)

rows, columns = 6, 6  #creates a 6 by 6 maze
maze = create_maze(rows, columns)
start_node, goal_node = set_start_goal(maze, rows, columns)
set_barriers(maze, rows, columns, 4) #places four barriers randomly
print_maze(maze)

#TASK 2 - DFS ALGORITHM

def moves_valid(x_value, y_value, maze): # defined to find all valid moves from a given cell in the maze
    next_valid_move = [] # empty list is initialized to store the valid moves.

    maze_directions = [(0, -1), #West
                       (-1, 0), #North
                       (0, 1), #East
                       (1, 0), #South
                       (-1, -1), #Northwest
                       (-1, 1), #Northeast
                       (1, -1), #Southwest
                       (1, 1) #Southeast
                    ]
    #  list contains eight possible directions to move in the form of (dx, dy).

    for x,y in maze_directions: # iterates over each direction in maze_directions
        new_x = x_value + x # calculate the coordinates of the neighboring cell in the current direction.
        new_y = y_value + y

        if 0 <= new_x < len(maze) and 0 <= new_y < len(maze[0]) and maze[new_x][new_y] != 1:
            # checks if the new coordinates are within the maze boundaries and if the cell is not a barrier then move is considered valid
            next_valid_move.append((new_x,new_y)) # adds the coordinates of the valid move to the list.

    return next_valid_move # returns the list of valid neighboring moves

# Apply DFS search to the maze to find the goal

def DFS_Process(maze, stack, visited_cells): #a stack to store cells to be explored, and a set to store visited cells.
    start = (start_cell_column, start_cell_row) #set the start and goal cells based on their row and column indices.
    goal = (goal_cell_column, goal_cell_row)
    stack.append(start) #  adds the start cell to the stack.

    while stack: #  runs as long as there are cells in the stack to be explored.
        current_explore_cell = stack.pop() #  removes and returns the last cell from the stack to be explored.

        if current_explore_cell == goal:
            print("Goal found")   #checks if the current cell is the goal. If it is, it prints “Goal found”
            print("This is the Final path", visited_cells)
            Total_Nodes = len(visited_cells)
            print("Time taken to find the goal: ", Total_Nodes, "minutes")
            return

        a = current_explore_cell[0] # extract the x and y coordinates of the current cell.
        b = current_explore_cell[1]
        next_steps = moves_valid(a, b, maze) #function to get a list of valid neighboring moves from the current cell.

        # Sort the next steps based on increasing order
        next_steps.sort() #  sorts these moves in increasing order

        for next_cell in next_steps: # iterates over each valid move.
            if next_cell in visited_cells: #  checks if a move has already been visited.if so,it skips that move
                continue
            visited_cells.add(next_cell) # adds that move to visited cells
            stack.append(next_cell) #adds it to the stack

start_cell_row, start_cell_column = 0, 0 #you initialize your start and goal cells with specific row and column indices
goal_cell_row, goal_cell_column = 4, 4

# "explore_cells_stack" list works as a stack
explore_cells_stack = [] #initializes stack
# visited set doesn't contain repeated cells.
visited = set()

DFS_Process(maze, explore_cells_stack, visited)

print("\n")
print("End of DFS Search Algorithm")

#Task 03 - heuristic cost

#This function computes the Manhattan distance between two grid points.
# It is used as a heuristic to estimate the cost of moving from one cell to another.
def manhattan_distance_calculation(goal_row_distance, goal_col_distance, row, column):
    return abs(goal_row_distance - row) + abs(goal_col_distance - column)

# A* search algorithm
def a_star_search(insert_maze, start_row, start_column, goal_row, goal_column):
    cell_value_list = []  #empty list to store the cell cordinates

    # Priority queue for A* search
    priority_queue = [] #initializing the priority queue

    visited_nodes_list = [] #list to store the visited nodes
    heuristic_value_dictionary = {}  # dictionary used to map the heuristic values to its respective cell

    start_node = (start_column, start_row) #initialize start and goal nodes
    goal_node = (goal_column, goal_row)
    child_node = start_node

    # Dictionary to find the path from start to the goal node
    path_reverse_dic = {}

    # Updating heuristic cost to dictionary from manhattan_distance_calculation function
    heuristic_value = [manhattan_distance_calculation(goal_row, goal_column, i, j) for i in range(6) for j in range(6)]
    print("Heuristic Values")
    print(heuristic_value)
    print("\n")

    # generate a list of all the cell coordinates in the maze, followed by a dictionary that maps each cell to
    # its matching heuristic value.
    for i in range(6): #The rows and columns of a 6x6 grid are iterated over in this nested loop
        for j in range(6):
            temp_cell = (j, i) #It generates a tuple temp_cell indicating the coordinates of a cell in the maze (j, i)
            cell_value_list.append(temp_cell) #for each combination of i and j and appends it to the cell_value_list.
    for i in range(36): #Iterates over the items of cell_value_list, which now includes every possible cell coordinates
        value = cell_value_list[i]
        heuristic_value_dictionary[value] = heuristic_value[i]#It assigns a corresponding heuristic value from the previously derived
        #heuristic_value list to each coordinate.
    print("Heuristic value and the respective cell")
    print(heuristic_value_dictionary)
    print("\n")

    # The dictionary g_score_dict defines the cost of the path from the start node to any other cell in the maze.
    g_score_dict = {cell: float('inf') for cell in cell_value_list}
    g_score_dict[start_node] = 0 #cost from start node to start node is set to zero

    # The dictionary f_score_dict represents the overall cost of the path from the start node to any other cell,
    # taking into account both the actual cost (g(n)) and the heuristic estimate to the goal (h(n)).
    f_score_dict = {cell: float('inf') for cell in cell_value_list}
    f_score_dict[start_node] = heuristic_value_dictionary[start_node]

    priority_queue.append(start_node) #adds the start node to priority queue

    #main loop for A* Search
    #A* searches are carried out repeatedly until the priority queue is cleared. In the priority queue,
    # the node with the lowest total cost is processed at each iteration.
    while len(priority_queue) != 0:
        curr_cell_node = priority_queue.pop(0) #pop the current node from the queue
        start_column = curr_cell_node[0] #updating the cordinates of the current cell
        start_row = curr_cell_node[1]
        visited_nodes_list.append(curr_cell_node)#add the current node to the visted list

        # Define the allowed directions to move - up, down, left, right, and diagonal
        west_search = insert_maze[start_column][start_row - 1]

        if start_column < 5:
            north_search = insert_maze[start_column + 1][start_row]

        if start_row < 5:
            east_search = insert_maze[start_column][start_row + 1]

        south_search = insert_maze[start_column - 1][start_row]

        if start_column < 5:
            L = insert_maze[start_row - 1][start_column + 1]  # northwest_search

        if start_column < 5 and start_row < 5:
            M = insert_maze[start_row + 1][start_column + 1]  # northeast_search

        if start_row < 5:
            P = insert_maze[start_row + 1][start_column - 1]  # southeast_search

        O = insert_maze[start_row - 1][start_column - 1]  # southwest_search

        if curr_cell_node == goal_node:
            print("Goal has been found")
            # print("Visited nodes")
            print(visited_nodes_list)
            length_node_list = len(visited_nodes_list)
            print("Time taken to find the goal: ", length_node_list, "minutes")
            break
        else:
            for i in "ESNWLMPO":  # Search process

                if i == "W":
                    if start_row - 1 >= 0 and west_search != 1:
                        child_node = (start_column, start_row - 1)

                elif i == "N":
                    if start_column + 1 < len(insert_maze) and north_search != 1:
                        child_node = (start_column + 1, start_row)

                elif i == "S":
                    if start_column - 1 > 0 and south_search != 1:
                        child_node = (start_column - 1, start_row)

                elif i == "E":
                    if start_row + 1 < len(insert_maze) and east_search != 1:
                        child_node = (start_column, start_row + 1)

                elif i == "L":
                    if (start_row - 1 >= 0 and start_column + 1 < len(insert_maze)) and L != 1:
                        child_node = (start_column + 1, start_row - 1)

                elif i == "M":
                    if (start_row + 1 < len(insert_maze) and start_column + 1 < len(insert_maze)) and M != 1:
                        child_node = (start_column + 1, start_row + 1)

                elif i == "P":
                    if (start_row + 1 < len(insert_maze) and start_column - 1 >= 0) and P != 1:
                        child_node = (start_column - 1, start_row + 1)

                elif i == "O":
                    if (start_row - 1 >= 0 and start_column - 1 >= 0) and O != 1:
                        child_node = (start_column - 1, start_row - 1)

                temp_g_value = g_score_dict[curr_cell_node] + 1
                temp_final_value = temp_g_value + heuristic_value_dictionary[child_node]

                if temp_final_value < f_score_dict[child_node]:
                    g_score_dict[child_node] = temp_g_value
                    f_score_dict[child_node] = temp_final_value
                    priority_queue.append(child_node)
                    path_reverse_dic[child_node] = curr_cell_node

    cell_temp = (goal_column, goal_row)
    path_forward = {}
    while cell_temp != start_node:
        path_forward[path_reverse_dic[cell_temp]] = cell_temp
        cell_temp = path_reverse_dic[cell_temp]

    print("\n")
    print("The final path :")
    print(path_forward.values())

# Calculate Manhattan distance for each cell in the maze
manhattan_distance_calculation(goal_cell_row, goal_cell_column, rows, columns)
# Perform A* search using Manhattan distance as the heuristic
a_star_search(maze, start_cell_row, start_cell_column, goal_cell_row, goal_cell_column)


from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
from statsmodels.tsa.stattools import kpss
import pandas as pd
import nasdaqdatalink
import matplotlib.pyplot as plt

df = (
    nasdaqdatalink.get(dataset="FRED/UNRATENSA",
                       start_date="2010-01-01",
                       end_date="2019-12-31")
        .rename(columns={"Value": "unemp_rate"})
)

def kpss_test(x, h0_type="c"):
    """A value of 'c' correspponds to the null hypothesis stating that the series is level-stationary, while 'ct'
    corresponds to trend-stationary (removing the trend from the series would makr it level-stationary"""
    indices = ["Test Statistic", "p-value", "# of Lags"]

    kpss_test = kpss(x, regression=h0_type)
    results = pd.Series(kpss_test[0:3], index=indices)

    for key, value in kpss_test[3].items():
        results[f"Critical Value ({key})"] = value

    return results

print(kpss_test(df["unemp_rate"]))

# Null hypothesis of the KPSS test states that the time series is stationary. With a p-value of 0.01 (or a test
# statistic greater than the selected critical value), we have reasons to reject the null hypothesis in favor of the
# alternative one, indicating that the series is not stationary.

# Generate the ACF/PACF plots:

N_LAGS = 40
# Choosing 5% which indicates the probability of rejecting the null hypothesis (H0) when it is in fact true.
SIGNIFICANCE_LEVEL = 0.05

fig, ax = plt.subplots(2, 1)
plot_acf(df["unemp_rate"], ax=ax[0], lags=N_LAGS, alpha=SIGNIFICANCE_LEVEL)
plot_pacf(df["unemp_rate"], ax=ax[1], lags=N_LAGS, alpha=SIGNIFICANCE_LEVEL)

plt.show()
# In this series of data and ACF plot we can see that there are "significant autocorrelations (above the 95% confidence
# interval, corresponding to the selected 5% signficance level). There are also some significant auto-correlations at
# lags 1 and 4 in the PACF plot.


from bs4 import BeautifulSoup
import requests

star_wars_movies = requests.get('https://en.wikipedia.org/wiki/Rogue_One')
soup = BeautifulSoup(star_wars_movies.content, 'lxml')

contents = soup.prettify()
info_box = soup.find(class_="infobox vevent")
info_rows = info_box.find_all('tr')
movie_info = {}

def get_content_value(row_data):
    if row_data.find('li'):
        return [li.get_text(' ', strip=True).replace('\xa0', '') for li in row_data.find_all('li')]
    else:
        return row_data.get_text(' ', strip=True).replace('\xa0', '')

for index, row in enumerate(info_rows):
    if index == 0:
        movie_info['title'] = row.find('th').get_text(' ', strip=True)
    elif index == 1:
        continue
    else:
        content_key = row.find('th').get_text(' ', strip=True)
        content_value = get_content_value(row.find('td'))
        movie_info[content_key] = content_value


import gzip
import json
import os
import sys
import requests
import warcio
from warcio.archiveiterator import ArchiveIterator
from collections import defaultdict
from bs4 import BeautifulSoup

from tqdm import tqdm

# Our process at the moment is (example forbes.com) :
# We go to index.commoncrawl.org and enter the url we want to search for -> this returns a json file with the scrapped data and the location of the files which include data about the website for example forbes
# Then we filter for the filenames and download and extract them -> WARC files

# TODO What is missing
# Look at what we need from the WARC files -> and save the text/title for sentiment analysis
# At the moment it is streamlined for one website (forbes) -> we need to change it so it can be used for multiple websites and automate it so we dont need to manually enter the websites

# Things to look into
# https://github.com/commoncrawl/cc-index-server
# https://github.com/webrecorder/pywb/wiki/CDX-Server-API#api-reference

# Good to know
# cc-index.paths.gz -> basically a list of all the files that are available
# cdx-00000.gz -> the index file for the first file in cc-index.paths.gz

########################################################################################################################
# Load the JSON file containing the links
def get_links_from_json(json_file_path):
    try:
        with open(json_file_path, 'r') as json_file:
            data = json.load(json_file)
            if 'links' in data:
                websites = [link['url'] for link in data['links']]
                for website in websites:
                    print(website)  # TODO: change depending on the filter requirements for commoncrawl.org data
            else:
                print("No 'links' key found in the JSON data.")
    except FileNotFoundError:
        print(f"The JSON file '{json_file_path}' was not found.")
    except json.JSONDecodeError as e:
        print(f"Error decoding JSON: {e}")

# This is used to show the progress of the download
def bar_custom(current, total, width=80):
    progress_message = "Downloading: %d%% [%d / %d]" % (current / total * 100, current, total)
    sys.stdout.write("\r" + progress_message)
    sys.stdout.flush()

# we can use this method to get the info on where the files are located that we need to download
def download_json_from_index_website():

    output_path = 'data/json_data/CC-MAIN-2023-23-index_forbes.json'
    if not os.path.exists(output_path):
        # wget.download(
        #     "https://index.commoncrawl.org/CC-MAIN-2023-23-index?url=https%3A%2F%2Fwww.forbes.com%2F*&output=json",
        #     out=output_path, bar=bar_custom)
        req_down(
            "https://index.commoncrawl.org/CC-MAIN-2023-23-index?url=https%3A%2F%2Fwww.forbes.com%2Fsites%2Fa*&output=json",
            out=output_path)
    else:
        print(f"{output_path} already exists -> skipping download")

# takes the downloaded json file from download_json_from_index_websites of a website and filters out the needed filenames
# data looks like this: 0,146,201,84)/en/projects/aleksandr-solzhenitsyn-conference 20230610172419 {"url": "https://84.201.146.0/en/projects/aleksandr-solzhenitsyn-conference", "mime": "text/html", "mime-detected": "text/html", "status": "200", "digest": "2TV6YNTSXUMKAV55PIQVFDFWCEBTJCZ4", "length": "11146", "offset": "99489863", "filename": "crawl-data/CC-MAIN-2023-23/segments/1685224657735.85/warc/CC-MAIN-20230610164417-20230610194417-00125.warc.gz", "charset": "UTF-8", "languages": "eng"}
def filter_through_json_for_filenames() -> [str]:
    output_list = []
    for line in open('data/json_data/CC-MAIN-2023-23-index_forbes.json', 'r'):
        json_line = json.loads(line)
        if json_line['filename'] not in output_list:
            output_list.append(json_line['filename'])
    return output_list

# gets a list of filenames as input and downloads & extracts the files
def download_and_extract_files(ls_file_paths: [str]) -> None:
    limit = 0
    # ls_file_paths.reverse()
    for file_path in ls_file_paths:
        file_name = file_path[-51:]
        # if file_name != "CC-MAIN-20230609164851-20230609194851-00737.warc.gz":
        #     continue
        # path = wget.download(
        #     url="https://data.commoncrawl.org/" + file_path,
        #     out=f'data/zipped_data/{file_name}', bar=bar_custom)
        path = req_down(
            url="https://data.commoncrawl.org/" + file_path,
            out=f'data/zipped_data/{file_name}')
        print(path)

        with gzip.open(path, 'rb') as f:
            with open(f'data/unzipped_data/{file_name[:48]}', 'wb') as fi:
                for line in tqdm(f, desc="Extracting file"):
                    fi.write(line)

        limit = limit + 1
        if limit > 15:
            print("done!")
            return

def req_down(url, out):
    with open(out, 'wb') as f:
        response = requests.get(url)
        f.write(response.content)
        return out

if __name__ == '__main__':
    # list so we dont need to download the json file every time
    example_list = [
        'crawl-data/CC-MAIN-2023-23/segments/1685224652184.68/warc/CC-MAIN-20230605221713-20230606011713-00686.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224654031.92/warc/CC-MAIN-20230608003500-20230608033500-00227.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224644309.7/warc/CC-MAIN-20230528150639-20230528180639-00284.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224652184.68/warc/CC-MAIN-20230605221713-20230606011713-00223.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224657720.82/warc/CC-MAIN-20230610131939-20230610161939-00223.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224643462.13/warc/CC-MAIN-20230528015553-20230528045553-00569.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224648858.14/warc/CC-MAIN-20230602204755-20230602234755-00304.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224644915.48/warc/CC-MAIN-20230530000715-20230530030715-00711.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224648635.78/warc/CC-MAIN-20230602104352-20230602134352-00284.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224649302.35/warc/CC-MAIN-20230603165228-20230603195228-00235.warc.gz',
        'crawl-data/CC-MAIN-2023-23/segments/1685224649293.44/warc/CC-MAIN-20230603133129-20230603163129-00205.warc.gz']

    download_json_from_index_website()
    ls_filenames = filter_through_json_for_filenames()
    download_and_extract_files(ls_filenames)
    # f = warc.open("data/zipped_data/CC-MAIN-20230527223515-20230528013515-00005.warc.gz")
    # for record in f:
    #     print(record.payload.read())


# jobs의 길이는 1 이상 500 이하입니다.
# jobs의 각 행은 하나의 작업에 대한 [작업이 요청되는 시점, 작업의 소요시간] 입니다.
# 각 작업에 대해 작업이 요청되는 시간은 0 이상 1,000 이하입니다.
# 각 작업에 대해 작업의 소요시간은 1 이상 1,000 이하입니다.
# 하드디스크가 작업을 수행하고 있지 않을 때에는 먼저 요청이 들어온 작업부터 처리합니다.

# 각 작업에 대해 [작업이 요청되는 시점, 작업의 소요시간]을 담은 2차원 배열 jobs가 매개변수로 주어질 때,
# 작업의 요청부터 종료까지 걸린 시간의 평균을 가장 줄이는 방법으로 처리하면 평균이 얼마가 되는지 return 하도록 solution 함수를 작성해주세요. (단, 소수점 이하의 수는 버립니다)

def solution(jobs):
    answer = 0
    start = 0  # 현재까지 진행된 작업 시간
    length = len(jobs)

    jobs = sorted(jobs, key=lambda x: x[1])  # 소요시간 기준으로 오름차순 정렬

    while len(jobs) != 0:
        for i in range(len(jobs)):
            if jobs[i][0] <= start:
                start += jobs[i][1]
                answer += start - jobs[i][0]
                jobs.pop(i)
                break
            # 해당시점에 아직 작업이 들어오지 않았으면 시간 +1
            if i == len(jobs) - 1:
                start += 1

    return answer // length


import warnings
import numpy as np

from utils.misc import normalize_func, get_r_Z, gamme_supp_select, select, get_W, random_split
from intercept import get_intercept
from construct import construct_spllitmodelx_matrix
from W_statistics import compute_W_statistics
import matplotlib.pyplot as plt

class SplitModelXFilter:
    """
    Performs ModelX-based inference, from start to finish.

    Parameters
    """

    def __init__(
        self,
    ):
        """
        Initialize the class.
        """

        # self.q = q
        # self.method = method
        pass

    def forward(
        self,
        X,
        y,
        D,
        Sigma,
        Sigma_M,
        split_ratio,
        intercept_method,
        lambdas,
        q=0.10,
        methods=('knockoff'),
        W_types=('st'),
        normalize=True,
    ):
        """
        Runs the modelx filter; returns whether each feature was rejected.

        Parameters
        ----------
        X : np.ndarray
            ``(n, p)``-shaped design matrix
        y : np.ndarray
            ``(n,)``-shaped response vector
        D : np.ndarrays
            ``(m, p)``-shaped transformation matrix
        split_ratio: float
            The split ratio of the dataset
        fdr : float
            The desired level of false discovery rate control.
        fstat_kwargs : dict
            Extra kwargs to pass to the feature statistic ``fit`` function,
            excluding the required arguments.
        knockoff_kwargs : dict
            Extra kwargs for instantiating the knockoff sampler argument if
            the ksampler argument is a string identifier. This can be
            the empty dict for some identifiers such as "gaussian" or "fx",
            but additional keyword arguments are required for complex samplers
            such as the "metro" identifier. Defaults to {}
        """
        # normalize the data
        if normalize:
            X, y = normalize_func(X), normalize_func(y)
        n, m, p = X.shape[0], D.shape[0], X.shape[1]

        # Get a solved beta first which will used in the W statistics computation
        #TODO Fake split now
        # X1, y1, X2, y2 = random_split(X, y ,split_ratio=split_ratio)
        X1, y1 = X, y
        X2, y2 = X, y
        n1 = len(X1)
        betas, stats = get_intercept(X1, y1, D, nu, None,
                                            method=intercept_method, n2=n-n1)

        #TODO: Actually the nu is not used now
        nu = 1

        # Generate M and M_tilde
        alpha, M, M_tilde = construct_spllitmodelx_matrix(X, D, Sigma, Sigma_M, nu)

        # Compute the W statistics
        Ws = compute_W_statistics(X, y, D, betas, M, M_tilde, lambdas)

        # Compute the chosen S and results
        results = {}
        for lambda_val in lambdas:
            for W_type in W_types:
                for method in methods:
                    W = Ws[lambda_val]
                    S = select(W, q, method)
                    # print(f'W:{W.shape}, S:{S.shape}')
                    results[f'W_{W_type}_l{lambda_val}'] = W
                    results[f'S_{W_type}_{method}_l{lambda_val}'] = S
        return results


def go_left(x,y):
    x-=1
    if x<1:
        return (x+1),y
    else:
        return x,y
def go_right(x,y,size):
    x+=1
    if x>=size:
        return (x-1),y
    else:
        return x,y
def go_up(x,y):
    y-=1
    if y<1:
        return x,(y+1)
    else:
        return x,y
def go_down(x,y,size):
    y+=1
    if y>=size:
        return x,(y-1)
    else:
        return x,y

import sys, time
start = time.time()
size = int(sys.stdin.readline())
direction = map(str,sys.stdin.readline().split())
x,y = 1,1
for d in direction:
    if d=='L':
        x,y = go_left(x,y)
    elif d=='R':
        x,y = go_right(x,y,size)
    elif d=='D':
        x,y = go_down(x,y,size)
    else:
        x,y = go_up(x,y)
end = time.time()
print(y,x)
print('걸린 시간 : ',end-start)


def VerifyString(input):
    if not input:
        print(True)
    else:
        print(False)

""" Test cases """
#input = None # True (None is the equivalent of null in Python)
input = "a" # False
#input = "" # True
#input = "null" # False
#input = "None" # False

VerifyString(input)


# https://leetcode.com/problems/candy/description/?envType=daily-question&envId=2023-09-13
class Solution:
    def candy(self, ratings: List[int]) -> int:
        n = len(ratings)
        total = 0
        left = [1]
        right = [1]

        for i in range(1, n):  # Start from index 1 for left-to-right traversal
            if ratings[i] > ratings[i - 1]:
                left.insert(i, left[i - 1] + 1)
            else:
                left.insert(i, 1)

        for i in range(n - 2, -1, -1):  # Start from the second-to-last element for right-to-left traversal
            if ratings[i] > ratings[i + 1]:
                right.insert(0, right[0] + 1)
            else:
                right.insert(0, 1)

        for i in range(n):
            total += max(left[i], right[i])

        return total


"""第三章"""

from urllib.request import urlopen
from bs4 import BeautifulSoup
import time

url = "http://python.org"
html = urlopen(url)
time.sleep(5)
bsobj = BeautifulSoup(html, 'html.parser')
for link in bsobj.findAll('a'):
    if 'href' in link.attrs:
        print(link.attrs['href'])

python/python学习小分队打卡/算法/每周练习题/第九周/python网络数据采集第三四章.py

python网络数据采集第三四章.py