import os
import gzip
import time
import zipfile

from .. import files
from .util import filtered_container_list

# TODO: Should we support tarfiles?
class FileOpener(object):
    """Context manager that will open a (possibly compressed) filesystem file.

    FileOpener will close any open files when the context is exited.
    """
    def __init__(self, file_entry, zip_filter, match_type='first'):
        """Create a new FileOpener

        Arguments:
            file_entry (dict): The file entry as retrieved from the database
            zip_filter (regex): The regular expression to match zip entries, if applicable
            match_type (str): The optional match type if multiple zip entries are matched (default is first)
        """
        self.file_entry = file_entry
        self.zip_filter = zip_filter
        self.match_type = match_type

        # This is the opened file_entry file
        self._system_fd = None

        # This is the optional ZipFile instance
        self._zipfile = None

        # This is the optional set of zipfile entries
        self._zip_entries = []

        # This is the final _fd for reading
        self._fd = None

        # The final filename
        self._name = file_entry['name']

    def is_gzip(self):
        """Check if the file described by file_entry is a gzip file"""
        _, ext = os.path.splitext(self.file_entry['name'])
        return ext == '.gz'

    def __enter__(self):
        # Try to open the file
        open_mode = 'r'
        gz = self.is_gzip()

        if self.zip_filter or gz:
            open_mode = 'rb'
        try:
            # Open the file using file_system
            file_path, file_system = files.get_valid_file(self.file_entry)
            self._system_fd = file_system.open(file_path, open_mode)

            if self.zip_filter:
                # Open zipfile
                self._zipfile = zipfile.ZipFile(self._system_fd)

                # Find zip entry
                zip_entries = []
                for zipinf in self._zipfile.infolist():
                    zip_entries.append({
                        'path': zipinf.filename,
                        'timestamp': time.mktime(zipinf.date_time + (0, 0, -1))
                    })

                self._zip_entries = filtered_container_list(zip_entries, [('path', self.zip_filter)],
                        match_type=self.match_type, date_key='timestamp')

                if not self._zip_entries:
                    raise RuntimeError('Could not find matching zip entry in zip file: {}'.format(self.file_entry['name']))

            elif gz:
                # Open as gzip
                self._fd = gzip.GzipFile(fileobj=self._system_fd, mode='r')
                self._name, _ = os.path.splitext(self._name)
            else:
                # Read file directly
                self._fd = self._system_fd

            return self
        except Exception:
            self.close()
            raise

    def __exit__(self, exception_type, exception_value, traceback):
        self.close()

    def files(self):
        """Iterator for each of the files opened by this file opener"""
        if self._fd:
            yield self._name, self._fd
        elif self._zipfile:
            for entry in self._zip_entries:
                fd = self._zipfile.open(entry['path'], 'r')
                yield entry['path'], fd

    def close(self):
        """Close any open files"""
        # NOTE: This function should be re-entrant
        if self._zipfile:
            self._zipfile.close()
            self._zipfile = None

        if self._system_fd:
            self._system_fd.close()
            self._system_fd = None


import geojson
from tqdm import tqdm
import sqlite3
from pathlib import Path

db = Path.cwd().parent / 'data/main.db'

def get_routes():
    """
    Extracts the buses and routes from bus-routes.geojson and saves them
    to the db in the tables 'buses' and 'routes_points'.

    Args:
        None

    Returns:
        None
    """

    conn = sqlite3.connect(db)
    cursor = conn.cursor()

    stmt_delete = 'DELETE FROM buses;'
    cursor.execute(stmt_delete)
    stmt_delete = 'DELETE FROM routes_points;'
    cursor.execute(stmt_delete)

    with open('../data/bus-routes.geojson', 'r') as f:
        gj = geojson.load(f)

    buses = []

    for item in gj['features']:
        try:
            if item['properties']['network'] not in ['Movia', 'Takst Sjælland']:
                continue
            if item['properties']['route'] != 'bus':
                continue
            bus = {}
            bus['name'] = item['properties']['ref']
        except:
            continue
        try:
            bus['from'] = item['properties']['from']
        except:
            bus['from'] = None
        try:
            bus['to'] = item['properties']['to']
        except:
            bus['to'] = None
        bus['route'] = []

        if item['geometry']['type'] == 'LineString':
            segment = []
            for point in item['geometry']['coordinates']:
                segment.append([point[1], point[0]])
            bus['route'].append(segment)
        elif item['geometry']['type'] == 'MultiLineString':
            for subroute in item['geometry']['coordinates']:
                segment = []
                for point in subroute:
                    segment.append(point[::-1])
                bus['route'].append(segment)
        buses.append(bus)

    stmt_buses = """INSERT INTO buses (name, from_station_name, to_station_name)
            VALUES (?, ?, ?);"""

    stmt_routes = """INSERT INTO routes_points (bus_id, segment, coordinates, seq)
            VALUES (?, ?, ?, ?);"""

    for bus in tqdm(buses):
        data = (bus['name'], bus['from'], bus['to'])
        cursor.execute(stmt_buses, data)
        bus_id = cursor.lastrowid
        for segment_index, segment in enumerate(bus['route']):
            for point_index, point in enumerate(segment):
                data = (bus_id, segment_index+1, ','.join(map(str, point)), point_index+1)
                cursor.execute(stmt_routes, data)

    conn.commit()
    cursor.close()
    conn.close()

if __name__ == '__main__':
    get_routes()


import logging

import boto3

import config

def obj_last_modified(myobj):
    """
    Gets the last modified date of the file
    :param myobj:
    :return: Last Modified timestamp of file
    """
    return myobj['LastModified']

def fileWithLeastTimestamp():
    """
    Gets the file with the least timestamp
    :return: file with the least timestamp
    """
    s3client = boto3.client('s3')
    try:
        response = s3client.list_objects(Bucket=config.BUCKET_NAME, Prefix=config.PREFIX)
        print('resonpnse is: ', response)
    except Exception as e:
        logging.error(e)
        print(f'Unexpected error while listing the dir {config.BUCKET_NAME}:{config.PREFIX}')
    if ('Contents' in response):
        l = []
        for key in response['Contents']:
            l.append(key)
        sortedObjects = sorted(l, key=obj_last_modified, reverse=False)
        return sortedObjects[0]['Key']
    return None


from atom.api import Unicode, Int

from enaml.core.declarative import d_

from html_object import HTMLObject
from lxml.html import builder as E

class Slider(HTMLObject):

    tag = E.INPUT

    text = d_(Unicode())
    id = d_(Unicode())
    value = d_(Int())
    min = d_(Int())
    max = d_(Int())

    def initialize(self):
        super(Slider, self).initialize()

    def buildHTML(self, *args):
        self.addTags()
        self.addText(self.text)
        self.addAttributes(id = self.id, type = "range", value = str(self.value), min = str(self.min), max = str(self.max))
        return super(Slider, self).buildHTML(*args)


import os
import tempfile
from multiprocessing import Pool
from pathlib import Path

import librosa
import luigi
import pandas as pd
from tqdm import tqdm

from workflows.utils.gcs import single_file_target
from workflows.utils.mixin import DynamicRequiresMixin
from workflows.utils.rsync import GSUtilRsyncTask

def read_path(path: Path) -> dict:
    y, sr = librosa.load(path)
    duration = librosa.get_duration(y=y, sr=sr)
    return dict(filename="/".join(path.parts[-2:]), duration=duration)

class TrainDurations(luigi.Task, DynamicRequiresMixin):
    """Get the duration of each song in the training set into a parquet file."""

    birdclef_root_path = luigi.Parameter()
    output_path = luigi.Parameter()
    parallelism = luigi.IntParameter(default=os.cpu_count())
    capture_output = True

    def output(self):
        return single_file_target(self.output_path)

    def run(self):
        paths = sorted(Path(self.birdclef_root_path).glob("train_audio/**/*.ogg"))
        if not paths:
            raise ValueError("No audio files found", self.birdclef_root_path)
        with Pool(self.parallelism) as pool:
            res = list(tqdm(pool.imap(read_path, paths), total=len(paths)))

        df = pd.DataFrame(res)
        print(df.head())
        # ensure the output directory exists
        Path(self.output_path).parent.mkdir(parents=True, exist_ok=True)
        df.to_parquet(self.output_path)

class TrainDurationsWorkflow(luigi.Task):
    birdclef_root_path = luigi.Parameter()
    output_path = luigi.Parameter()
    local_data_root = luigi.OptionalParameter(default=None)
    parallelism = luigi.IntParameter(default=os.cpu_count())

    def output(self):
        return single_file_target(self.output_path)

    def run(self):
        # create a temporary path to download the data, make it persistent since
        # we're only wrapping the task here
        if self.local_data_root is None:
            self.local_data_root = tempfile.mkdtemp()

        tmp_path = Path(self.local_data_root)
        tmp_birdclef_root_path = tmp_path / "raw" / Path(self.birdclef_root_path).name
        tmp_birdclef_root_path.mkdir(parents=True, exist_ok=True)
        tmp_output_path = tmp_path / "processed" / "train_durations.parquet"
        tmp_output_path.parent.mkdir(parents=True, exist_ok=True)

        download_task = GSUtilRsyncTask(
            input_path=self.birdclef_root_path,
            output_path=tmp_birdclef_root_path.as_posix(),
            is_dir=True,
        )
        yield download_task

        train_durations = TrainDurations(
            birdclef_root_path=tmp_birdclef_root_path.as_posix(),
            output_path=tmp_output_path.as_posix(),
            parallelism=self.parallelism,
            dynamic_requires=[download_task],
        )
        yield train_durations

        upload_task = GSUtilRsyncTask(
            input_path=tmp_output_path.as_posix(),
            output_path=self.output_path,
            is_dir=False,
            dynamic_requires=[download_task, train_durations],
        )
        yield upload_task

if __name__ == "__main__":
    luigi.build(
        [
            TrainDurations(
                birdclef_root_path="data/raw/birdclef-2022",
                output_path="data/processed/birdclef-2022/train_durations.parquet",
            )
        ],
        log_level="INFO",
    )


import cv2
import subprocess as sp
import pandas as pd

cap = cv2.VideoCapture(0)

_WIDTH = 640
_HEIGHT = 480
_FPS = 30
_FFMPEG_BIN = 'ffmpeg'
file_path = '/home/malov/test_frame.jpg'

# while True:
ret, frame = cap.read()
if ret:
    cv2.imwrite('/home/malov/test_frame.jpg', frame)

    fourcc1 = cv2.VideoWriter_fourcc(*'MJPG')
    fourcc2 = cv2.VideoWriter_fourcc(*'H264')

    stream1 = cv2.VideoWriter('/home/malov/mjpeg_test_frame.jpg', fourcc1, _FPS, (_WIDTH, _HEIGHT))
    stream2 = cv2.VideoWriter('/home/malov/h264_test_frame.avi', fourcc2, _FPS, (_WIDTH, _HEIGHT))

    stream1.write(frame)
    stream2.write(frame)

    command = [_FFMPEG_BIN, '-i', file_path, '-']
    pipe = sp.Popen(command, stdout=sp.PIPE, stderr=sp.PIPE)
    pipe.terminate()
    infos = pipe.stderr.read().decode()

    # duration = int(infos[infos.find('Duration') + 19 : infos.find('Duration') + 21])
    # bitrate = int(infos[infos.find('bitrate') + 8 : infos.find('kb/s')])

print(infos, 'test')
cap.release()


"""
Sync Consent Files

Organize all consent files from PTSC source bucket into proper awardee buckets.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
import logging
import os
import pytz
import shutil
import tempfile
from typing import Collection, Dict, List, Type
from zipfile import ZipFile

from sqlalchemy import or_
from sqlalchemy.orm import Session

from rdr_service import config
from rdr_service.api_util import copy_cloud_file, download_cloud_file, get_blob, list_blobs, parse_date
from rdr_service.dao import database_factory
from rdr_service.dao.participant_dao import ParticipantDao, ParticipantHistoryDao
from rdr_service.dao.participant_summary_dao import ParticipantSummaryDao
from rdr_service.model.consent_file import ConsentFile, ConsentSyncStatus, ConsentType
from rdr_service.model.organization import Organization
from rdr_service.model.participant import Participant
from rdr_service.model.participant_summary import ParticipantSummary
from rdr_service.model.site import Site
from rdr_service.model.utils import to_client_participant_id
from rdr_service.participant_enums import QuestionnaireStatus

from rdr_service.resource.tasks import dispatch_rebuild_consent_metrics_tasks
from rdr_service.services.gcp_utils import gcp_cp
from rdr_service.storage import GoogleCloudStorageProvider, GoogleCloudStorageZipFile

SOURCE_BUCKET = {
    "vibrent": "ptc-uploads-all-of-us-rdr-prod",
    "careevolution": "ce-uploads-all-of-us-rdr-prod"
}
DEFAULT_ORG_NAME = 'no-org-assigned'
DEFAULT_GOOGLE_GROUP = "no-site-assigned"
TEMP_CONSENTS_PATH = os.path.join(tempfile.gettempdir(), "temp_consents")

@dataclass
class ParticipantPairingInfo:
    hpo_name: str
    org_name: str
    site_name: str

    def __hash__(self):
        return f'{self.hpo_name}::{self.org_name}::{self.site_name}'.__hash__()

@dataclass
class PairingHistoryRecord:
    org_name: str
    start_date: datetime

class BaseFileSync(ABC):
    def __init__(self, dest_bucket: str, storage_provider: GoogleCloudStorageProvider, root_destination_folder: str,
                 org_name: str, site_name: str):
        self.files_to_sync: List[ConsentFile] = []
        self._dest_bucket = dest_bucket
        self._storage_provider = storage_provider
        self._root_destination_folder = root_destination_folder
        self._org_name = org_name or DEFAULT_ORG_NAME
        self._site_name = site_name or DEFAULT_GOOGLE_GROUP

    def sync_file_list(self) -> Collection[ConsentFile]:
        for file in self.files_to_sync:
            self._sync_file(file)
            file.sync_time = datetime.utcnow()
            file.sync_status = ConsentSyncStatus.SYNC_COMPLETE

        return self.files_to_sync

    @abstractmethod
    def _sync_file(self, file: ConsentFile):
        ...

    def _get_common_upload_prefix(self):
        return f'{self._dest_bucket}/{self._root_destination_folder}/{self._org_name}'

    @classmethod
    def _get_file_basename(cls, full_path: str):
        return os.path.basename(full_path)

class CopyFilesSync(BaseFileSync):
    def _sync_file(self, file: ConsentFile):
        file_name = self._get_file_basename(file.file_path)
        destination_path = f'{self._get_common_upload_prefix()}/{self._site_name}/P{file.participant_id}/{file_name}'
        self._storage_provider.copy_blob(source_path=file.file_path, destination_path=destination_path)

class ZipFilesSync(BaseFileSync):
    def __init__(self, *args, **kwargs):
        super(ZipFilesSync, self).__init__(*args, **kwargs)
        self._zip_file_handle: GoogleCloudStorageZipFile = None

    def sync_file_list(self) -> Collection[ConsentFile]:
        upload_path = f'{self._get_common_upload_prefix()}/{self._site_name}.zip'

        with GoogleCloudStorageZipFile(upload_path) as zip_file_handle:
            self._zip_file_handle = zip_file_handle
            return super(ZipFilesSync, self).sync_file_list()

    def _sync_file(self, file: ConsentFile):
        participant_id_str = to_client_participant_id(file.participant_id)
        file_name = self._get_file_basename(file.file_path)
        self._zip_file_handle.write_blob(
            blob_source_str=file.file_path,
            archive_name=f'{participant_id_str}/{file_name}'
        )

class ConsentSyncGuesser:
    _SYNC_OVERLAP_DAYS_DELTA = timedelta(days=10)

    def __init__(self, session: Session, participant_history_dao: ParticipantHistoryDao):
        self._session = session
        self._participant_history_dao = participant_history_dao

    def check_consents(self, files: Collection[ConsentFile], session):
        # Get the latest pairing information for the participants
        participant_ids = [file.participant_id for file in files]
        raw_pairing_data = self._participant_history_dao.get_pairing_history(
            session=self._session,
            participant_ids=participant_ids
        )
        latest_participant_pairings = {}
        for history_record in raw_pairing_data:
            participant_id = history_record.participantId
            possible_latest_record = PairingHistoryRecord(
                org_name=history_record.externalId,
                start_date=history_record.lastModified
            )
            if participant_id not in latest_participant_pairings:
                latest_participant_pairings[participant_id] = possible_latest_record
            else:
                currently_stored_pairing = latest_participant_pairings[participant_id]
                if currently_stored_pairing.start_date <= possible_latest_record.start_date \
                        and currently_stored_pairing.org_name != possible_latest_record.org_name:
                    latest_participant_pairings[participant_id] = possible_latest_record

        participant_summaries = ParticipantSummaryDao.get_by_ids_with_session(self._session, participant_ids)
        for summary in participant_summaries:
            pairing_info = latest_participant_pairings[summary.participantId]
            latest_participant_pairings[summary.participantId] = (pairing_info, summary)

        modified_files_count = 0
        for file in files:
            pairing_info, summary = latest_participant_pairings[file.participant_id]
            sync_date = self.get_sync_date(file=file, summary=summary, latest_pairing_info=pairing_info)
            if sync_date is not None:
                file.sync_time = sync_date
                file.sync_status = ConsentSyncStatus.SYNC_COMPLETE

                modified_files_count += 1
                if modified_files_count % 500 == 0:
                    session.commit()

    @classmethod
    def get_sync_date(cls, file: ConsentFile, summary: ParticipantSummary, latest_pairing_info: PairingHistoryRecord):
        """
        This encapsulates the code for determining if a consent would have been copied using the previous process
        for syncing the files. It checks to see if a file would have been available for sync based on any of the times
        that a sync would have been triggered by the data available for a participant.
        If any of them would have triggered a sync to the latest org's bucket, then the date is returned for the sync.
        Otherwise None is returned.
        """
        if not cls._org_date_valid_for_sync(
            upload_time=file.file_upload_time,
            latest_pair_time=latest_pairing_info.start_date
        ):
            return None

        primary_time = summary.consentForStudyEnrollmentTime
        if cls._in_sync_time_window(
            upload_time=file.file_upload_time,
            consent_time=primary_time
        ):
            return cls._determine_sync_month(primary_time)

        ehr_time = summary.consentForElectronicHealthRecordsTime
        if cls._in_sync_time_window(
            upload_time=file.file_upload_time,
            consent_time=ehr_time
        ):
            return cls._determine_sync_month(ehr_time)

        # GROR wouldn't have triggered it until the June 1st sync
        gror_time = summary.consentForGenomicsRORTime
        if (
            gror_time is not None
            and gror_time > datetime(2021, 4, 21)  # The June 1st sync would have looked for dates back to April 21st
            and cls._in_sync_time_window(upload_time=file.file_upload_time, consent_time=gror_time)
        ):
            return cls._determine_sync_month(gror_time)

        return None

    @classmethod
    def _in_sync_time_window(cls, consent_time: datetime, upload_time: datetime):
        if not consent_time:
            return False

        window_start_date = datetime(consent_time.year, consent_time.month, 1) - cls._SYNC_OVERLAP_DAYS_DELTA
        window_end_date = datetime(consent_time.year, consent_time.month, 1) + relativedelta(months=1)
        return window_start_date <= upload_time <= window_end_date

    @classmethod
    def _determine_sync_month(cls, timestamp: datetime):
        return datetime(timestamp.year, timestamp.month, 1) + relativedelta(months=1)

    @classmethod
    def _org_date_valid_for_sync(cls, upload_time: datetime, latest_pair_time: datetime):
        pair_window_start = datetime(latest_pair_time.year, latest_pair_time.month, 1) - cls._SYNC_OVERLAP_DAYS_DELTA
        return pair_window_start < upload_time

class ConsentSyncController:
    def __init__(self, consent_dao, participant_dao: ParticipantDao, storage_provider: GoogleCloudStorageProvider):
        self.consent_dao = consent_dao
        self.participant_dao = participant_dao
        self.storage_provider = storage_provider
        self._destination_folder = config.getSettingJson('consent_destination_prefix', default='Participant')

    def _build_sync_handler(self, zip_files: bool, bucket: str, pairing_info: ParticipantPairingInfo):
        sync_class: Type[BaseFileSync] = ZipFilesSync if zip_files else CopyFilesSync

        return sync_class(
            dest_bucket=bucket,
            storage_provider=self.storage_provider,
            root_destination_folder=self._destination_folder,
            org_name=pairing_info.org_name,
            site_name=pairing_info.site_name
        )

    def sync_ready_files(self, sync_config=None):
        """Syncs any validated consent files that are ready for syncing"""

        if sync_config is None:
            sync_config = config.getSettingJson(config.CONSENT_SYNC_BUCKETS)

        hpos_sync_config = sync_config['hpos']
        orgs_sync_config = sync_config['orgs']

        filters = {
            hpo_name: {
                'exclude_types': [
                    ConsentType(excluded_type_str) for excluded_type_str in options['exclude_types']
                ]
            }
            for hpo_name, options in hpos_sync_config.items()
            if 'exclude_types' in options
        }
        file_list: List[ConsentFile] = self.consent_dao.get_files_ready_to_sync(
            hpo_names=hpos_sync_config.keys(),
            org_names=orgs_sync_config.keys(),
            additional_filters=filters
        )

        pairing_info_map = self._build_participant_pairing_map(file_list)

        # Build out a FileSync for each possible PairingInfo
        sync_pairing_map: Dict[ParticipantPairingInfo, BaseFileSync] = {}
        for pairing_info in pairing_info_map.values():
            if pairing_info not in sync_pairing_map:
                org_config = orgs_sync_config.get(pairing_info.org_name)
                if org_config:
                    config_data = org_config
                else:
                    config_data = hpos_sync_config.get(pairing_info.hpo_name)

                if not config_data:
                    # No need to build sync handlers for anything not in the config
                    continue

                sync_pairing_map[pairing_info] = self._build_sync_handler(
                    zip_files=config_data['zip_consents'],
                    bucket=config_data['bucket'],
                    pairing_info=pairing_info
                )

        for file in file_list:
            pairing_info = pairing_info_map.get(file.participant_id, None)
            if not pairing_info:
                # Skip files for unpaired participants
                continue

            # Retrieve the sync handler based on the pairing information
            file_group = sync_pairing_map.get(pairing_info)
            if not file_group:
                # Ignore participants paired to an org or hpo we aren't syncing files for
                continue

            file_group.files_to_sync.append(file)

        with self.consent_dao.session() as session:
            for file_group in sync_pairing_map.values():
                files_synced = file_group.sync_file_list()

                # Update the database after each group syncs so ones
                # that have succeeded so far get saved if a later one fails
                if len(files_synced):
                    self.consent_dao.batch_update_consent_files(session=session, consent_files=files_synced)
                    session.commit()

                    # Queue tasks to rebuild consent metrics resource data records (for PDR)
                    dispatch_rebuild_consent_metrics_tasks([file.id for file in files_synced])

    def _build_participant_pairing_map(self, files: List[ConsentFile]) -> Dict[int, ParticipantPairingInfo]:
        """
        Returns a dictionary mapping each participant's id to the external id for their organization
        and the google group name for their site
        """
        participant_ids = {file.participant_id for file in files}
        participant_pairing_data = self.participant_dao.get_pairing_data_for_ids(participant_ids)
        return {
            participant_id: ParticipantPairingInfo(hpo_name=hpo_name, org_name=org_name, site_name=site_name)
            for participant_id, hpo_name, org_name, site_name in participant_pairing_data
        }

def get_consent_destination(zipping=False, add_protocol=False, **kwargs):
    if zipping:
        destination_pattern = TEMP_CONSENTS_PATH + '/{bucket_name}/{org_external_id}/{site_name}/P{p_id}/'
    else:
        prefix = 'gs://' if add_protocol else ''
        destination_pattern = prefix + '{bucket_name}/Participant/{org_external_id}/{site_name}/P{p_id}/'

    return destination_pattern.format(**kwargs)

def _directories_in(directory_path):
    with os.scandir(directory_path) as objects:
        return [directory_object for directory_object in objects if directory_object.is_dir()]

def _add_path_to_zip(zip_file, directory_path):
    for current_path, _, files in os.walk(directory_path):
        # os.walk will recurse into sub_directories, so we only need to handle the files in the current directory
        for file in files:
            file_path = os.path.join(current_path, file)
            archive_name = file_path[len(directory_path):]
            zip_file.write(file_path, arcname=archive_name)

def _format_debug_out(p_id, src, dest):
    logging.debug(" Participant: {0}".format(p_id))
    logging.debug("    src: {0}".format(src))
    logging.debug("   dest: {0}".format(dest))

def copy_file(source, destination, participant_id, dry_run=True, zip_files=False):
    if not dry_run:
        if zip_files:
            # gcp_cp doesn't create local directories when they don't exist
            os.makedirs(destination, exist_ok=True)

        copy_args = {'flags': '-m'}
        if not zip_files:
            copy_args['args'] = '-r'
        gcp_cp(source, destination, **copy_args)

    _format_debug_out(participant_id, source, destination)

def archive_and_upload_consents(dry_run=True):
    logging.info("zipping and uploading consent files...")
    storage_provider = GoogleCloudStorageProvider()
    for bucket_dir in _directories_in(TEMP_CONSENTS_PATH):
        for org_dir in _directories_in(bucket_dir):
            bucket = bucket_dir.name
            for site_dir in _directories_in(org_dir):
                zip_file_name = os.path.join(org_dir.path, site_dir.name + '.zip')
                with ZipFile(zip_file_name, 'w') as zip_file:
                    _add_path_to_zip(zip_file, site_dir.path)

                _, file_name = os.path.split(zip_file_name)
                destination = "{bucket_name}/Participant/{org_external_id}/{file_name}".format(
                    bucket_name=bucket,
                    org_external_id=org_dir.name,
                    file_name=file_name
                )
                if not dry_run:
                    logging.debug("Uploading file '{zip_file}' to '{destination}'".format(
                        zip_file=zip_file_name,
                        destination=destination
                    ))
                    storage_provider.upload_from_file(zip_file_name, destination)

    shutil.rmtree(TEMP_CONSENTS_PATH)

def do_sync_recent_consent_files(all_va=False, zip_files=False):
    # Sync everything from the start of the previous month
    start_date = datetime.now().replace(day=1) - timedelta(days=10)
    do_sync_consent_files(start_date=start_date.strftime('%Y-%m-01'), zip_files=zip_files, all_va=all_va)

def do_sync_consent_files(zip_files=False, **kwargs):
    """
    entrypoint
    """
    logging.info('Syncing consent files.')
    org_buckets = get_org_data_map()
    org_ids = [org_id for org_id, org_data in org_buckets.items()]
    start_date = kwargs.get('start_date')
    end_date = kwargs.get('end_date')
    all_va = kwargs.get('all_va')
    if start_date or end_date:
        logging.info(f'syncing consents from {start_date} to {end_date}')
    if zip_files:
        logging.info('zipping consent files')
    if all_va:
        logging.info('only interacting with VA files')
    file_filter = kwargs.get('file_filter', 'pdf')
    for participant_id, origin, site_google_group, org_external_id in _iter_participants_data(org_ids, **kwargs):
        logging.info(f'Syncing files for {participant_id}')
        source_bucket = SOURCE_BUCKET.get(origin, SOURCE_BUCKET[next(iter(SOURCE_BUCKET))])
        source = "/{source_bucket}/Participant/P{participant_id}/"\
            .format(source_bucket=source_bucket,
                    participant_id=participant_id)
        if all_va:
            destination_bucket = 'aou179'
        else:
            destination_bucket = org_buckets[org_external_id]
        destination = get_consent_destination(zip_files,
                                              bucket_name=destination_bucket,
                                              org_external_id=org_external_id,
                                              site_name=site_google_group or DEFAULT_GOOGLE_GROUP,
                                              p_id=participant_id)

        cloudstorage_copy_objects_task(source, destination, start_date=start_date,
                                       file_filter=file_filter, zip_files=zip_files)

    if zip_files:
        archive_and_upload_consents(dry_run=False)

def get_org_data_map():
    return config.getSettingJson(config.CONSENT_SYNC_BUCKETS)

def build_participant_query(session, org_ids, start_date=None, end_date=None, all_va=False, ids: list = None):
    participant_query = session.query(
        Participant.participantId,
        Participant.participantOrigin,
        Site.googleGroup,
        Organization.externalId
    ).outerjoin(
        Organization, Organization.organizationId == Participant.organizationId
    ).outerjoin(
        Site, Site.siteId == Participant.siteId
    ).outerjoin(
        ParticipantSummary, Participant.participantSummary
    ).filter(
        Participant.isGhostId.isnot(True),
        Participant.isTestParticipant.isnot(True),
        ParticipantSummary.consentForStudyEnrollment == int(QuestionnaireStatus.SUBMITTED),
        or_(
            ParticipantSummary.email.is_(None),
            ParticipantSummary.email.notlike('%@example.com')
        )
    )

    if start_date and end_date:
        participant_query = participant_query.filter(
            or_(
                ParticipantSummary.consentForStudyEnrollmentTime.between(start_date, end_date),
                ParticipantSummary.consentForElectronicHealthRecordsTime.between(start_date, end_date),
                ParticipantSummary.consentForGenomicsRORTime.between(start_date, end_date)
            )
        )
    elif start_date:
        participant_query = participant_query.filter(
            or_(
                ParticipantSummary.consentForStudyEnrollmentTime > start_date,
                ParticipantSummary.consentForElectronicHealthRecordsTime > start_date,
                ParticipantSummary.consentForGenomicsRORTime > start_date
            )
        )
    elif end_date:
        participant_query = participant_query.filter(
            or_(
                ParticipantSummary.consentForStudyEnrollmentTime < end_date,
                ParticipantSummary.consentForElectronicHealthRecordsTime < end_date,
                ParticipantSummary.consentForGenomicsRORTime < end_date
            )
        )

    if all_va:
        participant_query = participant_query.filter(Organization.externalId.like('VA_%'))
    else:
        participant_query = participant_query.filter(Organization.externalId.in_(org_ids))

    if ids:
        participant_query = participant_query.filter(Participant.participantId.in_(ids))

    return participant_query

def _iter_participants_data(org_ids, **kwargs):
    with database_factory.make_server_cursor_database().session() as session:
        participant_query = build_participant_query(session, org_ids, **kwargs)
        for participant_data in participant_query:
            yield participant_data

def _download_file(source, destination):
    os.makedirs(os.path.dirname(destination), exist_ok=True)
    download_cloud_file(source, destination)

def cloudstorage_copy_objects_task(source, destination, start_date: str = None,
                                   file_filter=None, zip_files=False):
    """
    Cloud Task: Copies all objects matching the source to the destination.
    Both source and destination use the following format: /bucket/prefix/
    """
    if start_date:
        timezone = pytz.timezone('Etc/Greenwich')
        start_date = timezone.localize(parse_date(start_date))

    path = source if source[0:1] != '/' else source[1:]
    bucket_name, _, prefix = path.partition('/')
    prefix = None if prefix == '' else prefix
    files_found = False
    for source_blob in list_blobs(bucket_name, prefix):
        if not source_blob.name.endswith('/'):  # Exclude folders
            source_file_path = os.path.normpath('/' + bucket_name + '/' + source_blob.name)
            destination_file_path = destination + source_file_path[len(source):]
            if (zip_files or _not_previously_copied(source_file_path, destination_file_path)) and\
                    _meets_date_requirements(source_blob, start_date) and\
                    _matches_file_filter(source_blob.name, file_filter):
                files_found = True
                move_file_function = _download_file if zip_files else copy_cloud_file
                move_file_function(source_file_path, destination_file_path)
    if not files_found:
        logging.warning(f'No files copied from {source}')

def _not_previously_copied(source_file_path, destination_file_path):
    destination_file_path = destination_file_path if destination_file_path[0:1] != '/' else destination_file_path[1:]
    destination_bucket_name, _, destination_blob_name = destination_file_path.partition('/')
    destination_blob = get_blob(destination_bucket_name, destination_blob_name)
    if destination_blob is None:
        return True

    source_file_path = source_file_path if source_file_path[0:1] != '/' else source_file_path[1:]
    source_bucket_name, _, source_blob_name = source_file_path.partition('/')
    source_blob = get_blob(source_bucket_name, source_blob_name)
    return source_blob.etag != destination_blob.etag

def _meets_date_requirements(source_blob, start_date):
    # Not filtering files by end_date in case we need to transfer files that have been re-uploaded
    return start_date is None or source_blob.updated >= start_date

def _matches_file_filter(source_blob_name, file_filter):
    return file_filter is None or source_blob_name.endswith(file_filter)


# Сортировка Quick Sort.
# Одна из лучших сортировок. Делается через рекурсию
# Ключевые слова: лево, право, центр
import random

array = [i for i in range(1, 30)]
random.shuffle(array)
array = array[:10]
print(array)

def quick_sort(array):
    if len(array) <= 1:
        return array
    element = array[0]

    left = list(filter(lambda i: i < element, array))
    center = [i for i in array if i == element]
    right = list(filter(lambda i: i > element, array))

    return quick_sort(left) + center + quick_sort(right)

print(quick_sort(array))


# -*- coding: utf-8 -*-

import mock
import pytest
import unittest
from json import dumps

from addons.base.tests.models import (OAuthAddonNodeSettingsTestSuiteMixin,
                                      OAuthAddonUserSettingTestSuiteMixin)
from addons.github.models import NodeSettings
from addons.github.tests import factories
from osf_tests.factories import ProjectFactory, UserFactory, DraftRegistrationFactory

from nose.tools import (assert_equal, assert_false, assert_in, assert_is,
                        assert_not_equal, assert_not_in, assert_true)

from github3 import GitHubError
from github3.repos import Repository

from tests.base import OsfTestCase, get_default_metaschema

from framework.auth import Auth
from addons.base import exceptions
from addons.github.exceptions import NotFoundError

from .utils import create_mock_github
mock_github = create_mock_github()

pytestmark = pytest.mark.django_db

class TestNodeSettings(OAuthAddonNodeSettingsTestSuiteMixin, unittest.TestCase):

    short_name = 'github'
    full_name = 'GitHub'
    ExternalAccountFactory = factories.GitHubAccountFactory

    NodeSettingsFactory = factories.GitHubNodeSettingsFactory
    NodeSettingsClass = NodeSettings
    UserSettingsFactory = factories.GitHubUserSettingsFactory

    ## Mixin Overrides ##

    def _node_settings_class_kwargs(self, node, user_settings):
        return {
            'user_settings': self.user_settings,
            'repo': 'mock',
            'user': 'abc',
            'owner': self.node
        }

    def test_set_folder(self):
        # GitHub doesn't use folderpicker, and the nodesettings model
        # does not need a `set_repo` method
        pass

    def test_serialize_settings(self):
        # GitHub's serialized_settings are a little different from
        # common storage addons.
        settings = self.node_settings.serialize_waterbutler_settings()
        expected = {'owner': self.node_settings.user, 'repo': self.node_settings.repo}
        assert_equal(settings, expected)

    @mock.patch(
        'addons.github.models.UserSettings.revoke_remote_oauth_access',
        mock.PropertyMock()
    )
    def test_complete_has_auth_not_verified(self):
        super(TestNodeSettings, self).test_complete_has_auth_not_verified()

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_to_json(self, mock_repos, mock_check_authorization):
        mock_repos.return_value = {}
        super(TestNodeSettings, self).test_to_json()

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_to_json_user_is_owner(self, mock_check_authorization, mock_repos):
        mock_check_authorization.return_value = True
        mock_repos.return_value = {}
        result = self.node_settings.to_json(self.user)
        assert_true(result['user_has_auth'])
        assert_equal(result['github_user'], 'abc')
        assert_true(result['is_owner'])
        assert_true(result['valid_credentials'])
        assert_equal(result.get('repo_names', None), [])

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_to_json_user_is_not_owner(self, mock_check_authorization, mock_repos):
        mock_check_authorization.return_value = True
        mock_repos.return_value = {}
        not_owner = UserFactory()
        result = self.node_settings.to_json(not_owner)
        assert_false(result['user_has_auth'])
        assert_equal(result['github_user'], 'abc')
        assert_false(result['is_owner'])
        assert_true(result['valid_credentials'])
        assert_equal(result.get('repo_names', None), None)

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_get_folders(self, mock_check_authorization, mock_repos):
        mock_repos.return_value = [Repository.from_json(dumps({'name': 'test',
                                                         'id': '12345',
                                                         'owner':
                                                             {'login': 'test name'}
                                                         }))
                                   ]
        result = self.node_settings.get_folders()

        assert_equal(len(result), 1)
        assert_equal(result[0]['id'], '12345')
        assert_equal(result[0]['name'], 'test')
        assert_equal(result[0]['path'], 'test name/test')
        assert_equal(result[0]['kind'], 'repo')

    @mock.patch('addons.github.api.GitHubClient.repos')
    @mock.patch('addons.github.api.GitHubClient.check_authorization')
    def test_get_folders_not_have_auth(self, mock_repos, mock_check_authorization):
        mock_repos.return_value = [Repository.from_json(dumps({'name': 'test',
                                                         'id': '12345',
                                                         'owner':
                                                             {'login': 'test name'}
                                                         }))
                                   ]
        self.node_settings.user_settings = None
        with pytest.raises(exceptions.InvalidAuthError):
            self.node_settings.get_folders()

class TestUserSettings(OAuthAddonUserSettingTestSuiteMixin, unittest.TestCase):

    short_name = 'github'
    full_name = 'GitHub'
    ExternalAccountFactory = factories.GitHubAccountFactory

    def test_public_id(self):
        assert_equal(self.user.external_accounts.first().display_name, self.user_settings.public_id)

class TestCallbacks(OsfTestCase):

    def setUp(self):

        super(TestCallbacks, self).setUp()

        self.project = ProjectFactory()
        self.consolidated_auth = Auth(self.project.creator)
        self.project.creator.save()
        self.non_authenticator = UserFactory()
        self.non_authenticator.save()
        self.project.save()
        self.project.add_contributor(
            contributor=self.non_authenticator,
            auth=self.consolidated_auth,
        )

        self.project.add_addon('github', auth=self.consolidated_auth)
        self.project.creator.add_addon('github')
        self.external_account = factories.GitHubAccountFactory()
        self.project.creator.external_accounts.add(self.external_account)
        self.project.creator.save()
        self.node_settings = self.project.get_addon('github')
        self.user_settings = self.project.creator.get_addon('github')
        self.node_settings.user_settings = self.user_settings
        self.node_settings.user = 'Queen'
        self.node_settings.repo = 'Sheer-Heart-Attack'
        self.node_settings.external_account = self.external_account
        self.node_settings.save()
        self.node_settings.set_auth
        self.user_settings.oauth_grants[self.project._id] = {self.external_account._id: []}
        self.user_settings.save()

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_make_public(self, mock_repo):
        mock_repo.side_effect = NotFoundError

        result = self.node_settings.before_make_public(self.project)
        assert_is(result, None)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_public_gh_public(self, mock_repo):
        self.project.is_public = True
        self.project.save()
        mock_repo.return_value = Repository.from_json(dumps({'private': False}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_false(message)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_public_gh_private(self, mock_repo):
        self.project.is_public = True
        self.project.save()
        mock_repo.return_value = Repository.from_json(dumps({'private': True}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_true(message)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_private_gh_public(self, mock_repo):
        mock_repo.return_value = Repository.from_json(dumps({'private': False}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_true(message)

    @mock.patch('addons.github.api.GitHubClient.repo')
    def test_before_page_load_osf_private_gh_private(self, mock_repo):
        mock_repo.return_value = Repository.from_json(dumps({'private': True}))
        message = self.node_settings.before_page_load(self.project, self.project.creator)
        mock_repo.assert_called_with(
            self.node_settings.user,
            self.node_settings.repo,
        )
        assert_false(message)

    def test_before_page_load_not_contributor(self):
        message = self.node_settings.before_page_load(self.project, UserFactory())
        assert_false(message)

    def test_before_page_load_not_logged_in(self):
        message = self.node_settings.before_page_load(self.project, None)
        assert_false(message)

    def test_before_remove_contributor_authenticator(self):
        message = self.node_settings.before_remove_contributor(
            self.project, self.project.creator
        )
        assert_true(message)

    def test_before_remove_contributor_not_authenticator(self):
        message = self.node_settings.before_remove_contributor(
            self.project, self.non_authenticator
        )
        assert_false(message)

    def test_after_remove_contributor_authenticator_self(self):
        message = self.node_settings.after_remove_contributor(
            self.project, self.project.creator, self.consolidated_auth
        )
        assert_equal(
            self.node_settings.user_settings,
            None
        )
        assert_true(message)
        assert_not_in('You can re-authenticate', message)

    def test_after_remove_contributor_authenticator_not_self(self):
        auth = Auth(user=self.non_authenticator)
        message = self.node_settings.after_remove_contributor(
            self.project, self.project.creator, auth
        )
        assert_equal(
            self.node_settings.user_settings,
            None
        )
        assert_true(message)
        assert_in('You can re-authenticate', message)

    def test_after_remove_contributor_not_authenticator(self):
        self.node_settings.after_remove_contributor(
            self.project, self.non_authenticator, self.consolidated_auth
        )
        assert_not_equal(
            self.node_settings.user_settings,
            None,
        )

    def test_after_fork_authenticator(self):
        fork = ProjectFactory()
        clone = self.node_settings.after_fork(
            self.project, fork, self.project.creator,
        )
        assert_equal(
            self.node_settings.user_settings,
            clone.user_settings,
        )

    def test_after_fork_not_authenticator(self):
        fork = ProjectFactory()
        clone = self.node_settings.after_fork(
            self.project, fork, self.non_authenticator,
        )
        assert_equal(
            clone.user_settings,
            None,
        )

    def test_after_delete(self):
        self.project.remove_node(Auth(user=self.project.creator))
        # Ensure that changes to node settings have been saved
        self.node_settings.reload()
        assert_true(self.node_settings.user_settings is None)

    @mock.patch('website.archiver.tasks.archive')
    def test_does_not_get_copied_to_registrations(self, mock_archive):
        registration = self.project.register_node(
            schema=get_default_metaschema(),
            auth=Auth(user=self.project.creator),
            draft_registration=DraftRegistrationFactory(branched_from=self.project),
        )
        assert_false(registration.has_addon('github'))

class TestGithubNodeSettings(unittest.TestCase):

    def setUp(self):
        super(TestGithubNodeSettings, self).setUp()
        self.user = UserFactory()
        self.user.add_addon('github')
        self.user_settings = self.user.get_addon('github')
        self.external_account = factories.GitHubAccountFactory()
        self.user_settings.owner.external_accounts.add(self.external_account)
        self.user_settings.owner.save()
        self.node_settings = factories.GitHubNodeSettingsFactory(user_settings=self.user_settings)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook(self, mock_delete_hook):
        self.node_settings.hook_id = 'hook'
        self.node_settings.save()
        args = (
            self.node_settings.user,
            self.node_settings.repo,
            self.node_settings.hook_id,
        )
        res = self.node_settings.delete_hook()
        assert_true(res)
        mock_delete_hook.assert_called_with(*args)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook_no_hook(self, mock_delete_hook):
        res = self.node_settings.delete_hook()
        assert_false(res)
        assert_false(mock_delete_hook.called)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook_not_found(self, mock_delete_hook):
        self.node_settings.hook_id = 'hook'
        self.node_settings.save()
        mock_delete_hook.side_effect = NotFoundError
        args = (
            self.node_settings.user,
            self.node_settings.repo,
            self.node_settings.hook_id,
        )
        res = self.node_settings.delete_hook()
        assert_false(res)
        mock_delete_hook.assert_called_with(*args)

    @mock.patch('addons.github.api.GitHubClient.delete_hook')
    def test_delete_hook_error(self, mock_delete_hook):
        self.node_settings.hook_id = 'hook'
        self.node_settings.save()
        mock_delete_hook.side_effect = GitHubError(mock.Mock())
        args = (
            self.node_settings.user,
            self.node_settings.repo,
            self.node_settings.hook_id,
        )
        res = self.node_settings.delete_hook()
        assert_false(res)
        mock_delete_hook.assert_called_with(*args)


import json
import time
import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Polygon
import numpy as np
import copy
import itertools
# from . import mask as maskUtils
import os
from collections import defaultdict
import sys
PYTHON_VERSION = sys.version_info[0]
if PYTHON_VERSION == 2:
    from urllib import urlretrieve
elif PYTHON_VERSION == 3:
    from urllib.request import urlretrieve

def _isArrayLike(obj):
    return hasattr(obj, '__iter__') and hasattr(obj, '__len__')

class adapter:
    def __init__(self, txtfile=None):
        """
        Constructor of Microsoft COCO helper class for reading and visualizing annotations.
        :param txtfile (str): location of annotation file
        :return:
        """
        # load dataset
        self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict()
        self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list)

        # anns :
        #   annid -- imgid + idx int
        #       imgid
        #       bbox
        #       levelid
        #       annid
        #       iscrowd 0

        # cats :
        #   id
        #   type

        # imgs :
        #   file_path
        #   imgid   int

        if not txtfile == None:
            print('loading annotations into memory...')
            tic = time.time()
            nms = []
            with open(txtfile, 'r') as f:
                line = f.readline().strip('\n')
                # print("Hello2")
                while line:
                    nms.append(line)
                    line = f.readline().strip('\n')
            # print(nms)

            self.createIndex(nms)
            # dataset = json.load(open(txtfile, 'r'))
            # assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset))
            # print('Done (t={:0.2f}s)'.format(time.time()- tic))
            # self.dataset = dataset
            # self.createIndex()
    def createIndex(self, files):
        print('creating index...')
        dataset, anns, cats, imgs = {}, {}, {}, {}
        imgToAnns,catToImgs = defaultdict(list),defaultdict(list)
        cnt = 0
        for file in files:
            contents = json.load(open("./annotations/" + file, 'r'))
            if cnt == 0:
                # print(contents)
                cnt = cnt + 1
            imgid = int(file.split('/')[1].split('.')[0])
            # print(imgid)

            ## imgs
            img = {}
            img['id'] = int(imgid)
            img['file_path'] = contents['raw_file_path']
            imgs[img['id']] = img

            for idx, item in enumerate(contents['results']):

                ## anns
                annid = int(str(imgid)+ str(idx))
                bbox = [item['x'],item['y'],item['width'],item['height']]
                levelid = item['id']
                cartype = item['type']

                ann = {}
                ann['id'] = annid
                ann['bbox'] = bbox
                ann['levelid'] = levelid
                ann['cartype'] = cartype
                ann['image_id'] = imgid
                ann['iscrowd'] = 0
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                if not 'area' in ann:
                    ann['area'] = bb[2]*bb[3]

                imgToAnns[ann['image_id']].append(ann)
                anns[ann['id']] = ann
        dataset['images'] = list(imgs.values())
        dataset['annotations'] = list(anns.values())

        print('index created!')
        # create class members
        self.dataset = dataset
        self.anns = anns
        self.imgToAnns = imgToAnns
        self.catToImgs = catToImgs
        self.imgs = imgs
        self.cats = cats

    def createIndexforResults(self):
        # create index
        print('creating index...')
        anns, cats, imgs = {}, {}, {}
        imgToAnns,catToImgs = defaultdict(list),defaultdict(list)
        if 'annotations' in self.dataset:
            for ann in self.dataset['annotations']:
                imgToAnns[ann['image_id']].append(ann)
                anns[ann['id']] = ann

        if 'images' in self.dataset:
            for img in self.dataset['images']:
                imgs[img['id']] = img

        if 'categories' in self.dataset:
            for cat in self.dataset['categories']:
                cats[cat['id']] = cat

        if 'annotations' in self.dataset and 'categories' in self.dataset:
            for ann in self.dataset['annotations']:
                catToImgs[ann['category_id']].append(ann['image_id'])

        print('index created!')

        # create class members
        self.anns = anns
        self.imgToAnns = imgToAnns
        self.catToImgs = catToImgs
        self.imgs = imgs
        self.cats = cats

    def loadRes(self, resFile):
        """
        Load result file and return a result api object.
        :param   resFile (str)     : file name of result file
        :return: res (obj)         : result api object
        """
        res = adapter()
        res.dataset['images'] = [img for img in self.dataset['images']]

        print('Loading and preparing results...')
        tic = time.time()
        if type(resFile) == str or (PYTHON_VERSION == 2 and type(resFile) == unicode):
            nms = []
            with open(resFile, 'r') as f:
                line = f.readline().strip('\n')
                # print("Hello2")
                while line:
                    nms.append(line)
                    line = f.readline().strip('\n')
        dataset, anns, cats, imgs = {}, {}, {}, {}
        cnt = 0
        for file in nms:
            contents = json.load(open("./results/" + file, 'r'))
            if cnt == 0:
                # print(contents)
                cnt = cnt + 1
            imgid = int(file.split('/')[1].split('.')[0])
            # print(imgid)

            for idx, item in enumerate(contents['results']):

                ## anns
                annid = int(str(imgid) + str(idx))
                bbox = [item['x'],item['y'],item['width'],item['height']]
                levelid = item['id']
                cartype = item['type']

                ann = {}
                ann['id'] = annid
                ann['bbox'] = bbox
                ann['levelid'] = levelid
                ann['cartype'] = cartype
                ann['image_id'] = imgid
                ann['iscrowd'] = 0
                ann['score'] = item['score']
                anns[ann['id']] = ann

        anns = list(anns.values())
        assert type(anns) == list, 'results in not an array of objects'
        # print(anns)
        annsImgIds = [ann['image_id'] for ann in anns]
        # print(len(annsImgIds))
        assert set(annsImgIds) == (set(annsImgIds) & set(list(self.imgs.keys()))), \
               'Results do not correspond to current coco set'
        if 'bbox' in anns[0] and not anns[0]['bbox'] == []:
            # res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                ann['area'] = bb[2]*bb[3]
                ann['id'] = id+1
                ann['iscrowd'] = 0
        # print("BERESF", anns[0])

        print('DONE (t={:0.2f}s)'.format(time.time()- tic))

        res.dataset['annotations'] = anns
        res.createIndexforResults()
        return res

        return 0
        assert type(anns) == list, 'results in not an array of objects'
        annsImgIds = [ann['image_id'] for ann in anns]
        assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \
               'Results do not correspond to current coco set'
        if 'caption' in anns[0]:
            imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns])
            res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds]
            for id, ann in enumerate(anns):
                ann['id'] = id+1
        if 'bbox' in anns[0] and not anns[0]['bbox'] == []:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                bb = ann['bbox']
                x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]]
                if not 'segmentation' in ann:
                    ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
                ann['area'] = bb[2]*bb[3]
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'segmentation' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                # now only support compressed RLE format as segmentation results
                ann['area'] = maskUtils.area(ann['segmentation'])
                if not 'bbox' in ann:
                    ann['bbox'] = maskUtils.toBbox(ann['segmentation'])
                ann['id'] = id+1
                ann['iscrowd'] = 0
        elif 'keypoints' in anns[0]:
            res.dataset['categories'] = copy.deepcopy(self.dataset['categories'])
            for id, ann in enumerate(anns):
                s = ann['keypoints']
                x = s[0::3]
                y = s[1::3]
                x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y)
                ann['area'] = (x1-x0)*(y1-y0)
                ann['id'] = id + 1
                ann['bbox'] = [x0,y0,x1-x0,y1-y0]
        print('DONE (t={:0.2f}s)'.format(time.time()- tic))

        res.dataset['annotations'] = anns
        res.createIndex()
        return res

    def loadAnns(self, ids=[]):
        """
        Load anns with the specified ids.
        :param ids (int array)       : integer ids specifying anns
        :return: anns (object array) : loaded ann objects
        """
        if _isArrayLike(ids):
            return [self.anns[id] for id in ids]
        elif type(ids) == int:
            return [self.anns[ids]]

    def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None):
        """
        Get ann ids that satisfy given filter conditions. default skips that filter
        :param imgIds  (int array)     : get anns for given imgs
               catIds  (int array)     : get anns for given cats
               areaRng (float array)   : get anns for given area range (e.g. [0 inf])
               iscrowd (boolean)       : get anns for given crowd label (False or True)
        :return: ids (int array)       : integer array of ann ids
        """
        imgIds = imgIds if _isArrayLike(imgIds) else [imgIds]
        catIds = catIds if _isArrayLike(catIds) else [catIds]

        if len(imgIds) == len(catIds) == len(areaRng) == 0:
            anns = self.dataset['annotations']
        else:
            if not len(imgIds) == 0:
                lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns]
                # print(len(lists), "lists")
                # print(lists)
                anns = list(itertools.chain.from_iterable(lists))
            else:
                anns = self.dataset['annotations']
            anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]]
        if not iscrowd == None:
            ids = [int(ann['id']) for ann in anns if ann['iscrowd'] == iscrowd]
        else:
            # for ann in anns:
            #     print("a", ann)
            ids = [int(ann['id']) for ann in anns]
        return ids


from sympy import symbols, diff, pprint, factorial, ln, exp, sin, log , cos
import numpy as np

def regla_falsa (f,x0,x1,tol,nmax):
    temp,err,cont,xcentro = 0,0,1,0
    xant = f(x0)
    xact = f(x1)
    if (xant == 0):
        print(x0, "es raiz")
    else:
        if (xact == 0):
            print(x1, "es raiz")
        else:
            if (xant*xact < 0):
                xcentro = x0 - (xant*(x1-x0)/(xact-xant))
                fcentro = f(xcentro)
                err = tol + 1
                print("\n-------------------------------------------------------")
                print("iteracion | xMedio | fcentro | error absoluto")
                #print(cont, "|", x0,"|",x1,"|",xcentro,"|",fcentro,"|",err)
                print(cont, "|",xcentro,"|",fcentro,"|",err)
                while((err > tol) and (fcentro != 0) and cont < nmax):
                    if (xant*fcentro < 0):
                        x1 = xcentro
                        xact = fcentro
                    else:
                        x0 = xcentro
                        xant = fcentro
                    temp = xcentro
                    xcentro = x0 - (xant*(x1-x0)/(xact-xant))
                    fcentro = f(xcentro)
                    err = abs(xcentro - temp)
                    cont += 1
                    #print(cont, "|", x0,"|",x1,"|",xcentro,"|",fcentro,"|",err)
                    print(cont, "|",xcentro,"|",fcentro,"|",err)
                if (fcentro == 0):
                    print(xcentro,"es raiz")
                else:
                    if(err < tol):
                        print('\n')
                        print(xcentro,"es una aproximacion a la raiz")
                        print(err, "con error")
                    else:
                        print("el metodo fracaso")
            else:
                print("el intervalo no sirve")

def f(x):
    return exp(-x+1) - x**2 + 4*x + (7*(cos(x**2 - 4)**2)) -7

#          (f,x0,x1,tol,nmax):
regla_falsa(f,0.0,3.3,1e-7,100)


import unittest
from unittest import mock
import Network
import json
import base64
import os

class MyTestCase(unittest.TestCase):

    def test_receiving_get_request(self):
        chat = Network.Network('name')
        mock_socket = mock.Mock()
        msg = chat.create_file_data(file_location='local', file_name='name',
                                    action='get', address=chat.host)
        chat.extract_messages(msg, mock_socket)
        self.assertEqual(chat._files_to_send, {mock_socket:('local', 'name')})
        chat.receiving_socket.close()

    def test_accepting_new_connection(self):
        chat = Network.Network('name')
        chat.exit_condition.set()
        mock_recv_socket = mock.Mock()
        mock_new_sock = mock.Mock()
        mock_new_sock.getpeername.return_value = 'address'
        chat.receiving_socket.close()
        chat.receiving_socket = mock_recv_socket
        mock_recv_socket.accept.return_value = (mock_new_sock, 'address')
        chat.server_handler()
        mock_recv_socket.accept.assert_called_once()
        mock_new_sock.getpeername.assert_called_once()

    def test_sending_file(self):
        chat = Network.Network('name')
        mock_sock = mock.Mock()
        mock_sock.send.return_value = None
        chat._files_to_send[mock_sock] = (os.path.join(os.getcwd(), 'file'), 'file')
        with open('file', 'wb') as f:
            f.write(b'abc')
        chat._send_file(mock_sock)
        self.assertEqual(chat._files_to_send, {})
        mock_sock.send.assert_called_once()
        chat.receiving_socket.close()
        os.remove(os.path.join(os.getcwd(), 'file'))

    def test_server_has_connections(self):
        chat = Network.Network('name')
        chat.exit_condition.set()
        mock_sock = mock.Mock()
        chat._socket_connections[mock_sock] = 'address'
        mock_handler = mock.Mock()
        mock_handler.return_value = None
        chat._connections_handler = mock_handler
        chat.server_handler()
        mock_handler.assert_called_once()

    def test_removing_closed_sockets(self):
        chat = Network.Network('name')
        mock_sock1 = mock.Mock()
        mock_sock2 = mock.Mock()
        mock_sock1.fileno.return_value = -1
        mock_sock2.fileno.return_value = 1
        chat._socket_connections[mock_sock1] = None
        chat._socket_connections[mock_sock2] = None
        chat._check_connections()
        self.assertEqual(chat._socket_connections, {mock_sock2: None})
        chat.receiving_socket.close()

    def test_receiving_msg(self):
        chat = Network.Network('name')
        mock_sock = mock.Mock()
        message = chat._create_message('message')
        chat.extract_messages(message, mock_sock)
        self.assertEqual(chat.messages_from_users.get(), json.loads(message))
        chat.receiving_socket.close()

    def test_receiving_unknown_contacts(self):
        chat = Network.Network('name')
        mock_socket = mock.Mock()
        new_connections = [['host1', 'username1'], ['host2', 'username2']]
        message = chat.create_data(
            connections=new_connections,
            host='host2',
            user='username2',
        )
        chat.extract_messages(message, mock_socket)
        connections = {x: y for x, y in new_connections}
        chat.host_connections.pop(chat.host)
        self.assertEqual(connections, chat.host_connections)
        chat.receiving_socket.close()

    def test_disconnect(self):
        chat = Network.Network('name')
        mock_socket = mock.Mock()
        mock_socket.send = mock.Mock()
        mock_socket.close = mock.Mock()
        chat._socket_connections[mock_socket] = ''
        chat._disconnect(mock_socket)
        chat.receiving_socket.close()

        mock_socket.send.assert_called_once()
        mock_socket.close.assert_called_once()
        self.assertEqual(chat._socket_connections, {})

    def test_download_file(self):
        if not os.path.exists('Downloads'):
            os.mkdir('Downloads')
        file_data = base64.b64encode(b'abc').decode('utf-8')
        msg = json.loads(Network.Network.create_file_data(file=file_data, file_name='file_name'))
        Network.Network._download_file(msg)
        with open(r'Downloads\file_name') as f:
            text = f.read()
        self.assertTrue(os.path.exists(os.path.join(os.getcwd(), 'Downloads', 'file_name')))
        self.assertTrue(text == 'abc')
        os.remove(os.path.join(os.getcwd(), 'Downloads', 'file_name'))

if __name__ == '__main__':
    unittest.main()


from sys import stdin, setrecursionlimit

setrecursionlimit(10**6)

def main():
    N = int(stdin.readline())
    A = []
    B = []
    for _ in range(N):
        a, b = map(int, stdin.readline().split())
        A.append(a)
        B.append(b)

    # 絶対値が最大になる瞬間である4パターンを探索すれば良い
    # Aの表が最大になる場合
    pattern1_A = 0
    pattern1_B = 0
    # Aの裏が最大になる場合
    pattern2_A = 0
    pattern2_B = 0
    # Bの表が最大になる場合
    pattern3_A = 0
    pattern3_B = 0
    # Bの裏が最大になる場合
    pattern4_A = 0
    pattern4_B = 0
    for i in range(N):
        if A[i] + B[i] > 0:
            pattern1_A += A[i]
            pattern1_B += B[i]
        if -A[i]-B[i] > 0:
            pattern2_A += A[i]
            pattern2_B += B[i]
        if A[i] - B[i] > 0:
            pattern3_A += A[i]
            pattern3_B += B[i]
        if -A[i] + B[i] > 0:
            pattern4_A += A[i]
            pattern4_B += B[i]
    print(max(abs(pattern1_A)+abs(pattern1_B), abs(pattern2_A)+abs(pattern2_B),
          abs(pattern3_A)+abs(pattern3_B), abs(pattern4_A)+abs(pattern4_B)))

if __name__ == '__main__':
    main()


import pandas as pd

def to_file(data_frame, year1, year2, output_file):

    df1 = data_frame[data_frame[year2 + " - " + year1] > 0]
    df2 = data_frame[data_frame[year2 + " - " + year1] < 0]
    df3 = data_frame[data_frame[year2 + " - " + year1] == 0]

    with pd.ExcelWriter(output_file) as writer:

        if not df1.empty:

            df1.to_excel(writer, sheet_name="Większe dochody w " + year2, index=False)

        if not df2.empty:

            df2.to_excel(writer, sheet_name="Większe dochody w " + year1, index=False)

        if not df3.empty:

            df3.to_excel(writer, sheet_name="Dochody bez zmian", index=False)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This module contains the MatchTem class.
"""
from harpia.GUI.fieldtypes import *
from harpia.model.plugin import Plugin

class MatchTem(Plugin):
    """
    This class contains methods related the MatchTem class.
    """
    # -------------------------------------------------------------------------

    def __init__(self):
        Plugin.__init__(self)
        self.help = "Operação de filtragem destinada a suavizar uma imagem."
        self.label = "Match Template"
        self.color = "180:180:10:150"
        self.in_ports = [{"type":"harpia.extensions.c.ports.image",
                          "name":"first_image",
                          "label":"First Image"},
                         {"type":"harpia.extensions.c.ports.image",
                          "name":"second_image",
                          "label":"Second Image"}
                         ]
        self.out_ports = [{"type":"harpia.extensions.c.ports.image",
                           "name":"output_image",
                           "label":"Output Image"}]
        self.group = "Feature Detection"

        self.properties = [{"label": "Scale Factor",
                            "name": "scaleFactor",
                            "type": HARPIA_INT,
                            "lower": 0,
                            "upper": 99,
                            "step": 1,
                            "value":6
                            },
                           {"label": "Method",
                            "name": "method",
                            "type": HARPIA_COMBO,
                            "value":'CV_TM_SQDIFF',
                            "values": ["CV_TM_CCOEFF_NORMED", "CV_TM_CCOEFF",
                                       "CV_TM_CCORR_NORMED", "CV_TM_CCORR",
                                       "CV_TM_SQDIFF_NORMED", "CV_TM_SQDIFF"
                                       ]
                            }
                           ]

        # ------------------------------C/OpenCv code--------------------------
        self.codes[1] = \
            'IplImage * block$id$_img_i0 = NULL;\n' + \
            'IplImage * block$id$_img_i1 = NULL;\n' + \
            'IplImage * block$id$_img_t0 = NULL;\n' + \
            'IplImage * block$id$_img_o0 = NULL;\n'

        self.codes[2] =  \
            'if(block$id$_img_i0 && block$id$_img_i1){\n' + \
            '\tdouble width$id$ = block$id$_img_i0->width - ' + \
            'block$id$_img_i1->width +1;\n' + \
            '\tdouble height$id$ = block$id$_img_i0->height - ' + \
            'block$id$_img_i1->height +1;\n' + \
            '\tCvSize size$id$ = cvSize(width$id$,height$id$);\n' + \
            '\tblock$id$_img_t0 = cvCreateImage(size$id$,32,1);\n' + \
            '\tblock$id$_img_o0 = cvCreateImage(size$id$,8,1);\n' + \
            '\tcvMatchTemplate(block$id$_img_i0 , block$id$_img_i1, ' + \
            'block$id$_img_t0, $method$);\n' + \
            '\tcvConvertScale(block$id$_img_t0, block$id$_img_o0, pow(10,-($scaleFactor$)),0);\n' + \
            '}\n'

        self.codes[3] = \
            'cvReleaseImage(&block$id$_img_o0);\n' + \
            'cvReleaseImage(&block$id$_img_t0);\n' + \
            'cvReleaseImage(&block$id$_img_i1);\n' + \
            'cvReleaseImage(&block$id$_img_i0);\n'
        self.language = "c"
        self.framework = "opencv"
# -----------------------------------------------------------------------------


from urllib.request import urlopen
from bs4 import BeautifulSoup

def obtain_reqs(URL,list_len,answer_list,list_begin):
    url = URL
    html = urlopen(url).read()
    soup = BeautifulSoup(html)

    for script in soup(["script", "style"]):
        script.decompose()

    strips = list(soup.stripped_strings)

    start_idx = strips.index(list_begin)
    ans = answer_list

    info = {}
    for i in range(1,list_len+1):
        info["Question " +str(i)+ ": " +strips[start_idx+i]] = "Answer " +str(i)+ ": " +ans[i-1]
    return info

def return_info():
    info = obtain_reqs(URL="https://fetch-hiring.s3.amazonaws.com/data-engineer/text-similarity.html",
           list_len=5,
           answer_list=["My application does not count punctuation",
                      "My application removes common stop words",
                      "My application does not care about the ordering of words",
                      "Number of unique matched words divided by total unique words in both samples",
                      "Dictionaries, lists, sets, and tuples are employed. Sets are used for comparing unique words between two samples and are helpful because they are more performant for lookups than lists."],
           list_begin="You will have to make a number of decisions as you develop this solution:")

    return info


def random_walk_2D(np, ns, plot_step):
    xpositions = numpy.zeros(np)
    ypositions = numpy.zeros(np)
    moves = numpy.random.random_integers(1, 4, size=ns*np)
    moves.shape = (ns, np)

    # Estimate max and min positions
    xymax = 3*numpy.sqrt(ns); xymin = -xymax

    NORTH = 1;  SOUTH = 2;  WEST = 3;  EAST = 4  # constants

    for step in range(ns):
        this_move = moves[step,:]
        ypositions += numpy.where(this_move == NORTH, 1, 0)
        ypositions -= numpy.where(this_move == SOUTH, 1, 0)
        xpositions += numpy.where(this_move == EAST,  1, 0)
        xpositions -= numpy.where(this_move == WEST,  1, 0)

        # Just plot every plot_step steps
        if (step+1) % plot_step == 0:
            plot(xpositions, ypositions, 'ko',
                 axis=[xymin, xymax, xymin, xymax],
                 title='%d particles after %d steps' %
                       (np, step+1),
                 savefig='tmp_%03d.eps' % (step+1))
    return xpositions, ypositions

# Main program
from scitools.std import plot
import numpy, sys
numpy.random.seed(11)

np = int(sys.argv[1])  # number of particles
ns = int(sys.argv[2])  # number of steps
plot_step = int(sys.argv[3])  # plot each plot_step step
x, y = random_walk_2D(np, ns, plot_step)


import torch
import numpy as np
import torch.nn as nn
import torch.optim as optim
from torch_geometric.nn import GCNConv
from tqdm import tqdm

# Define the Graph Neural Network model
# concatenate with its own node feature
class GCNAnomalyDetector(nn.Module):
    def __init__(self, num_features, hidden_size):
        super(GCNAnomalyDetector, self).__init__()
        self.conv1 = GCNConv(num_features, hidden_size)
        self.conv2 = GCNConv(hidden_size, hidden_size*2)
        self.fc = nn.Linear(hidden_size*2+10, 1)

    def forward(self, x, edge_index):
        x_1 = x
        x = self.conv1(x, edge_index)
        x = torch.relu(x)
        x = self.conv2(x, edge_index)
        x_2 = torch.relu(x)
        x = torch.cat((x_1, x_2), dim=1)
        x = self.fc(x).squeeze(1)
        return x

# Load the training dataset
train_data = torch.load('train_sub-graph_tensor.pt') # torch_geometric.data.data.Data
train_edge_index = train_data.edge_index # 2, 6784824
train_feature = train_data.feature.float() # Convert to double
train_label = train_data.label # 15742

# Load the testing dataset
test_data = torch.load('test_sub-graph_tensor_noLabel.pt') # torch_geometric.data.data.Data
test_edge_index = test_data.edge_index # 2, 7000540
test_feature = test_data.feature.float() # Convert to double

# Create a mask to specify training nodes
# torch.Tensor, 39357
train_mask = torch.from_numpy(np.load('train_mask.npy')).bool()

# Create a mask to specify testing nodes
# torch.Tensor, 39657
test_mask = torch.from_numpy(np.load('test_mask.npy')).bool()

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Create the model and optimizer
num_epochs = 10000
model = GCNAnomalyDetector(num_features=train_feature.shape[1], hidden_size=32).to(device)
optimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=5e-4)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-5)

# Training loop
model.train()
pbar = tqdm(total=num_epochs, desc='Training')
for epoch in range(num_epochs):
    optimizer.zero_grad()
    train_feature = train_feature.to(device)
    train_edge_index = train_edge_index.to(device)
    train_label = train_label.to(device)
    logits = model(train_feature, train_edge_index)
    loss = nn.CrossEntropyLoss()(logits[train_mask], train_label.float())
    loss.backward()
    optimizer.step()
    scheduler.step()
    pbar.update(1)
    pbar.set_postfix({'Epoch': epoch+1, 'Loss': loss.item()})
pbar.close()

# Save the trained model
torch.save(model.state_dict(), 'gcn_anomaly_model.pt')

# Testing
# node idx, node anomaly score
# save the anomaly score for each node into a numpy array

# initialize the storage with np.zeros
anomaly_score = np.zeros(test_feature.shape[0], dtype=np.float64)

# load the trained model
model.load_state_dict(torch.load('gcn_anomaly_model.pt'))

model.eval()
with torch.no_grad():
    test_feature = test_feature.to(device)
    test_edge_index = test_edge_index.to(device)
    logits = model(test_feature, test_edge_index)
    # calculate the probability of each node being anomalous
    prob = torch.sigmoid(logits)

# save into csv file called anomaly_score.csv
# there are two columns: node idx, node anomaly score

idx = 0

with open('anomaly_score.csv', 'w') as f:
    f.write('node idx,node anomaly score\n')
    for i in test_mask:
        if i == True:
            # write the node idx and anomaly score into the csv line by line
            f.write(str(idx) + ',' + str(prob[idx].item()) + '\n')
        idx += 1


import os
import sqlite3

import yaml

BASE_DE_DATOS = os.path.join('data', 'database.sqlite')
ARCHIVO_SQL = os.path.join('data', 'declaraciones_sql.yml')

class DatabaseHelper(object):
    def __init__(self):
        self._db_connection = None

    def __enter__(self):
        self._db_connection = sqlite3.connect(BASE_DE_DATOS)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._db_connection.close()

    def crear_tablas(self):
        """
        Creamos la base de datos con las instrucciones que se
        encuentran en el archivo declaraciones.yml
        """
        with open(ARCHIVO_SQL, 'r+') as f:
            declaraciones = yaml.safe_load(f)

        declaraciones_crear = declaraciones.get('crear', list())

        for s in declaraciones_crear:
            self.insertar(s)

    def eliminar_tablas(self):
        """
        Elimina la base de datos según las instrucciones
        destruir, dentro del archivo declaraciones.yml
        """
        with open(ARCHIVO_SQL, 'r+') as f:
            declaraciones = yaml.safe_load(f)

        declaraciones_eliminar = declaraciones.get('eliminar', list())

        for s in declaraciones_eliminar:
            self.insertar(s)

    @property
    def cursor(self):
        """
        Retorna una conexión (cursor)
        a la base de datos
        """
        if not self._db_connection:
            self._db_connection = sqlite3.connect(BASE_DE_DATOS)
        return self._db_connection.cursor()

    def reiniciar_base_de_datos(self):
        """
        Elimina y crea nuevamente las tablas dentro de
        la base de datos.
        """
        self.eliminar_tablas()
        self.crear_tablas()

        with open(ARCHIVO_SQL, 'r+') as f:
            declaraciones = yaml.safe_load(f)

        reiniciar_base_de_datos_declaraciones = declaraciones.get('reiniciar', list())

        for s in reiniciar_base_de_datos_declaraciones:
            self.insertar(s)

    def select(self, consulta):
        """
        Ejecuta la consulta en la base de datos
        y lo traslada a cadenas (con el método fetchall())
        """
        cursor = self.cursor
        cursor.execute(consulta)
        return cursor.fetchall()

    def insertar(self, consulta):
        """
        Ejecuta la consulta cruda en la bases de datos
        """
        cursor = self.cursor
        cursor.executescript(consulta)
        self._db_connection.commit()

    def select_safe(self, consulta, params):
        """
        Ejecuta la consulta de forma junto a los
        parámetros (esto lo hace resistente a la inyección SQL)
        """
        cursor = self.cursor
        if params:
            cursor.execute(consulta, params)
        else:
            cursor.execute(consulta)
        return cursor.fetchall()


from sourpea.primitives import *
from sourpea.util import *
import pandas as pd
import os
_dir=os.path.dirname(__file__)
 ### DERIVED FACTORS
elfx = Factor("elfx", ["zbydej","bpqtrb","vxtnpa","xfhmhy","qpilm","fzj"])
def ejony(elfx):
    return elfx[-1] == "vxtnpa" and elfx[-3] != "qpilm"
def rld(elfx):
    return elfx[-1] != "vxtnpa" or elfx[-3] == "qpilm"
kmgcl = Factor("ypqk", [DerivedLevel("mxonf", Window(ejony, [elfx],4)), DerivedLevel("dhoai", Window(rld, [elfx],4))])
### EXPERIMENT
design=[elfx,kmgcl]
crossing=[kmgcl]
### APPENDIX
block=Block(design,crossing,[])
sequence_code_1=trials_from_csv(os.path.join(_dir,"out_code_1/2_1_3_0.csv"))
nr_input_factors=1
nr_output_levels=2
test_code_1=block.test(sequence_code_1)
try:
	sequence_code_2=trials_from_csv(os.path.join(_dir,"out_code_2/2_1_3_0.csv"))
	test_code_2=block.test(sequence_code_2)
except:
	test_code_2={"pValue":0}
df=pd.read_csv(os.path.join(_dir,"result_sour_c1.csv"))
df.loc[len(df.index)] = [test_code_1["pValue"], test_code_2["pValue"], test_code_1["levels"], test_code_2["levels"], nr_input_factors, nr_output_levels]
df.to_csv(os.path.join(_dir,"result_sour_c1.csv"), index=False)


from googlesearch import search
from bs4 import BeautifulSoup
import requests
from docx import Document
from docx.shared import Inches

# Search query
query = "Eryngium billardieri"

# Number of search results
num_results = 40

# Perform the search
search_results = search(query, num_results=num_results, lang='en')

# Create a Word document
doc = Document()
doc.add_heading(f"Articles about '{query}'", level=1)

# Create a table with headers
table = doc.add_table(rows=1, cols=4)
table.style = 'Table Grid'
table.autofit = False
table.columns[0].width = Inches(0.5)
table.columns[1].width = Inches(2)
table.columns[2].width = Inches(2)
table.columns[3].width = Inches(4)

# Set the header text
header_cells = table.rows[0].cells
header_cells[0].text = "No."
header_cells[1].text = "Article"
header_cells[2].text = "Link"
header_cells[3].text = "Abstract"

# Add the search results to the table
for index, result in enumerate(search_results):
    row_cells = table.add_row().cells
    row_cells[0].text = str(index + 1)
    row_cells[2].text = result

    try:
        response = requests.get(result)
        soup = BeautifulSoup(response.text, 'html.parser')
        title = soup.title.text.strip()
        abstract = soup.find('meta', attrs={'name': 'description'})
        abstract = abstract['content'].strip() if abstract else "N/A"
        row_cells[1].text = title
        row_cells[3].text = abstract
    except:
        row_cells[1].text = "N/A"
        row_cells[3].text = "N/A"

# Save the document
doc.save("articles.docx")

print(f"Articles about '{query}' saved to 'articles.docx'.")


from django import forms
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
import urllib
from django.contrib.auth.models import User
from django.core.servers.basehttp import FileWrapper
from django.conf import settings
import game.swf
import magic
import pprint
import game.models as models

## helper function

def _check_u(f):
    def ff(request,*args,**kwargs):
        username=request.GET.get("u",None)
        if username!=None:
            try:
                user = User.objects.get(username__exact = username)
                return f(request,*args,user=user,**kwargs)
            except User.DoesNotExist:
                return HttpResponseRedirect(reverse("game.views.index"))
        if not request.user.is_authenticated():
            return HttpResponseRedirect(reverse("game.views.index"))
        p = {"u":request.user.username}
        return HttpResponseRedirect(request.path+"?"+urllib.urlencode(p))
    return ff

def _check_game_entry(f):
    def ff(request,game_entry,*args,**kwargs):
        try:
            db_game = game.models.Game.objects.get(pk = game_entry)
            return f(request,*args,db_game=db_game,**kwargs)
        except game.models.Game.DoesNotExist:
            return HttpResponseRedirect(reverse("game.views.index"))
    return ff

# view func

def index(request):
    return render(request,"game/index.tmpl")

@_check_u
def get_user_profile(request,user):
    return render(request,"dummy.tmpl")

@_check_u
def get_game_score_list(request, user):
    return render(request,"dummy.tmpl")

get_game_list_select_map={
    "comment_count" : 'SELECT COUNT(*) FROM game_gamecomment WHERE game_gamecomment.game_id = game_game.id',
    "report_count"  : 'SELECT COUNT(*) FROM game_scorereport WHERE game_scorereport.game_id = game_game.id',
}

@_check_u
def get_game_list(request, user):
    db_game_list = game.models.Game.objects.filter(author__exact=user).extra(select=get_game_list_select_map,)
    db_gamediff_list = game.models.GameDiff.objects.filter(game__in=db_game_list).select_related("game")
    game_map = {}
    for db_game in db_game_list:
        g = {}
        for k in ["id","title","create_date","comment_count","report_count"]:
            g[k] = getattr(db_game,k)
        g["star"]=game.create_empty_star_list()
        game_map[g["id"]]=g
    for db_gamediff in db_gamediff_list :
        game_id = db_gamediff.game.id
        ura_idx = 1 if db_gamediff.ura else 0
        diff_idx = db_gamediff.diff
        star = db_gamediff.star
        game_map[game_id]["star"][ura_idx][diff_idx] = star
    return render(request,"game/get_game_list.tmpl",{"game_list":game_map.values(),"list_user":user,"is_me":request.user==user})

@_check_game_entry
def get_game(request, db_game):
    comment_form = AddGameCommentForm(initial={"game_entry":db_game.id})

    comment_list=[]

    db_gamecomment_list = models.GameComment.objects.filter(game=db_game).order_by("-id")
    for db_gamecomment in db_gamecomment_list:
        comment_list.append({
            "author":db_gamecomment.player.username,
            "content":db_gamecomment.content,
            "datetime":db_gamecomment.create_date
        })

    return render(request,"game/get_game.tmpl",{
        "game":db_game,
        "comment_form":comment_form,
        "comment_list":comment_list
    })

@_check_game_entry
def get_game_data(request, db_game):
    return HttpResponse(FileWrapper(db_game.data),content_type="text/plain")

@_check_game_entry
def get_game_bgm(request, db_game):
    return HttpResponse(FileWrapper(db_game.bgm),content_type="audio/mpeg")

@_check_game_entry
def get_game_swf(request, db_game):
    return HttpResponse(
        FileWrapper(open("%(root)s/res/%(id)s.swf"%{"root":settings.OWATUBE_PATH,"id":db_game.swf},"r")),
        content_type="application/x-shockwave-flash"
    )

@login_required
def add_game_comment(request):
    if request.method == "POST":
        comment_form = AddGameCommentForm(request.POST)
        if comment_form.is_valid():
            game_entry = comment_form.cleaned_data["game_entry"]
            game_data = game.models.Game.objects.get(pk=game_entry)
            comment = comment_form.cleaned_data["comment"]
            comment = comment.encode("utf-8")
            comment = comment.strip()

            game.models.GameComment.objects.create(
                game = game_data,
                player = request.user,
                content = comment,
            )

            return HttpResponseRedirect(reverse("game.views.get_game",kwargs={"game_entry":game_entry}))
    return HttpResponseRedirect(reverse("game.views.index"))

class AddGameCommentForm (forms.Form):
    game_entry = forms.CharField(widget=forms.widgets.HiddenInput())
    comment = forms.CharField(widget=forms.widgets.Textarea())

@login_required
def add_game_rank(request):
    return render(request,"dummy.tmpl")

@login_required
def add_game(request):
    if request.method == "POST":
        form = AddGameForm(request.POST, request.FILES)
        if form.is_valid():
            db_game = game.models.Game.objects.create(
                author=request.user,
                state=models.GAME_STATE_EDIT,
            )

            form.to_db(db_game, request)

            return HttpResponseRedirect(reverse("game.views.edit_game",kwargs={"game_entry":db_game.id}))

    else:
        form = AddGameForm(initial={"id":-1})

    return render(request,"game/add_game.tmpl",{"form" : form})

@login_required
@_check_game_entry
def edit_game(request, db_game):
    if request.user != db_game.author :
        # TODO give error
        return HttpResponseRedirect(reverse("game.views.index"))

    if request.method == "POST":
        form = AddGameForm(request.POST, request.FILES)
        if form.is_valid():
            form.to_db(db_game, request)

    form = AddGameForm(initial={"id":db_game.id})
    submit_form = SubmitGameForm(initial={"id":db_game.id})
    game_star = game.get_game_star(db_game)

    return render(request,"game/edit_game.tmpl",{
        "form" : form,
        "submit_form" : submit_form,
        "title" : db_game.title,
        "music_by" : db_game.music_by,
        "data_by" : db_game.data_by,
        "star" : game_star,
        "game_id" : db_game.id
    })

class AddGameForm (forms.Form):
    id = forms.IntegerField(widget=forms.widgets.HiddenInput())
    data = forms.FileField(required=False)
    bgm = forms.FileField(required=False)
    swf = forms.ChoiceField(choices=game.swf.SWF_CHOICE)
    # pic

    def clean_data(self):
        data = self.cleaned_data['data'];

        if self.cleaned_data['id'] != -1 and data == None:
            return data
        if data == None:
            raise forms.ValidationError("No file")

        data.open()
        buf=data.read()

        ms = magic.open(magic.MAGIC_MIME_TYPE)
        ms.load()
        t=ms.buffer(buf)
        ms.close()

        if(t!="text/plain"):raise forms.ValidationError("Not text file")

        return data

    def clean_bgm(self):
        bgm = self.cleaned_data['bgm'];

        if self.cleaned_data['id'] != -1 and bgm == None:
            return bgm
        if bgm == None:
            raise forms.ValidationError("No file")

        bgm.open()
        buf=bgm.read()

        ms = magic.open(magic.MAGIC_MIME_TYPE)
        ms.load()
        t=ms.buffer(buf)
        ms.close()

        if(t!="audio/mpeg"):raise forms.ValidationError("Not mp3 file")

        return bgm

    def to_db(self,db_game,request):
        swf_key = self.cleaned_data["swf"]
        db_game.swf = swf_key

        data_f=self.cleaned_data["data"]
        if data_f != None:
            data_f.open()
            data_buf = data_f.read()
            data_data = game.swf.parse_data(swf_key, data_buf)
            db_game.title = data_data["title"]
            db_game.music_by = data_data["music_by"]
            db_game.data_by=data_data["data_by"]
            db_game.data=request.FILES['data']
            game.models.GameDiff.objects.filter(
                game = db_game,
                ura = False,
            ).delete()
            for i in range(len(data_data["diff"])):
                star = data_data["diff"][i]
                if star == 0 : continue
                game.models.GameDiff.objects.create(
                    game = db_game,
                    ura = False,
                    diff = i,
                    star = star,
                )

        bgm_f = self.cleaned_data["bgm"]
        if bgm_f != None:
            db_game.bgm = request.FILES['bgm']

        db_game.save()

@login_required
def submit_game(request):
    if request.method != "POST":
        # TODO give error
        return HttpResponseRedirect(reverse("game.views.index"))

    form = SubmitGameForm(request.POST)
    if not form.is_valid():
        # TODO give error
        return HttpResponseRedirect(reverse("game.views.index"))

    game_id = form.cleaned_data["id"]

    db_game = game.models.Game.objects.get(pk = game_id)
    if db_game.author != request.user:
        # TODO give error
        return HttpResponseRedirect(reverse("game.views.index"))
    if db_game.state != models.GAME_STATE_EDIT:
        # TODO give error
        return HttpResponseRedirect(reverse("game.views.index"))

    db_game.state = models.GAME_STATE_PUBLIC
    db_game.save()

    return HttpResponseRedirect(reverse("game.views.get_game",kwargs={"game_entry":db_game.id}))

class SubmitGameForm (forms.Form):
    id = forms.IntegerField(widget=forms.widgets.HiddenInput())