import atexit
import logging
import sys

from datetime import datetime
from pathlib import Path

import bakta
import bakta.constants as bc
import bakta.config as cfg
import bakta.io.fasta as fasta
import bakta.io.json as json
import bakta.io.tsv as tsv
import bakta.io.gff as gff
import bakta.io.insdc as insdc
import bakta.expert.amrfinder as exp_amr
import bakta.expert.protein_sequences as exp_aa_seq
import bakta.features.annotation as anno
import bakta.features.t_rna as t_rna
import bakta.features.tm_rna as tm_rna
import bakta.features.r_rna as r_rna
import bakta.features.nc_rna as nc_rna
import bakta.features.nc_rna_region as nc_rna_region
import bakta.features.crispr as crispr
import bakta.features.orf as orf
import bakta.features.cds as feat_cds
import bakta.features.signal_peptides as sig_peptides
import bakta.features.s_orf as s_orf
import bakta.features.gaps as gaps
import bakta.features.ori as ori
import bakta.db as db
import bakta.utils as bu
import bakta.ups as ups
import bakta.ips as ips
import bakta.psc as psc
import bakta.pscc as pscc
import bakta.plot as plot

def main():
    args = bu.parse_arguments()  # parse arguments

    ############################################################################
    # Setup logging
    ############################################################################
    cfg.prefix = args.prefix if args.prefix else Path(args.genome).stem
    output_path = cfg.check_output_path(args.output, args.force)
    bu.setup_logger(output_path, cfg.prefix, args)
    log = logging.getLogger('MAIN')

    ############################################################################
    # Checks and configurations
    # - check parameters and setup global configuration
    # - test database
    # - test binary dependencies
    ############################################################################
    cfg.setup(args)  # check parameters and prepare global configuration
    cfg.db_info = db.check(cfg.db_path)
    bu.test_dependencies()
    if(cfg.verbose):
        print(f'Bakta v{bakta.__version__}')
        print('Options and arguments:')
        print(f'\tinput: {cfg.genome_path}')
        print(f"\tdb: {cfg.db_path}, version {cfg.db_info['major']}.{cfg.db_info['minor']}, {cfg.db_info['type']}")
        if(cfg.user_proteins): print(f'\tuser proteins: {cfg.user_proteins}')
        if(cfg.replicons): print(f'\treplicon table: {cfg.replicons}')
        if(cfg.prodigal_tf): print(f'\tprodigal training file: {cfg.prodigal_tf}')
        print(f'\toutput: {cfg.output_path}')
        if(cfg.force): print(f'\tforce: {cfg.force}')
        print(f'\ttmp directory: {cfg.tmp_path}')
        print(f'\tprefix: {cfg.prefix}')
        print(f'\tthreads: {cfg.threads}')
        if(cfg.debug): print(f'\tdebug: {cfg.debug}')
        if(cfg.meta): print(f'\tmeta mode: {cfg.meta}')
        print(f'\ttranslation table: {cfg.translation_table}')
        if(cfg.taxon): print(f'\ttaxon: {cfg.taxon}')
        if(cfg.plasmid): print(f'\tplasmid: {cfg.plasmid}')
        if(cfg.gram != '?'): print(f'\tgram: {cfg.gram}')
        if(cfg.locus): print(f'\tlocus prefix: {cfg.locus}')
        if(cfg.locus_tag): print(f'\tlocus tag prefix: {cfg.locus_tag}')
        if(cfg.complete): print(f'\tcomplete replicons: {cfg.complete}')
        if(cfg.compliant): print(f'\tINSDC compliant: {cfg.compliant}')
        if(cfg.keep_contig_headers): print(f'\tkeep contig headers: {cfg.keep_contig_headers}')
        if(cfg.skip_trna): print(f'\tskip tRNA: {cfg.skip_trna}')
        if(cfg.skip_tmrna): print(f'\tskip tmRNA: {cfg.skip_tmrna}')
        if(cfg.skip_rrna): print(f'\tskip rRNA: {cfg.skip_rrna}')
        if(cfg.skip_ncrna): print(f'\tskip ncRNA: {cfg.skip_ncrna}')
        if(cfg.skip_ncrna_region): print(f'\tskip ncRNA region: {cfg.skip_ncrna_region}')
        if(cfg.skip_crispr): print(f'\tskip CRISPR: {cfg.skip_crispr}')
        if(cfg.skip_cds): print(f'\tskip CDS: {cfg.skip_cds}')
        if(cfg.skip_sorf): print(f'\tskip sORF: {cfg.skip_sorf}')
        if(cfg.skip_gap): print(f'\tskip gap: {cfg.skip_gap}')
        if(cfg.skip_ori): print(f'\tskip oriC/V/T: {cfg.skip_ori}')
        if(cfg.skip_plot): print(f'\tskip plot: {cfg.skip_plot}')

    if(cfg.debug):
        print(f"\nBakta runs in DEBUG mode! Temporary data will not be destroyed at: {cfg.tmp_path}")
    else:
        atexit.register(bu.cleanup, log, cfg.tmp_path)  # register cleanup exit hook

    ############################################################################
    # Import genome
    # - parse contigs in Fasta file
    # - apply contig length filter
    # - rename contigs
    ############################################################################
    print('\nparse genome sequences...')
    try:
        contigs = fasta.import_contigs(cfg.genome_path)
        log.info('imported sequences=%i', len(contigs))
        print(f'\timported: {len(contigs)}')
    except:
        log.error('wrong genome file format!', exc_info=True)
        sys.exit('ERROR: wrong genome file format!')
    replicons = bu.parse_replicon_table(cfg.replicons) if cfg.replicons else None
    contigs, complete_genome = bu.qc_contigs(contigs, replicons)
    print(f'\tfiltered & revised: {len(contigs)}')
    no_chromosomes = len([c for c in contigs if c['type'] == bc.REPLICON_CHROMOSOME])
    if(no_chromosomes > 0):
        print(f"\tchromosomes: {no_chromosomes}")
    no_plasmids = len([c for c in contigs if c['type'] == bc.REPLICON_PLASMID])
    if(no_plasmids > 0):
        print(f"\tplasmids: {no_plasmids}")
    no_contigs = len([c for c in contigs if c['type'] == bc.REPLICON_CONTIG])
    if(no_contigs > 0):
        print(f"\tcontigs: {no_contigs}")
    if(len(contigs) == 0):
        log.warning('no valid contigs!')
        sys.exit('Error: input file contains no valid contigs.')
    contigs_path = cfg.tmp_path.joinpath('contigs.fna')
    fasta.export_contigs(contigs, contigs_path)
    genome = {
        'genus': cfg.genus,
        'species': cfg.species,
        'strain': cfg.strain,
        'taxon': cfg.taxon,
        'gram': cfg.gram,
        'translation_table': cfg.translation_table,
        'size': sum([c['length'] for c in contigs]),
        'complete': cfg.complete or complete_genome,
        'features': {},
        'contigs': contigs
    }
    if(cfg.plasmid):
        genome['plasmid'] = cfg.plasmid
    print('\nstart annotation...')

    ############################################################################
    # tRNA prediction
    ############################################################################
    if(cfg.skip_trna):
        print('skip tRNA prediction...')
    else:
        print('predict tRNAs...')
        log.debug('start tRNA prediction')
        genome['features'][bc.FEATURE_T_RNA] = t_rna.predict_t_rnas(genome, contigs_path)
        print(f"\tfound: {len(genome['features'][bc.FEATURE_T_RNA])}")

    ############################################################################
    # tmRNA prediction
    ############################################################################
    if(cfg.skip_tmrna):
        print('skip tmRNA prediction...')
    else:
        print('predict tmRNAs...')
        log.debug('start tmRNA prediction')
        genome['features'][bc.FEATURE_TM_RNA] = tm_rna.predict_tm_rnas(genome, contigs_path)
        print(f"\tfound: {len(genome['features'][bc.FEATURE_TM_RNA])}")

    ############################################################################
    # rRNA prediction
    ############################################################################
    if(cfg.skip_rrna):
        print('skip rRNA prediction...')
    else:
        print('predict rRNAs...')
        log.debug('start rRNA prediction')
        genome['features'][bc.FEATURE_R_RNA] = r_rna.predict_r_rnas(genome, contigs_path)
        print(f"\tfound: {len(genome['features'][bc.FEATURE_R_RNA])}")

    ############################################################################
    # ncRNA gene prediction
    ############################################################################
    if(cfg.skip_ncrna):
        print('skip ncRNA prediction...')
    else:
        print('predict ncRNAs...')
        log.debug('start ncRNA prediction')
        genome['features'][bc.FEATURE_NC_RNA] = nc_rna.predict_nc_rnas(genome, contigs_path)
        print(f"\tfound: {len(genome['features'][bc.FEATURE_NC_RNA])}")

    ############################################################################
    # ncRNA region prediction
    ############################################################################
    if(cfg.skip_ncrna_region):
        print('skip ncRNA region prediction...')
    else:
        print('predict ncRNA regions...')
        log.debug('start ncRNA region prediction')
        genome['features'][bc.FEATURE_NC_RNA_REGION] = nc_rna_region.predict_nc_rna_regions(genome, contigs_path)
        print(f"\tfound: {len(genome['features'][bc.FEATURE_NC_RNA_REGION])}")

    ############################################################################
    # CRISPR prediction
    ############################################################################
    if(cfg.skip_crispr):
        print('skip CRISPR array prediction...')
    else:
        print('predict CRISPR arrays...')
        log.debug('start CRISPR prediction')
        genome['features'][bc.FEATURE_CRISPR] = crispr.predict_crispr(genome, contigs_path)
        print(f"\tfound: {len(genome['features'][bc.FEATURE_CRISPR])}")

    ############################################################################
    # CDS prediction
    # - Prodigal prediction
    # - lookup UPS matches
    # - lookup IPS matches
    # - search PSC for unannotated CDSs
    # - conduct expert systems analysis
    # - lookup & combine annotations
    # - analyze hypotheticals
    ############################################################################
    if(cfg.skip_cds):
        print('skip CDS prediction...')
    else:
        print('predict & annotate CDSs...')
        log.debug('predict CDS')
        cdss = feat_cds.predict(genome, contigs_path)
        print(f"\tpredicted: {len(cdss)} ")

        if(len(cdss) > 0):
            log.debug('detect spurious CDS')
            discarded_cdss = orf.detect_spurious(cdss)
            print(f'\tdiscarded spurious: {len(discarded_cdss)}')
            cdss = [cds for cds in cdss if 'discarded' not in cds]

        if(len(cdss) > 0):
            log.debug('revise translational exceptions')
            no_revised = feat_cds.revise_translational_exceptions(genome, cdss)
            print(f'\trevised translational exceptions: {no_revised}')
            cdss = [cds for cds in cdss if 'discarded' not in cds]

        if(len(cdss) > 0):
            log.debug('lookup CDS UPS/IPS')
            cdss_ups, cdss_not_found = ups.lookup(cdss)
            cdss_ips, sorf_pscs = ips.lookup(cdss_ups)
            cdss_not_found.extend(sorf_pscs)
            print(f'\tdetected IPSs: {len(cdss_ips)}')

            if(len(cdss_not_found) > 0):
                if(cfg.db_info['type'] == 'full'):
                    log.debug('search CDS PSC')
                    cdss_psc, cdss_pscc, cdss_not_found = psc.search(cdss_not_found)
                    print(f'\tfound PSCs: {len(cdss_psc)}')
                    print(f'\tfound PSCCs: {len(cdss_pscc)}')
                else:
                    log.debug('search CDS PSCC')
                    cdss_pscc, cdss_not_found = pscc.search(cdss_not_found)
                    print(f'\tfound PSCCs: {len(cdss_pscc)}')
            print('\tlookup annotations...')
            log.debug('lookup CDS PSCs')
            psc.lookup(cdss)  # lookup PSC info
            pscc.lookup(cdss)  # lookup PSCC info

            print('\tconduct expert systems...')  # conduct expert systems annotation
            cds_aa_path = cfg.tmp_path.joinpath('cds.expert.faa')
            orf.write_internal_faa(cdss, cds_aa_path)
            log.debug('conduct expert system: amrfinder')
            expert_amr_found = exp_amr.search(cdss, cds_aa_path)
            print(f'\t\tamrfinder: {len(expert_amr_found)}')
            log.debug('conduct expert system: aa seqs')
            diamond_db_path = cfg.db_path.joinpath('expert-protein-sequences.dmnd')
            expert_aa_found = exp_aa_seq.search(cdss, cds_aa_path, 'expert_proteins', diamond_db_path)
            print(f'\t\tprotein sequences: {len(expert_aa_found)}')

            if(cfg.user_proteins):
                log.debug('conduct expert system: user aa seqs')
                user_aa_path = cfg.tmp_path.joinpath('user-proteins.faa')
                exp_aa_seq.write_user_protein_sequences(user_aa_path)
                user_aa_found = exp_aa_seq.search(cdss, cds_aa_path, 'user_proteins', user_aa_path)
                print(f'\t\tuser protein sequences: {len(user_aa_found)}')

            if(cfg.gram != bc.GRAM_UNKNOWN):
                sig_peptides_found = sig_peptides.search(cdss, cds_aa_path)
                print(f'\tsignal peptides: {len(sig_peptides_found)}')

            print('\tcombine annotations and mark hypotheticals...')
            log.debug('combine CDS annotations')
            for cds in cdss:
                anno.combine_annotation(cds)  # combine IPS & PSC annotations and mark hypotheticals

            hypotheticals = [cds for cds in cdss if 'hypothetical' in cds and 'edge' not in cds and cds.get('start_type', 'Edge') != 'Edge']
            if(len(hypotheticals) > 0  and  not cfg.skip_pseudo  and  cfg.db_info['type'] == 'full'):
                print('\tdetect pseudogenes...')
                log.debug('search pseudogene candidates')
                pseudo_candidates = feat_cds.predict_pseudo_candidates(hypotheticals)
                print(f'\t\tpseudogene candidates: {len(pseudo_candidates)}')
                pseudogenes = feat_cds.detect_pseudogenes(pseudo_candidates, cdss, genome) if len(pseudo_candidates) > 0 else []
                psc.lookup(pseudogenes, pseudo=True)
                pscc.lookup(pseudogenes, pseudo=True)
                for pseudogene in pseudogenes:
                    anno.combine_annotation(pseudogene)
                print(f'\t\tfound pseudogenes: {len(pseudogenes)}')
            hypotheticals = [cds for cds in cdss if 'hypothetical' in cds]
            if(len(hypotheticals) > 0):
                log.debug('analyze hypotheticals')
                print(f'analyze hypothetical proteins: {len(hypotheticals)}')
                pfam_hits = feat_cds.predict_pfam(hypotheticals)
                print(f"\tdetected Pfam hits: {len(pfam_hits)} ")
                feat_cds.analyze_proteins(hypotheticals)
                print('\tcalculated proteins statistics')

            print('\trevise special cases...')
            feat_cds.revise_special_cases_annotated(genome, cdss)

        genome['features'][bc.FEATURE_CDS] = cdss

    ############################################################################
    # sORF prediction
    # - in-mem sORF extraction
    # - overlap filtering (tRNA, tmRNA, rRNA, CDS)
    # - lookup UPS matches
    # - lookup IPS matches
    # - filter sORFs w/o IPS match
    ############################################################################
    if(cfg.skip_sorf):
        print('skip sORF prediction...')
    else:
        print('extract sORF...')
        log.debug('predict sORF')
        sorfs = s_orf.extract(genome)
        print(f'\tpotential: {len(sorfs)}')

        log.debug('apply sORF overlap filter')
        sorfs, discarded_sorfs = s_orf.overlap_filter(genome, sorfs)
        print(f'\tdiscarded due to overlaps: {len(discarded_sorfs)}')

        if(len(sorfs) > 0):
            log.debug('detect spurious sORF')
            discarded_sorfs = orf.detect_spurious(sorfs)
            print(f'\tdiscarded spurious: {len(discarded_sorfs)}')
            sorfs = [sorf for sorf in sorfs if 'discarded' not in sorf]

        log.debug('lookup sORF UPS/IPS')
        sorf_upss, sorfs_not_found = ups.lookup(sorfs)
        sorf_ipss, tmp = ips.lookup(sorf_upss)
        sorfs_not_found.extend(tmp)
        print(f'\tdetected IPSs: {len(sorf_ipss)}')

        sorf_pscs_psccs = []
        if(len(sorfs_not_found) > 0):
            if(cfg.db_info['type'] == 'full'):
                log.debug('search sORF PSC')
                sorf_pscs, sorfs_not_found = s_orf.search_pscs(sorfs_not_found)
                sorf_pscs_psccs.extend(sorf_pscs)
                print(f'\tfound PSCs: {len(sorf_pscs_psccs)}')
            else:
                log.debug('search sORF PSCC')
                sorf_psccs, sorfs_not_found = s_orf.search_psccs(sorfs_not_found)
                sorf_pscs_psccs.extend(sorf_psccs)
                print(f'\tfound PSCCs: {len(sorf_pscs_psccs)}')

        print("\tlookup annotations...")
        log.debug('lookup sORF PSCs')
        sorf_pscs_psccs.extend(sorf_ipss)
        psc.lookup(sorf_pscs_psccs)  # lookup PSC info
        log.debug('lookup sORF PSCCs')
        pscc.lookup(sorf_pscs_psccs)  # lookup PSC info
        print('\tfilter and combine annotations...')
        log.debug('filter sORF by annotations')
        sorfs_filtered = s_orf.annotation_filter(sorfs)
        log.debug('combine sORF annotations')
        for feat in sorfs_filtered:
            anno.combine_annotation(feat)  # combine IPS and PSC annotations
        genome['features'][bc.FEATURE_SORF] = sorfs_filtered
        print(f'\tfiltered sORFs: {len(sorfs_filtered)}')

        if(cfg.gram != bc.GRAM_UNKNOWN  and  len(sorfs_filtered) > 0):
            sorf_aa_path = cfg.tmp_path.joinpath('sorfs.faa')
            with sorf_aa_path.open(mode='wt') as fh:
                for sorf in sorfs_filtered:
                    fh.write(f">{sorf['aa_hexdigest']}-{sorf['contig']}-{sorf['start']}\n{sorf['aa']}\n")
            sig_peptides_found = sig_peptides.search(sorfs_filtered, sorf_aa_path)
            print(f"\tsignal peptides: {len(sig_peptides_found)}")

    ############################################################################
    # gap annotation
    # - in-mem gap detection
    # - gap annotation
    ############################################################################
    if(cfg.skip_gap):
        print('skip gap annotation...')
    else:
        print('detect gaps...')
        log.debug('detect gaps')
        assembly_gaps = gaps.detect_assembly_gaps(genome)
        genome['features'][bc.FEATURE_GAP] = assembly_gaps
        print(f'\tfound: {len(assembly_gaps)}')

    ############################################################################
    # oriC/T prediction
    ############################################################################
    if(cfg.skip_ori):
        print('skip oriC/T annotation...')
    else:
        print('detect oriCs/oriVs...')
        log.debug('detect oriC/V')
        oriCs = ori.predict_oris(genome, contigs_path, bc.FEATURE_ORIC)
        genome['features'][bc.FEATURE_ORIC] = oriCs
        print(f'\tfound: {len(oriCs)}')

        print('detect oriTs...')
        log.debug('detect oriT')
        oriTs = ori.predict_oris(genome, contigs_path, bc.FEATURE_ORIT)
        genome['features'][bc.FEATURE_ORIT] = oriTs
        print(f'\tfound: {len(oriTs)}')

    ############################################################################
    # Filter overlapping features
    ############################################################################
    print('apply feature overlap filters...')
    anno.detect_feature_overlaps(genome)

    ############################################################################
    # Create annotations
    # - filter features based on precedence and overlaps
    # - sort features
    # - create locus tags for features
    ############################################################################
    print('select features and create locus tags...')
    log.debug('start feature selection and creation of locus tags')
    features_by_contig = {k['id']: [] for k in genome['contigs']}
    feature_id = 1
    feature_id_prefix = bu.create_locus_tag_prefix(contigs, length=10)
    for feature_type in [
            bc.FEATURE_T_RNA,
            bc.FEATURE_TM_RNA,
            bc.FEATURE_R_RNA,
            bc.FEATURE_NC_RNA,
            bc.FEATURE_NC_RNA_REGION,
            bc.FEATURE_CRISPR,
            bc.FEATURE_CDS,
            bc.FEATURE_SORF,
            bc.FEATURE_GAP,
            bc.FEATURE_ORIC,
            bc.FEATURE_ORIV,
            bc.FEATURE_ORIT
        ]:
        feature_list = genome['features'].get(feature_type, [])
        for feature in feature_list:
            if('discarded' not in feature):
                feature['id'] = f'{feature_id_prefix}_{feature_id}'
                feature_id += 1
                contig_features = features_by_contig.get(feature['contig'])
                contig_features.append(feature)
    features = []
    for contig in genome['contigs']:
        contig_features = features_by_contig[contig['id']]
        contig_features.sort(key=lambda k: k['start'])
        features.extend(contig_features)
    log.info('selected features=%i', len(features))
    print(f'selected: {len(features)}')

    locus_tag_nr = 5
    # use user provided locus tag if not None/non-empty or generate a sequence based locus prefix
    locus_tag_prefix = cfg.locus_tag if cfg.locus_tag else bu.create_locus_tag_prefix(contigs)
    log.info('locus tag prefix=%s', locus_tag_prefix)
    for feature in features:
        locus_tag = f'{locus_tag_prefix}_{locus_tag_nr:05}'
        if(feature['type'] in [bc.FEATURE_T_RNA, bc.FEATURE_TM_RNA, bc.FEATURE_R_RNA, bc.FEATURE_NC_RNA, bc.FEATURE_CDS, bc.FEATURE_SORF]):
            feature['locus'] = locus_tag
            locus_tag_nr += 5

    ############################################################################
    # Improve annotations
    # - select CDS/sORF gene symbols based on adjacent genes
    ############################################################################
    print('improve annotations...')
    genes_with_improved_symbols = anno.select_gene_symbols([feature for feature in features if feature['type'] in [bc.FEATURE_CDS, bc.FEATURE_SORF]])
    print(f'\trevised gene symbols: {len(genes_with_improved_symbols)}')

    ############################################################################
    # Print summary
    # - genome stats
    # - annotation stats
    ############################################################################
    print('\ngenome statistics:')
    genome_stats = bu.calc_genome_stats(genome, features)
    print(f"\tGenome size: {genome['size']:,} bp")
    print(f"\tContigs/replicons: {len(genome['contigs'])}")
    print(f"\tGC: {100 * genome_stats['gc']:.1f} %")
    print(f"\tN50: {genome_stats['n50']:,}")
    print(f"\tN ratio: {100 * genome_stats['n_ratio']:.1f} %")
    print(f"\tcoding density: {100 * genome_stats['coding_ratio']:.1f} %")

    print('\nannotation summary:')
    print(f"\ttRNAs: {len([f for f in features if f['type'] == bc.FEATURE_T_RNA])}")
    print(f"\ttmRNAs: {len([f for f in features if f['type'] == bc.FEATURE_TM_RNA])}")
    print(f"\trRNAs: {len([f for f in features if f['type'] == bc.FEATURE_R_RNA])}")
    print(f"\tncRNAs: {len([f for f in features if f['type'] == bc.FEATURE_NC_RNA])}")
    print(f"\tncRNA regions: {len([f for f in features if f['type'] == bc.FEATURE_NC_RNA_REGION])}")
    print(f"\tCRISPR arrays: {len([f for f in features if f['type'] == bc.FEATURE_CRISPR])}")
    cdss = [f for f in features if f['type'] == bc.FEATURE_CDS]
    print(f"\tCDSs: {len(cdss)}")
    print(f"\t\thypotheticals: {len([cds for cds in cdss if 'hypothetical' in cds])}")
    print(f"\t\tpseudogenes: {len([cds for cds in cdss if 'pseudogene' in cds])}")
    print(f"\t\tsignal peptides: {len([cds for cds in cdss if bc.FEATURE_SIGNAL_PEPTIDE in cds])}")
    print(f"\tsORFs: {len([f for f in features if f['type'] == bc.FEATURE_SORF])}")
    print(f"\tgaps: {len([f for f in features if f['type'] == bc.FEATURE_GAP])}")
    print(f"\toriCs/oriVs: {len([f for f in features if (f['type'] == bc.FEATURE_ORIC or f['type'] == bc.FEATURE_ORIV)])}")
    print(f"\toriTs: {len([f for f in features if f['type'] == bc.FEATURE_ORIT])}")

    cfg.run_end = datetime.now()
    run_duration = (cfg.run_end - cfg.run_start).total_seconds()
    genome['run'] = {
        'start': cfg.run_start.strftime('%Y-%m-%d %H:%M:%S'),
        'end': cfg.run_end.strftime('%Y-%m-%d %H:%M:%S'),
        'duration': f'{(run_duration / 60):.2f} min'
    }

    ############################################################################
    # Write output files
    # - write comprehensive annotation results as JSON
    # - write optional output files in GFF3/GenBank/EMBL formats
    # - remove temp directory
    ############################################################################
    print(f'\nexport annotation results to: {cfg.output_path}')
    print('\thuman readable TSV...')
    tsv_path = cfg.output_path.joinpath(f'{cfg.prefix}.tsv')
    tsv.write_tsv(genome['contigs'], features_by_contig, tsv_path)

    print('\tGFF3...')
    gff3_path = cfg.output_path.joinpath(f'{cfg.prefix}.gff3')
    gff.write_gff3(genome, features_by_contig, gff3_path)

    print('\tINSDC GenBank & EMBL...')
    genbank_path = cfg.output_path.joinpath(f'{cfg.prefix}.gbff')
    embl_path = cfg.output_path.joinpath(f'{cfg.prefix}.embl')
    insdc.write_insdc(genome, features, genbank_path, embl_path)

    print('\tgenome sequences...')
    fna_path = cfg.output_path.joinpath(f'{cfg.prefix}.fna')
    fasta.export_contigs(genome['contigs'], fna_path, description=True, wrap=True)

    print('\tfeature nucleotide sequences...')
    ffn_path = cfg.output_path.joinpath(f'{cfg.prefix}.ffn')
    fasta.write_ffn(features, ffn_path)

    print('\ttranslated CDS sequences...')
    faa_path = cfg.output_path.joinpath(f'{cfg.prefix}.faa')
    fasta.write_faa(features, faa_path)

    if(cfg.skip_plot  or  cfg.meta):
        print('\tskip generation of circular genome plot...')
    else:
        print('\tcircular genome plot...')
        plot.write_plot(features, contigs, cfg.output_path)

    if(cfg.skip_cds is False):
        hypotheticals = [feat for feat in features if feat['type'] == bc.FEATURE_CDS and 'hypothetical' in feat]
        print('\thypothetical TSV...')
        tsv_path = cfg.output_path.joinpath(f'{cfg.prefix}.hypotheticals.tsv')
        tsv.write_hypotheticals_tsv(hypotheticals, tsv_path)

        print('\ttranslated hypothetical CDS sequences...')
        faa_path = cfg.output_path.joinpath(f'{cfg.prefix}.hypotheticals.faa')
        fasta.write_faa(hypotheticals, faa_path)

    print('\tmachine readable JSON...')
    json_path = cfg.output_path.joinpath(f'{cfg.prefix}.json')
    json.write_json(genome, features, json_path)

    print('\tgenome and annotation summary...')
    summary_path = cfg.output_path.joinpath(f'{cfg.prefix}.txt')
    with summary_path.open('w') as fh_out:
        fh_out.write('Sequence(s):\n')
        fh_out.write(f"Length: {genome['size']:}\n")
        fh_out.write(f"Count: {len(genome['contigs'])}\n")
        fh_out.write(f"GC: {100 * genome_stats['gc']:.1f}\n")
        fh_out.write(f"N50: {genome_stats['n50']:}\n")
        fh_out.write(f"N ratio: {100 * genome_stats['n_ratio']:.1f}\n")
        fh_out.write(f"coding density: {100 * genome_stats['coding_ratio']:.1f}\n")
        fh_out.write('\nAnnotation:\n')
        fh_out.write(f"tRNAs: {len([f for f in features if f['type'] == bc.FEATURE_T_RNA])}\n")
        fh_out.write(f"tmRNAs: {len([f for f in features if f['type'] == bc.FEATURE_TM_RNA])}\n")
        fh_out.write(f"rRNAs: {len([f for f in features if f['type'] == bc.FEATURE_R_RNA])}\n")
        fh_out.write(f"ncRNAs: {len([f for f in features if f['type'] == bc.FEATURE_NC_RNA])}\n")
        fh_out.write(f"ncRNA regions: {len([f for f in features if f['type'] == bc.FEATURE_NC_RNA_REGION])}\n")
        fh_out.write(f"CRISPR arrays: {len([f for f in features if f['type'] == bc.FEATURE_CRISPR])}\n")
        fh_out.write(f"CDSs: {len(cdss)}\n")
        fh_out.write(f"pseudogenes: {len([cds for cds in cdss if 'pseudogene' in cds])}\n")
        fh_out.write(f"hypotheticals: {len([cds for cds in cdss if 'hypothetical' in cds])}\n")
        fh_out.write(f"signal peptides: {len([cds for cds in cdss if bc.FEATURE_SIGNAL_PEPTIDE in cds])}\n")
        fh_out.write(f"sORFs: {len([f for f in features if f['type'] == bc.FEATURE_SORF])}\n")
        fh_out.write(f"gaps: {len([f for f in features if f['type'] == bc.FEATURE_GAP])}\n")
        fh_out.write(f"oriCs: {len([f for f in features if f['type'] == bc.FEATURE_ORIC])}\n")
        fh_out.write(f"oriVs: {len([f for f in features if f['type'] == bc.FEATURE_ORIV])}\n")
        fh_out.write(f"oriTs: {len([f for f in features if f['type'] == bc.FEATURE_ORIT])}\n")
        fh_out.write('\nBakta:\n')
        fh_out.write(f'Software: v{bakta.__version__}\n')
        fh_out.write(f"Database: v{cfg.db_info['major']}.{cfg.db_info['minor']}, {cfg.db_info['type']}\n")
        fh_out.write('DOI: 10.1099/mgen.0.000685\n')
        fh_out.write('URL: github.com/oschwengers/bakta\n')

    print(f'\nIf you use these results please cite Bakta: https://doi.org/{bc.BAKTA_DOI}')
    print(f'Annotation successfully finished in {int(run_duration / 60):01}:{int(run_duration % 60):02} [mm:ss].')

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Jan 30 13:26:38 2019

@author: daniel
"""

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#coding=utf8

def ask_yes_no(question):
    """
    Ask a yes or no questions.
    """
    response = None
    while response not in ('y', 'n'):
        response = input(question).lower()
    return response

ask_yes_no("Please input 'y' or 'n' for continue or abort the test:")


# -*- coding: utf-8 -*-
"""
Created on Fri Apr  5 19:05:53 2019

@author: User
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Mar 19 15:25:50 2019

@author: Joan

This program adds all the words in an Amazon review to 2 dictionaries (Good and Bad).

"""
import time
import csv
import json
import re
from collections import Counter, OrderedDict
from nltk.corpus import stopwords
import all_clean_funcs
import operator
from textblob import TextBlob
import emoji

name_of_bad = str(input('Enter file name for bad dictionary:'))
name_of_good = str(input('Enter file name for good dictionary:'))

time_start = time.time()

good = []
bad = []
c=0

with open('all_out.csv', mode = 'r', encoding = 'utf8') as infile:
    reader = csv.reader(infile)
    for rows in reader:
        print(c)
        # --- STRING --- #
        # Remove all punctuation and symbols
        no_punct = remove_symbols(rows[5])

        # Remove emojis
        emojidict = {}
        no_punct_emoji = extractemoji(no_punct, emojidict)

        # --- LIST OF WORDS ---#
        # Split string of review into list of words as elements
        #no_punct_split = no_punct.split()

        # Remove prepositions, coordinating conjunctions,
        # cardinal number, determiner, to
        grammar_cleaned, grammar_removed = remove_grammar_words(no_punct_emoji)

        # Remove stopwords
        filtered_sentence = remove_stop_words(grammar_cleaned)

        # Put words in either good or bad dicts
        if rows[6] == '1.0 out of 5 stars' or rows[6] == '2.0 out of 5 stars' or rows[6] == '3.0 out of 5 stars':
            bad.extend(filtered_sentence)
        else:
            good.extend(filtered_sentence)
        c+=1

    # Count frequencies of words in dicts
    bad_dict = dict(Counter(bad))
    good_dict = dict(Counter(good))

    # Remove words that appear in both dicts
    cle_bad_dict, cle_good_dict = clean(bad_dict, good_dict, 0.3)

    # Sort dicts by descending value
    sort_bad_dict, sort_good_dict = sort_dict_desc(cle_bad_dict, cle_good_dict)

    print('Number of pairs in bad dict:', len(sort_bad_dict))
    print('\nNumber of pairs in good dict:', len(sort_good_dict))

    print(sort_bad_dict)

save_dicts(name_of_bad, name_of_good, sort_bad_dict, sort_good_dict)

time_elapsed = time.time() - time_start

print('Time Elapsed:', time_elapsed)


"""Execution code for address processing."""
import argparse
import datetime
import logging
import os

import arcetl
from etlassist.pipeline import Job, execute_pipeline

from helper.communicate import send_email
from helper import dataset
from helper.misc import address_intid_to_uuid_map
from helper import path
from helper.value import concatenate_arguments, is_numeric, maptaxlot_separated
from helper import transform

LOG = logging.getLogger(__name__)
"""logging.Logger: Script-level logger."""

OVERRIDE_ATTRS = {
    # Outside Lane county, but necessary for Central Lane PSAP response.
    "97089 Five Rivers Rd": {
        "where_sql": """
            house_nbr = 97089
            and street_name = 'FIVE RIVERS'
            and street_type_code = 'RD'
            and city_name_abbr = 'TID'
        """,
        "overlay_kwargs": [
            {"field_name": "ambulance_district", "value": "NW"},
            {"field_name": "psap_code", "value": "LI"},
        ],
        "constant_kwargs": [{"field_name": "county_name", "value": "Lincoln"}],
    },
    # Outside Lane county, but necessary for Central Lane PSAP response.
    "26629 Bennett Blvd": {
        "where_sql": """
            house_nbr = 26629
            and street_name = 'BENNETT'
            and street_type_code = 'BLVD'
            and city_name_abbr = 'MON'
        """,
        "overlay_kwargs": [
            {"field_name": "ambulance_district", "value": "NC"},
            {"field_name": "psap_code", "value": "CL"},
        ],
        "constant_kwargs": [{"field_name": "county_name", "value": "Benton"}],
    },
}

KWARGS_ISSUES_MESSAGE = {
    "subject": "Address Publication Update Issues",
    "recipients": ["keith.oregon@gmail.com"],
    "copy_recipients": ["david@davidrenz.com"],
    "blind_copy_recipients": ["david@davidrenz.com"],
    "reply_to": "david@davidrenz.com",
    "body": """

Issues were found that prevent certain addresses from updating to
            publication datasets.

            Refer to feature class Addressing.dbo.Issues for all current
            issues.

            {}{}

    """,
    "body_format": "HTML",
}
KWARGS_NEW_LINCOLN_MESSAGE = {
    "subject": "Recently-added Addresses in Lincoln PSAP Area",
    "recipients": ["keith.oregon@gmail.com"],
    "copy_recipients": ["david@davidrenz.com"],
    "blind_copy_recipients": ["david@davidrenz.com"],
    "reply_to": "david@davidrenz.com",
    "body": """

Recently-added addresses were found in the Lincoln PSAP area.

            {}{}

        """,
    "body_format": "HTML",
}

# Helpers.

def city_state_zip(**kwargs):
    """Return "{city}, {state code}  {ZIP code}"."""
    result = "{city_name}, {state_code}".format(**kwargs)
    if kwargs["five_digit_zip_code"]:
        # RLID for some reason has two spaces between state & ZIP.
        result += "  {five_digit_zip_code}".format(**kwargs)
    return result

def concat_address_full(**kwargs):
    """Return concatenated full-address for RLID."""
    result = "{concat_address} {city_name}, {state_code}".format(**kwargs)
    if kwargs["five_digit_zip_code"]:
        result += " {five_digit_zip_code}".format(**kwargs)
    if kwargs["four_digit_zip_code"]:
        result += "-{four_digit_zip_code}".format(**kwargs)
    return result

def send_new_lincom_address_message():
    """Send message for any recently-added address in the Lincoln PSAP area."""
    keys = ["city_name", "concat_address", "geofeat_id", "initial_create_date"]
    addresses = sorted(
        addr
        for addr in arcetl.attributes.as_iters(
            dataset.SITE_ADDRESS.path("pub"),
            field_names=keys,
            dataset_where_sql="psap_code = 'LI' ",
        )
        if (datetime.datetime.now() - addr[-1]).days < 15
    )
    table_header = "
{}
".format(
        "".join("{}	".format(key) for key in keys)
    )
    row_template = "
{}	{}	{}	{}
"
    if addresses:
        LOG.warning("Found new addresses in Lincoln PSAP area: sending email.")
        table_rows = "".join(row_template.format(*addr) for addr in addresses)
        KWARGS_NEW_LINCOLN_MESSAGE["body"] = KWARGS_NEW_LINCOLN_MESSAGE["body"].format(
            table_header, table_rows
        )
        send_email(**KWARGS_NEW_LINCOLN_MESSAGE)
    else:
        LOG.info("No new addresses in Lincoln PSAP area found. Not sending email.")

def send_publication_issues_message():
    """Send message of issues that affect address publication."""
    keys = ["description", "city_name", "concat_address", "geofeat_id"]
    issues = sorted(
        arcetl.attributes.as_iters(
            dataset.ADDRESS_ISSUES.path(),
            field_names=keys,
            dataset_where_sql="update_publication = 0",
        )
    )
    table_header = "
{}
".format(
        "".join("{}	".format(key) for key in keys)
    )
    row_template = "
{}	{}	{}	{}
"
    if issues:
        LOG.warning("Found validation publication issues: sending email.")
        table_rows = "".join(row_template.format(*issue) for issue in issues)
        KWARGS_ISSUES_MESSAGE["body"] = KWARGS_ISSUES_MESSAGE["body"].format(
            table_header, table_rows
        )
        send_email(**KWARGS_ISSUES_MESSAGE)
    else:
        LOG.info("No validation publication issues found. Not sending email.")

# ETLs.

def facility_etl():
    """Run ETL for facilities.

    Currently only undertaken for other ETL purposes--not publication.
    """
    with arcetl.ArcETL("Facilities") as etl:
        etl.extract(dataset.FACILITY.path("maint"))
        etl.transform(
            arcetl.dataset.rename_field,
            field_name="geofeat_id",
            new_field_name="address_intid",
        )
        # Clean maintenance values.
        transform.clear_nonpositive(etl, field_names=["address_intid"])
        transform.clean_whitespace(
            etl, field_names=["label", "label_full", "type", "type_full"]
        )
        transform.force_lowercase(etl, field_names=["type"])
        transform.force_uppercase(etl, field_names=["label"])
        transform.add_missing_fields(etl, dataset.FACILITY, tags=["pub"])
        # Assign geometry attributes.
        coordinate_system_xy_keys = {
            2914: {"x": "x_coordinate", "y": "y_coordinate"},
            4326: {"x": "longitude", "y": "latitude"},
        }
        for spatial_reference_id, xy_key in coordinate_system_xy_keys.items():
            for axis, key in xy_key.items():
                etl.transform(
                    arcetl.attributes.update_by_geometry,
                    field_name=key,
                    spatial_reference_item=spatial_reference_id,
                    geometry_properties=["centroid", axis],
                )
        etl.transform(
            arcetl.attributes.update_by_mapping,
            field_name="address_uuid",
            mapping=address_intid_to_uuid_map,
            key_field_names=["address_intid"],
        )
        etl.load(dataset.FACILITY.path("pub"))

def site_address_etl():
    """Run ETL for site addresses."""
    with arcetl.ArcETL("Site Addresses") as etl:
        etl.extract(dataset.SITE_ADDRESS.path("maint"))
        # Clean maintenance values.
        transform.clear_nonpositive(etl, field_names=["house_nbr"])
        transform.clean_whitespace(
            etl,
            field_names=[
                "house_suffix_code",
                "pre_direction_code",
                "street_name",
                "street_type_code",
                "unit_type_code",
                "unit_id",
                "city_name",
                "landuse",
                "maptaxlot",
                "account",
            ],
        )
        transform.force_uppercase(
            etl,
            field_names=[
                "house_suffix_code",
                "pre_direction_code",
                "street_name",
                "street_type_code",
                "unit_type_code",
                "unit_id",
                "maptaxlot",
                "valid",
                "archived",
            ],
        )
        transform.clear_non_numeric_text(etl, field_names=["account"])
        etl.transform(
            arcetl.attributes.update_by_function,
            field_name="landuse",
            function=(lambda x: x if is_numeric(x) else "0"),
        )
        transform.force_yn(etl, field_names=["archived"], default="N")
        transform.force_yn(etl, field_names=["valid"], default="Y")
        transform.add_missing_fields(etl, dataset.SITE_ADDRESS, tags=["pub"])
        # Assign geometry attributes.
        coordinate_system_xy_keys = {
            2914: {"x": "x_coordinate", "y": "y_coordinate"},
            4326: {"x": "longitude", "y": "latitude"},
        }
        for spatial_reference_id, xy_key in coordinate_system_xy_keys.items():
            for axis, key in xy_key.items():
                etl.transform(
                    arcetl.attributes.update_by_geometry,
                    field_name=key,
                    spatial_reference_item=spatial_reference_id,
                    geometry_properties=["centroid", axis],
                )
        # Assign overlays.
        overlay_kwargs = [
            # City attributes.
            {
                "field_name": "geocity",
                "overlay_field_name": "inccityabbr",
                "overlay_dataset_path": dataset.INCORPORATED_CITY_LIMITS.path(),
            },
            {
                "field_name": "annexhist",
                "overlay_field_name": "annexnum",
                "overlay_dataset_path": dataset.ANNEXATION_HISTORY.path("pub"),
            },
            # Have to do overlay rather than join because some lack codes.
            {
                "field_name": "yearanx",
                "overlay_field_name": "annexyear",
                "overlay_dataset_path": dataset.ANNEXATION_HISTORY.path("pub"),
            },
            {
                "field_name": "ugb",
                "overlay_field_name": "ugbcity",
                "overlay_dataset_path": dataset.UGB.path("pub"),
            },
            # Planning & zoning attributes.
            {
                "field_name": "greenwy",
                "overlay_field_name": "greenway",
                "overlay_dataset_path": dataset.WILLAMETTE_RIVER_GREENWAY.path("pub"),
            },
            {
                "field_name": "nodaldev",
                "overlay_field_name": "nodearea",
                "overlay_dataset_path": dataset.NODAL_DEVELOPMENT_AREA.path("pub"),
            },
            {
                "field_name": "plandes_id",
                "overlay_field_name": "plandes_id",
                "overlay_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
            },
            {
                "field_name": "sprsvcbndy",
                "overlay_field_name": "is_inside",
                "overlay_dataset_path": dataset.SPRINGFIELD_HANSEN_EXTENT.path(),
            },
            # Public safety attributes.
            {
                "field_name": "ambulance_district",
                "overlay_field_name": "asacode",
                "overlay_dataset_path": dataset.AMBULANCE_SERVICE_AREA.path("pub"),
            },
            {
                "field_name": "firedist",
                "overlay_field_name": "fireprotprov",
                "overlay_dataset_path": dataset.FIRE_PROTECTION_AREA.path("pub"),
            },
            {
                "field_name": "police_beat",
                "overlay_field_name": "CAD",
                "overlay_dataset_path": os.path.join(
                    path.LCOG_GIS_PROJECTS,
                    "Public_Safety\\PSAPS\\CLPSAP\\SunGard_CAD\\Maintained_Layers",
                    "Maintained_Layers.gdb\\Fire_Law_Tow\\law_beat",
                ),
            },
            {
                "field_name": "psap_code",
                "overlay_field_name": "psap_code",
                "overlay_dataset_path": dataset.PSAP_AREA.path("pub"),
            },
            # Election attributes.
            {
                "field_name": "electionpr",
                "overlay_field_name": "precntnum",
                "overlay_dataset_path": dataset.ELECTION_PRECINCT.path("pub"),
            },
            {
                "field_name": "ccward",
                "overlay_field_name": "ward",
                "overlay_dataset_path": dataset.CITY_WARD.path(),
            },
            {
                "field_name": "clpud_subdivision",
                "overlay_field_name": "SUBDIVISIO",
                "overlay_dataset_path": os.path.join(
                    path.LCOG_GIS_PROJECTS,
                    "UtilityDistricts\\CentralLincolnPUD\\Redistricting2012",
                    "CLPUD_Subdivisions.shp",
                ),
            },
            {
                "field_name": "cocommdist",
                "overlay_field_name": "commrdist",
                "overlay_dataset_path": (
                    dataset.COUNTY_COMMISSIONER_DISTRICT.path("pub")
                ),
            },
            {
                "field_name": "epud",
                "overlay_field_name": "boardid",
                "overlay_dataset_path": dataset.EPUD_SUBDISTRICT.path("pub"),
            },
            {
                "field_name": "hwpud_subdivision",
                "overlay_field_name": "BoardZone",
                "overlay_dataset_path": os.path.join(
                    path.LCOG_GIS_PROJECTS,
                    "UtilityDistricts\\HecetaWaterPUD\\NewBoardSubzones",
                    "HecetaData.gdb",
                    "ScenarioB",
                ),
            },
            {
                "field_name": "lcczone",
                "overlay_field_name": "lccbrdzone",
                "overlay_dataset_path": dataset.LCC_BOARD_ZONE.path("pub"),
            },
            {
                "field_name": "senatedist",
                "overlay_field_name": "sendist",
                "overlay_dataset_path": dataset.STATE_SENATOR_DISTRICT.path("pub"),
            },
            {
                "field_name": "strepdist",
                "overlay_field_name": "repdist",
                "overlay_dataset_path": (
                    dataset.STATE_REPRESENTATIVE_DISTRICT.path("pub")
                ),
            },
            {
                "field_name": "swcd",
                "overlay_field_name": "swcdist",
                "overlay_dataset_path": (
                    dataset.SOIL_WATER_CONSERVATION_DISTRICT.path("pub")
                ),
            },
            {
                "field_name": "swcdzone",
                "overlay_field_name": "swczone",
                "overlay_dataset_path": (
                    dataset.SOIL_WATER_CONSERVATION_DISTRICT.path("pub")
                ),
            },
            # Education attributes.
            {
                "field_name": "schooldist",
                "overlay_field_name": "district",
                "overlay_dataset_path": dataset.SCHOOL_DISTRICT.path("pub"),
            },
            {
                "field_name": "elem",
                "overlay_field_name": "attend",
                "overlay_dataset_path": dataset.ELEMENTARY_SCHOOL_AREA.path("pub"),
            },
            {
                "field_name": "middle",
                "overlay_field_name": "attend",
                "overlay_dataset_path": dataset.MIDDLE_SCHOOL_AREA.path("pub"),
            },
            {
                "field_name": "high",
                "overlay_field_name": "attend",
                "overlay_dataset_path": dataset.HIGH_SCHOOL_AREA.path("pub"),
            },
            # Transportation attributes.
            {
                "field_name": "ltddist",
                "overlay_field_name": "LTD",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\ltd\\2012 LTD Boundary.shp"
                ),
            },
            {
                "field_name": "ltdridesrc",
                "overlay_field_name": "LTD",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\ltd\\2015 RideSource Boundary.shp"
                ),
            },
            {
                "field_name": "cats",
                "overlay_field_name": "CATSBNDY",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\eug\\catsbndy.shp"
                ),
            },
            {
                "field_name": "trans_analysis_zone",
                "overlay_field_name": "TAZ_NUM",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "transport\\MTAZ16.shp"
                ),
            },
            # Natural attributes.
            {
                "field_name": "firmnumber",
                "overlay_field_name": "firm_pan",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\FIRMPanel"
                ),
            },
            {
                "field_name": "soilkey",
                "overlay_field_name": "mukey",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\soils\\Soils.gdb\\Soil"
                ),
            },
            {
                "field_name": "wetland",
                "overlay_field_name": "WET_TYPE",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\eug\\Wetland\\wetlands.shp"
                ),
            },
            # Census attributes.
            {
                "field_name": "ctract",
                "overlay_field_name": "TRACT",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "federal\\census\\lane\\2010",
                    "lc_census2010.gdb\\lc_tracts2010",
                ),
            },
            {
                "field_name": "blockgr",
                "overlay_field_name": "BlockGroup",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "federal\\census\\lane\\2010",
                    "lc_census2010.gdb\\lc_blockgroups2010",
                ),
            },
            # Other district attributes.
            {
                "field_name": "neighbor",
                "overlay_field_name": "NEIBORHD",
                "overlay_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "boundary\\districts\\eug",
                    "Boundary.gdb\\EugNeighborhoods",
                ),
            },
        ]
        for kwargs in overlay_kwargs:
            etl.transform(
                arcetl.attributes.update_by_overlay,
                overlay_central_coincident=True,
                **kwargs
            )
        # Override overlays for special cases.
        for override in OVERRIDE_ATTRS:
            for kwargs in OVERRIDE_ATTRS[override].get("overlay_kwargs", []):
                etl.transform(
                    arcetl.attributes.update_by_value,
                    dataset_where_sql=OVERRIDE_ATTRS[override].get("where_sql"),
                    **kwargs
                )
        # Clean overlay values.
        transform.clean_whitespace(
            etl, field_names=["police_beat", "wetland", "ctract", "blockgr", "neighbor"]
        )
        transform.force_uppercase(etl, field_names=["cats", "ltddist", "ltdridesrc"])
        # Set default overlay values where missing.
        transform.force_yn(
            etl,
            field_names=["greenwy", "sprsvcbndy", "cats", "ltddist", "ltdridesrc"],
            default="N",
        )
        # Remove invalid overlay values.
        transform.clear_nonpositive(etl, field_names=["ctract", "blockgr"])
        etl.transform(
            arcetl.attributes.update_by_function,
            field_name="neighbor",
            function=(lambda x: x if x and int(x) != 99 else None),
        )
        # Assign joinable field values after overlays.
        join_kwargs = [
            # Core attributes.
            {
                "field_name": "pre_direction",
                "join_field_name": "description",
                "join_dataset_path": dataset.STREET_DIRECTION.path(),
                "on_field_pairs": [("pre_direction_code", "code")],
            },
            {
                "field_name": "street_type",
                "join_field_name": "description",
                "join_dataset_path": dataset.STREET_TYPE.path(),
                "on_field_pairs": [("street_type_code", "code")],
            },
            {
                "field_name": "unit_type",
                "join_field_name": "description",
                "join_dataset_path": dataset.UNIT_TYPE.path(),
                "on_field_pairs": [("unit_type_code", "code")],
            },
            {
                "field_name": "city_name_abbr",
                "join_field_name": "CityNameAbbr",
                "join_dataset_path": dataset.CITY.path(),
                "on_field_pairs": [("city_name", "CityName")],
            },
            # Extended attributes.
            {
                "field_name": "five_digit_zip_code",
                "join_field_name": "zip_code",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            # Any addresses not assigned zip from USPS gets an overlay zip.
            {
                "field_name": "five_digit_zip_code",
                "dataset_where_sql": "five_digit_zip_code is null",
                "join_field_name": "zip_code_overlay",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "four_digit_zip_code",
                "join_field_name": "plus_four_code",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_delivery_point_code",
                "join_field_name": "delivery_point_code",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "postal_carrier_route",
                "join_field_name": "carrier_route",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_is_cmra",
                "join_field_name": "is_cmra",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_is_vacant",
                "join_field_name": "is_vacant",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "usps_has_mail_service",
                "join_field_name": "has_mail_service",
                "join_dataset_path": dataset.ADDRESS_POSTAL_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            {
                "field_name": "landuse_desc",
                "join_field_name": "ludesc",
                "join_dataset_path": dataset.LAND_USE_CODES_DETAILED.path("pub"),
                "on_field_pairs": [("landuse", "landusec")],
            },
            {
                "field_name": "usecode",
                "join_field_name": "usecode",
                "join_dataset_path": dataset.LAND_USE_CODES_DETAILED.path("pub"),
                "on_field_pairs": [("landuse", "landusec")],
            },
            {
                "field_name": "usedesc",
                "join_field_name": "ucname",
                "join_dataset_path": dataset.LAND_USE_CODES_USE_CODES.path("pub"),
                "on_field_pairs": [("usecode", "usecode")],
            },
            # A&T attributes.
            {
                "field_name": "tca",
                "join_field_name": "tax_code_overlay",
                "join_dataset_path": dataset.ADDRESS_ASSESS_TAX_INFO.path(),
                "on_field_pairs": [("geofeat_id", "geofeat_id")],
            },
            # City attributes.
            {
                "field_name": "geocity_name",
                "join_field_name": "inccityname",
                "join_dataset_path": dataset.INCORPORATED_CITY_LIMITS.path(),
                "on_field_pairs": [("geocity", "inccityabbr")],
            },
            {
                "field_name": "ugb_city_name",
                "join_field_name": "ugbcityname",
                "join_dataset_path": dataset.UGB.path("pub"),
                "on_field_pairs": [("ugb", "ugbcity")],
            },
            # Planning & zoning attributes.
            {
                "field_name": "nodaldev_name",
                "join_field_name": "nodename",
                "join_dataset_path": dataset.NODAL_DEVELOPMENT_AREA.path("pub"),
                "on_field_pairs": [("nodaldev", "nodearea")],
            },
            {
                "field_name": "plandesjuris",
                "join_field_name": "planjuris",
                "join_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
                "on_field_pairs": [("plandes_id", "plandes_id")],
            },
            {
                "field_name": "plandes",
                "join_field_name": "plandes",
                "join_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
                "on_field_pairs": [("plandes_id", "plandes_id")],
            },
            {
                "field_name": "plandesdesc",
                "join_field_name": "plandesnam",
                "join_dataset_path": dataset.PLAN_DESIGNATION.path("pub"),
                "on_field_pairs": [("plandes_id", "plandes_id")],
            },
            # Public safety attributes.
            {
                "field_name": "ambulance_service_area",
                "join_field_name": "asa",
                "join_dataset_path": dataset.AMBULANCE_SERVICE_AREA.path("pub"),
                "on_field_pairs": [("ambulance_district", "asacode")],
            },
            {
                "field_name": "ambulance_service_provider",
                "join_field_name": "provider",
                "join_dataset_path": dataset.AMBULANCE_SERVICE_AREA.path("pub"),
                "on_field_pairs": [("ambulance_district", "asacode")],
            },
            {
                "field_name": "fire_protection_provider",
                "join_field_name": "fpprovname",
                "join_dataset_path": dataset.FIRE_PROTECTION_AREA.path("pub"),
                "on_field_pairs": [("firedist", "fireprotprov")],
            },
            {
                "field_name": "psap_name",
                "join_field_name": "psap_name",
                "join_dataset_path": dataset.PSAP_AREA.path("pub"),
                "on_field_pairs": [("psap_code", "psap_code")],
            },
            {
                "field_name": "emergency_service_number",
                "join_field_name": "emergency_service_number",
                "join_dataset_path": dataset.EMERGENCY_SERVICE_NUMBER.path(),
                "on_field_pairs": [
                    # City used as proxy for police.
                    ("geocity", "city_limits"),
                    ("ambulance_district", "asa_code"),
                    ("firedist", "fire_district"),
                    ("psap_code", "psap_code")
                ],
            },
            {
                "field_name": "emergency_service_number",
                "join_field_name": "emergency_service_number",
                "join_dataset_path": dataset.EMERGENCY_SERVICE_NUMBER.path(),
                "on_field_pairs": [
                    # City used as proxy for police.
                    ("geocity", "city_limits"),
                    ("ambulance_district", "asa_code"),
                    ("firedist", "fire_district"),
                ],
                "dataset_where_sql": "emergency_service_number is null",
            },
            # Election attributes.
            {
                "field_name": "city_councilor",
                "join_field_name": "councilor",
                "join_dataset_path": dataset.CITY_WARD.path(),
                "on_field_pairs": [("ccward", "ward")],
            },
            {
                "field_name": "cocommdist_name",
                "join_field_name": "cmdistname",
                "join_dataset_path": dataset.COUNTY_COMMISSIONER_DISTRICT.path("pub"),
                "on_field_pairs": [("cocommdist", "commrdist")],
            },
            {
                "field_name": "county_commissioner",
                "join_field_name": "commrname",
                "join_dataset_path": dataset.COUNTY_COMMISSIONER_DISTRICT.path("pub"),
                "on_field_pairs": [("cocommdist", "commrdist")],
            },
            {
                "field_name": "eweb_commissioner_name",
                "join_field_name": "eweb_commissioner_name",
                "join_dataset_path": dataset.EWEB_COMMISSIONER.path("pub"),
                "on_field_pairs": [("ccward", "city_council_ward")],
            },
            {
                "field_name": "state_representative",
                "join_field_name": "repname",
                "join_dataset_path": dataset.STATE_REPRESENTATIVE_DISTRICT.path("pub"),
                "on_field_pairs": [("strepdist", "repdist")],
            },
            {
                "field_name": "state_senator",
                "join_field_name": "senname",
                "join_dataset_path": dataset.STATE_SENATOR_DISTRICT.path("pub"),
                "on_field_pairs": [("senatedist", "sendist")],
            },
            # Education attributes.
            {
                "field_name": "schooldist_name",
                "join_field_name": "names",
                "join_dataset_path": dataset.SCHOOL_DISTRICT.path("pub"),
                "on_field_pairs": [("schooldist", "district")],
            },
            {
                "field_name": "elem_name",
                "join_field_name": "elem_school",
                "join_dataset_path": dataset.ELEMENTARY_SCHOOL_AREA.path("pub"),
                "on_field_pairs": [("elem", "attend")],
            },
            {
                "field_name": "middle_name",
                "join_field_name": "middle_school",
                "join_dataset_path": dataset.MIDDLE_SCHOOL_AREA.path("pub"),
                "on_field_pairs": [("middle", "attend")],
            },
            {
                "field_name": "high_name",
                "join_field_name": "high_school",
                "join_dataset_path": dataset.HIGH_SCHOOL_AREA.path("pub"),
                "on_field_pairs": [("high", "attend")],
            },
            # Natural attributes.
            {
                "field_name": "firmprinted",
                "join_field_name": "panel_printed",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\FIRMPanel"
                ),
                "on_field_pairs": [("firmnumber", "firm_pan")],
            },
            {
                "field_name": "firm_community_id",
                "join_field_name": "com_nfo_id",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\CommunityInfo"
                ),
                "on_field_pairs": [("geocity", "community_code")],
            },
            {
                "field_name": "firm_community_post_firm_date",
                "join_field_name": "in_frm_dat",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\flood\\Flood.gdb\\CommunityInfo"
                ),
                "on_field_pairs": [("geocity", "community_code")],
            },
            {
                "field_name": "soiltype",
                "join_field_name": "musym",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA, "natural\\soils\\Soils.gdb\\MUAggAtt"
                ),
                "on_field_pairs": [("soilkey", "mukey")],
            },
            # Other district attributes.
            {
                "field_name": "neighborhood_name",
                "join_field_name": "NAME",
                "join_dataset_path": os.path.join(
                    path.REGIONAL_DATA,
                    "boundary\\districts\\eug\\Boundary.gdb\\EugNeighborhoods",
                ),
                "on_field_pairs": [("neighbor", "NEIBORHD")],
            },
        ]
        for kwargs in join_kwargs:
            etl.transform(arcetl.attributes.update_by_joined_value, **kwargs)
        # Clean join values.
        transform.clean_whitespace(etl, field_names=["neighborhood_name"])
        # Remove Metro Plan designations, per City of Eugene request.
        transform.clear_all_values(
            etl,
            field_names=["plandes", "plandesdesc"],
            dataset_where_sql="plandesjuris = 'MTP'",
        )
        # Remove +4 ZIP where initial ZIP is missing.
        transform.clear_all_values(
            etl,
            field_names=["four_digit_zip_code"],
            dataset_where_sql="five_digit_zip_code is null",
        )
        # Assign constants.
        constant_kwargs = [
            {"field_name": "state_code", "value": "OR"},
            {"field_name": "state_name", "value": "Oregon"},
            {"field_name": "county_name", "value": "Lane"},
        ]
        for kwargs in constant_kwargs:
            etl.transform(arcetl.attributes.update_by_value, **kwargs)
        # Override constants for special cases.
        for override in OVERRIDE_ATTRS:
            for kwargs in OVERRIDE_ATTRS[override].get("constant_kwargs", []):
                etl.transform(
                    arcetl.attributes.update_by_value,
                    dataset_where_sql=OVERRIDE_ATTRS[override].get("where_sql"),
                    **kwargs
                )
        # Build values from functions.
        function_kwargs = [
            {
                "field_name": "street_name_full",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "pre_direction_code",
                    "street_name",
                    "street_type_code",
                ],
            },
            {
                "field_name": "city_state_zip",
                "function": city_state_zip,
                "kwarg_field_names": ["city_name", "state_code", "five_digit_zip_code"],
            },
            {
                "field_name": "concat_address_no_unit",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "house_nbr",
                    "house_suffix_code",
                    "street_name_full",
                ],
            },
            {
                "field_name": "concat_address",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "concat_address_no_unit",
                    "unit_type_code",
                    "unit_id",
                ],
            },
            {
                "field_name": "concat_address_no_direction",
                "function": concatenate_arguments,
                "arg_field_names": [
                    "house_nbr",
                    "house_suffix_code",
                    "street_name",
                    "street_type_code",
                    "unit_type_code",
                    "unit_id",
                ],
            },
            {
                "field_name": "concat_address_full",
                "function": concat_address_full,
                "kwarg_field_names": [
                    "concat_address",
                    "city_name",
                    "state_code",
                    "five_digit_zip_code",
                    "four_digit_zip_code",
                ],
            },
            {
                "field_name": "mapnumber",
                "function": (lambda x: x[:8] if x else None),
                "arg_field_names": ["maptaxlot"],
            },
            {
                "field_name": "taxlot",
                "function": (lambda x: x[-5:] if x else None),
                "arg_field_names": ["maptaxlot"],
            },
            {
                "field_name": "maptaxlot_hyphen",
                "function": maptaxlot_separated,
                "arg_field_names": ["maptaxlot"],
            },
        ]
        for kwargs in function_kwargs:
            etl.transform(
                arcetl.attributes.update_by_function, field_as_first_arg=False, **kwargs
            )
        # Take care of addresses flagged not to update in publication.
        ids = {}
        id_set_kwargs = {
            "in_publication": {"dataset_path": dataset.SITE_ADDRESS.path("pub")},
            "in_transform": {"dataset_path": etl.transform_path},
            "no_update": {
                "dataset_path": dataset.ADDRESS_ISSUES.path(),
                "dataset_where_sql": "update_publication = 0",
            },
        }
        for key, kwargs in id_set_kwargs.items():
            ids[key] = set(
                _id
                for _id, in arcetl.attributes.as_iters(
                    field_names="site_address_gfid", **kwargs
                )
            )
        ids["rollback"] = ids["no_update"] & ids["in_transform"] & ids["in_publication"]
        ids["hold"] = ids["no_update"] & (ids["in_transform"] - ids["in_publication"])
        rollback_features = [
            feat
            for feat in arcetl.attributes.as_dicts(dataset.SITE_ADDRESS.path("pub"))
            if feat["site_address_gfid"] in ids["rollback"]
        ]
        # Strip OIDs (not part of update).
        for feat in rollback_features:
            del feat["oid@"]
        if rollback_features:
            etl.transform(
                arcetl.features.update_from_dicts,
                update_features=rollback_features,
                id_field_names="site_address_gfid",
                field_names=rollback_features[0].keys(),
                delete_missing_features=False,
            )
        etl.transform(
            arcetl.features.delete_by_id,
            delete_ids=ids["hold"],
            id_field_names="site_address_gfid",
        )
        LOG.info("%s addresses held from publication", len(ids["hold"]))
        LOG.info("%s addresses rolled-back from publication", len(ids["rollback"]))
        if any([ids["hold"], ids["rollback"]]):
            send_publication_issues_message()
        etl.load(dataset.SITE_ADDRESS.path("pub"))
    send_new_lincom_address_message()

# Jobs.

WEEKLY_JOB = Job("Address_Datasets_Weekly", etls=[site_address_etl, facility_etl])

# Execution.

def main():
    """Script execution code."""
    args = argparse.ArgumentParser()
    args.add_argument("pipelines", nargs="*", help="Pipeline(s) to run")
    available_names = {key for key in list(globals()) if not key.startswith("__")}
    pipeline_names = args.parse_args().pipelines
    if pipeline_names and available_names.issuperset(pipeline_names):
        pipelines = [globals()[arg] for arg in args.parse_args().pipelines]
        for pipeline in pipelines:
            execute_pipeline(pipeline)
    else:
        console = logging.StreamHandler()
        LOG.addHandler(console)
        if not pipeline_names:
            LOG.error("No pipeline arguments.")
        for arg in pipeline_names:
            if arg not in available_names:
                LOG.error("`%s` not available in exec.", arg)
        LOG.error(
            "Available objects in exec: %s",
            ", ".join("`{}`".format(name) for name in sorted(available_names)),
        )

if __name__ == "__main__":
    main()


from botocore.exceptions import BotoCoreError, ClientError
from fastapi.requests import Request
from fastapi.responses import JSONResponse
from sqlalchemy.exc import DBAPIError, SQLAlchemyError

from .crud import database_logger

class NoSuchBucket(Exception):
    """Custom exception that will be raised if minio bucket doesn't exist."""

    def __init__(self, message: str):
        self.message = message

def sqlalchemy_exception_handler(
    request: Request, error: SQLAlchemyError
) -> JSONResponse:
    database_logger.error(error)
    return JSONResponse(
        status_code=500,
        content={"detail": f"Error: SQLAlchemy error ({error})"},
    )

def dbapi_exception_handler(
    request: Request, error: DBAPIError
) -> JSONResponse:
    database_logger.error(error.__cause__)
    return JSONResponse(
        status_code=500,
        content={"detail": f"Error: DBAPI error ({error.__cause__})"},
    )

def no_such_bucket_error_handler(
    request: Request, exc: NoSuchBucket
) -> JSONResponse:
    return JSONResponse(
        status_code=404,
        content={"detail": f"Error: {exc.message}"},
    )

def botocore_error_handler(
    request: Request, exc: BotoCoreError
) -> JSONResponse:
    return JSONResponse(
        status_code=500,
        content={"detail": f"Error: {exc}"},
    )

def minio_client_error_handler(
    request: Request, exc: ClientError
) -> JSONResponse:
    return JSONResponse(
        status_code=500,
        content={"detail": f"Error: {exc}"},
    )


# Мурованый Андрей.КНИТ16-А
# Условие: Вывести в файл свою анкету.
import os
flag = True
while flag:
    os.chdir('D:/NewFold/')
    os.chdir('D:/NewFold/MyFold')
    print('Мы находимся в следующуей директории:', os.getcwd())
    print('Ну что, начнем анкетирование? :)')
    while True:
        name = input('Как Вас зовут ?\n'
                     '>>>')
        if not (name.isalpha() and name.istitle()):
            print('Нет, введите имя')
            flag = False
        else:
            year = input('Сколько Вам лет?\n'
                         '>>>')
            if int(year) not in range(10, 101):
                print('Вы допустили ошибку!')
                flag = False
            else:
                phone = input('Ваш номер телефона?\n'
                              '>>>')
                if phone[0:4] != '+380':
                    print('Ошибка ввода номера!')
                    flag = False
                else:
                    email = input('Ваш электронный адрес?\n'
                                  '>>>')
                    if not ('@' and '.') in email:
                        print('Вы допустили ошибку!')
                        flag = False
                    else:
                        with open('MyAnket.txt', 'w') as g:
                            g.write('Имя: ' + name + '\n')
                            g.write('Возраст: ' + year + '\n')
                            g.write('Моб. номер: ' + phone + '\n')
                            g.write('Электроннный адрес: ' + email + '\n')
                        with open('MyAnket.txt', 'r') as f:
                            line = f.readlines()
                        for i in line:
                            print(i, sep='')
        if input('Press Enter to continue...') != '':
            flag = False
            break


"""
Let's say we have an input of N=9

the output should look as follow;

    .#######.
    ##*****##
    #**###**#
    #*##*##*#
    #*#***#*#
    #*##*##*#
    #**###**#
    ##*****##
    .#######.
"""
N = 9
grid = [["." for i in range(N)] for j in range(N)]

def print_grille(grid):
   for i in range(N):
       # print(end=" ")
       print()
       for j in range(N):
           print(grid[i][j], end="")

def draw(grid):
    shapes = [".", "#", "*"]
    row_start = 0
    col_start = 0
    col_len = N
    row_len = N
    tour = 1

    for i in range(N-1):

        for droite in range(row_start, col_len):
            grid[row_start][droite] = shapes[tour]

        for gauche in range(col_len-1, col_start, -1):
            grid[row_len-1][gauche] = shapes[tour]

        for bas in range(row_start+1, row_len):
            grid[bas][col_len-1] = shapes[tour]

        for haut in range(row_len-1, row_start, -1):
            grid[haut][col_start] = shapes[tour]

        if i == 0:
            grid[row_start][col_start] = shapes[0]
            grid[row_start][col_len-1] = shapes[0]
            grid[row_len-1][col_start] = shapes[0]
            grid[row_len-1][col_len-1] = shapes[0]
        else:
            grid[row_start][col_start] = shapes[3-tour]
            grid[row_start][col_len-1] = shapes[3-tour]
            grid[row_len-1][col_start] = shapes[3-tour]
            grid[row_len-1][col_len-1] = shapes[3-tour]

        row_start += 1
        col_start += 1
        row_len -= 1
        col_len -= 1
        tour = 3 - tour

    return grid

print_grille(draw(grid))

Meilleur Dev de France Septembre 2017 (Session 2)/dodecagone.py


import sys
import os
import logging
import math
from functools import wraps
from flask import request, jsonify
from os import path
from datetime import datetime
from pathlib import Path
import uuid
import logging
from .model_file_manager import load_for_shared_mem, get_model_file_info, load_model, store_model
from .db_manager import get_ratings_for_user

class ModelManager:
    models_cache = {}
    model_directory_path = "lkweb/models"

    def __init__(self, app):
        self.app = app

    def get_model_info(self, algo):
        model_file_dir_path = f'{ModelManager.model_directory_path}/{algo}.bpk'
        creation_date = None
        updated_date = None
        size = 0
        if path.exists(model_file_dir_path):
            logging.info("Getting model information")
            creation_date = datetime.utcfromtimestamp(path.getctime(model_file_dir_path))
            updated_date = datetime.utcfromtimestamp(path.getmtime(model_file_dir_path))
            size = path.getsize(model_file_dir_path) / 1000
            # dates are in UTC format and size is in KB
            return jsonify({'model': {
                "creation_date": creation_date.strftime('%Y-%m-%d %H:%M:%S'),
                "updated_date": updated_date.strftime('%Y-%m-%d %H:%M:%S'),
                "size": size
            }})
        else:
            logging.info("No model found for the algorithm")
            return jsonify({'model': {}})

    def upload_model(self, algo):
        keys = list(request.files.keys())
        if len(keys) > 0:
            file = request.files.get(keys[0], None)

            logging.info("Create folder if not exists")
            Path(ModelManager.model_directory_path).mkdir(exist_ok=True)

            logging.info("Save the model with a temporary file name")
            temp_model_name = f'{algo}_{uuid.uuid1()}.bpk'
            temp_file_name = Path(f'{ModelManager.model_directory_path}/{temp_model_name}')
            file.save(temp_file_name)

            logging.info("Save the model with sharing mode")
            temp_model = load_model(temp_model_name)
            store_model(temp_model, temp_model_name, True)

            logging.info("Rename the temp file name to the actual algorithm name")
            file_name = Path(f'{ModelManager.model_directory_path}/{algo}.bpk')
            os.rename(temp_file_name, file_name)

            return jsonify({'result': 200})
        else:
            return jsonify({'result': 'No file sent'})

    def get_db_ratings(self, user_id):
        ratings = get_ratings_for_user(user_id, self.app.config)
        if len(ratings) > 0:
            ratings.set_index('item', inplace=True)
            return ratings.iloc[:, 0]
        else:
            return None

    def get_param_value(self, key, *args):
        """First try to get the value from values (query string or form data), if not, from json data. """
        value = request.values.get(key, '')
        if value == '':
            value = request.json.get(key, '')
        return value

    def get_recs_params(self, *args):
        user_id = self.get_param_value('user_id')
        return user_id, self.get_param_value('num_recs'), self.get_db_ratings(user_id)

    def get_preds_params(self):
        user_id = int(self.get_param_value('user_id'))
        items = list(map(int, self.get_param_value('items').split(',')))
        return user_id, items, self.get_db_ratings(user_id)

    def get_recommendations_from_model(self, model, *args):
        user = None
        try:
            user, nr_recs, ratings = args[0][0], args[0][1], args[0][2]
            results = []
            df_recs = model.recommend(int(user), int(nr_recs), ratings=ratings)
            for index, row in df_recs.iterrows():
                results.append({'item': row['item'], 'score': row['score']})
            return results
        except:
            logging.error(f"Unexpected recs error for user: {user}. Error: {sys.exc_info()[0]}")
            raise

    def get_predictions_from_model(self, model, *args):
        user, items = None, None
        try:
            user, items, ratings = args[0][0], args[0][1], args[0][2]
            results = []
            df_preds = model.predict_for_user(user, items, ratings)
            for index, value in df_preds.iteritems():
                if not math.isnan(value):
                    results.append({'item': index, 'score': value})
            return results
        except:
            logging.error(f"Unexpected preds error for user: {user}, with items: {items}. Error: {sys.exc_info()[0]}")
            raise

    def get_worst_predictions_from_model(self, model, *args):
        user, items = None, None
        try:
            user, items, ratings = args[0][0], args[0][1], args[0][2]
            results = []
            df_preds = model.predict_for_user(user, items, ratings)
            for index, value in df_preds.iteritems():
                if not math.isnan(value):
                    results.append({'item': index, 'score': value})
            results = sorted(results, key = lambda i: i['score'])
            return results
        except:
            logging.error(f"Unexpected preds error for user: {user}, with items: {items}. Error: {sys.exc_info()[0]}")
            raise

    @classmethod
    def get_recommendations_from_default(model, *args):
        return ModelManager.get_recommendations_from_model(model, *args)

    def get_model(self, algo):
        if algo not in ModelManager.models_cache:
            logging.info(f'Adding algo {algo} to cache')
            model = load_for_shared_mem(algo)
            info = get_model_file_info(algo)
            ModelManager.models_cache[algo] = { "model": model, "info": info }
            return model
        else:
            # check the modified datetime of the model to see if we need to reload it.
            logging.info(f'Reading algo {algo} from cache')
            model_data = ModelManager.models_cache[algo]
            info = get_model_file_info(algo)
            if model_data['info']['updated_date'] != info['updated_date']:
                logging.info(f'Updating algo {algo} in cache')
                model = load_for_shared_mem(algo)
                ModelManager.models_cache[algo] = { "model": model, "info": info }
            return ModelManager.models_cache[algo]['model']

    def execute_model(self, algo, base_class, get_data_func, get_params_func):
        logging.info("Loading the model")
        model = self.get_model(algo)
        if isinstance(model, base_class):
            logging.info("Executing the model")
            return get_data_func(model, get_params_func())
        # else:
        #     return abort(404, description="Model not found")

    def model_method(self, name, base_class, get_data_func, get_params_func, default_algo=False, methods=['GET', 'POST']):
        def deco_wrap(func):
            @wraps(func)
            def wrapper(algo=None):
                if default_algo:
                    algo = self.app.config["DEFAULT_ALGORITHM"]
                return func(self.execute_model(algo, base_class, get_data_func, get_params_func))

            if default_algo:
                route = f'/{name}'
            else:
                route = f'/algorithms/<algo>/{name}'
            return self.app.route(route, methods=methods)(wrapper)
        return deco_wrap


from Volley.genetics import SuperGene
import matplotlib.pyplot as plt

class FitnessFunction:
    def __init__(self, function, super_genes: list[SuperGene], start_kill=1, kill_percent_reduce = .1):
        self.start_kill = start_kill
        self.function = function
        self.super_genes = super_genes
        self.kill_percent_reduce = kill_percent_reduce
        self.epoch = 0
        self.history = []

    def fit(self):
        data = [super_gene.current_gene for super_gene in self.super_genes]
        fitness = self.function(data)
        self.history.append(fitness)
        self.epoch += 1
        for super_gene in self.super_genes:
            super_gene.current_gene.fit(fitness)
        if self.epoch >= self.start_kill:
            for super_gene in self.super_genes:
                super_gene.check(self.kill_percent_reduce)
        return fitness

    def shuffle_all(self):
        for gene in self.super_genes:
            gene.shuffle()

    def plot(self):
        plt.plot(self.history)
        plt.show()


""" Monte-Carlo integration """
import numpy as np
import matplotlib.pyplot as plt

generator = np.random.default_rng()

def f1(x):
    return np.exp(-x) * np.sin(x)

def f2(x):
    return np.sin(x**2) / np.sqrt(1 + x**4)

def integrate_1D(n, function, a, b):
    """ Basic Monte-Carlo 1D integration """
    result = 0
    for _ in range(n):
        x = (b - a) * generator.random() + a
        result += function(x)

    result = (b - a) / n * result
    return result

def integrate_1D_array(n, function, a, b):
    """ Monte Carlo integration via array.
        It requieres more memory, but it is much faster. """
    x = (b - a) * generator.random(n) + a
    result = (b - a) / n * sum(function(x))
    return result

""" Specific function to integrate just the I3 integral """
def integral3(n):
    hits = 0
    result = 0

    for _ in range(n):
        x, y, z, w = generator.random(4)

        if (x - 0.5)**2 + (y - 0.5)**2 + (z - 0.5)**2 + (w - 0.5)**2 <= 0.25:
            hits += 1
            result += np.sin(np.sqrt(np.log(x + y + z + w + 2)))

    volume = hits / n
    result = result / hits * volume

    #print(f"I3 = {result} (number of hits: {hits}, volume of the integration region: {volume})")

    return result

""" More abstract way - works for any integration regions and any function """
def is_inside_region3(X):
    """ Returns true if the point X is inside the integration region """
    x, y, z, w = X
    return (x - 0.5)**2 + (y - 0.5)**2 + (z - 0.5)**2 + (w - 0.5)**2 <= 0.25

def f3(X):
    x, y, z, w = X
    return np.sin(np.sqrt(np.log(x + y + z + w + 2)))

def integral_ND(n, function, is_inside_region, dimension, box_min=0, box_max=1):
    """ Monte-Carlo integration of a general-dimensional function.
        box_min and box_max specify the position of the hypercube from which the program draws the random numbers
    """
    hits = 0
    result = 0

    for _ in range(n):
        X = generator.uniform(box_min, box_max, dimension)

        if is_inside_region(X):
            hits += 1
            result += function(X)

    volume = hits / n
    result = result / hits * volume

    print(f"I3 = {result} (number of hits: {hits}, volume of the integration region: {volume})")

    return result

if __name__ == "__main__":
    print("I1 = {}".format(integrate_1D_array(1000000, f1, 0, 2 * np.pi)))
    print("I2 = {}".format(integrate_1D_array(1000000, f2, 0, np.sqrt(10 * np.pi))))
    print("I3 = {}".format(integral3(1000000)))
    print("I3 = {}".format(integral_ND(1000000, f3, is_inside_region3, dimension=4)))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

############################################################################
#
#  @author Akim Glushkov
#  @group 1453
#
# Необходимо написать скрипт, обрабатывающий лог файл Nginx
# выводящий список IP адресов, с которых производились запросы.
# Адреса из общей подсети \24 необходимо группировать при выводе(напр. 10.40.0.4 и 10.40.0.231 относятся к одной подсети).
#
# Базовый язык - Python 2.7 или Python 3.5. Требуется использовать библиотеку re для RegEx.
#
#  Задание на языке python требуется сдать до 23:59 (UTC+7) 8 Ноября.
############################################################################

import re
import sys
import os.path
import os

def fch(str, ch, posl):
    pos = 0
    for c in str:
        if ch == c:
            if posl == 0:
                return pos
            else:
                posl = posl - 1
        pos = pos + 1

    raise RuntimeError("Не вернули позицию символа подстроки, т.к. дошли до конца строки")

def prepare(ips):
    result = list()
    prefix = None
    for ip in sorted(ips):
        try:
            if prefix is None:
                prefix = ip[0:fch(ip, '.', 2)]
                result.append(ip)
                continue
            elif prefix == ip[0:fch(ip, '.', 2)]:
                del result[-1]
                result.append(prefix + ".0/24")
            else:
                prefix = ip[0:fch(ip, '.', 2)]
                result.append(ip)

        except RuntimeError:
            pass

    return result

if __name__ == '__main__':

    if len(sys.argv) < 2:
        print("Отсутствуют необходимые параметры (имя файла). Например, \"access.log\"")
        exit(-1)
    arg = sys.argv[1]
    if not os.path.isfile(arg):
        print("Входящий параметр '" + arg + "' не является файлом")
        exit(-1)

    try:
        file = open(arg)
        ips = set()
        c = 0
        for line in file:
            c = c + 1
            ip_str = re.match("([0-9.]*)", line).group(1)
            ips.add(ip_str)

        # print("Считано " + str(len(ips)) + " из " + str(c) + " различный адресов")
        print('\n'.join(prepare(ips)))

    except IOError:
        print("Ошибка чтения файла. '" + arg + "' не доступен для чтения или произошла ошибка при чтении")
        exit(-1)


import datetime
import os
import time
import cv2
import ffmpeg
import numpy as np

# filepath = "vtest.avi"
# cap = cv2.VideoCapture(filepath)
# Webカメラを使うときはこちら
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)

avg = None

_, frame = cap.read()

# 動体検知参考資料
#  https://qiita.com/KMiura95/items/4eed79a7da6b3dafa96d
# ffmpeg参考資料
#  https://qiita.com/mitayuki6/items/73943628b625e0b2ab30

contours_area_threshold = 90000
t_delta = datetime.timedelta(hours=9)
JST = datetime.timezone(t_delta, 'JST')
video_path = None
initial_not_detect_time = None
video_file_divide_sec = 10
while True:
    # 1フレームずつ取得する。
    ret, frame = cap.read()
    if not ret:
        break

    # グレースケールに変換
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    # 比較用のフレームを取得する
    if avg is None:
        avg = gray.copy().astype("float")
        continue

    # 現在のフレームと移動平均との差を計算
    cv2.accumulateWeighted(gray, avg, 0.6)
    frameDelta = cv2.absdiff(gray, cv2.convertScaleAbs(avg))

    # デルタ画像を閾値処理を行う
    thresh = cv2.threshold(frameDelta, 3, 255, cv2.THRESH_BINARY)[1]
    # 画像の閾値に輪郭線を入れる
    contours, hierarchy = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    contours2 = list(filter(lambda x: cv2.contourArea(x) >= contours_area_threshold, contours))
    # contoured_frame = cv2.drawContours(frame, contours2, -1, (0, 255, 0), 3)
    if len(contours2) == 0 and video_path is not None:
        now = time.time()
        if initial_not_detect_time is None:
            initial_not_detect_time = now
        elif now - initial_not_detect_time > video_file_divide_sec:
            video_path = None
            initial_not_detect_time = None
            try:
                process.stdin.close()
                process.wait()
            except:
                pass
    elif len(contours2) > 0 and video_path is None:
        initial_not_detect_time = None
        now = datetime.datetime.now(JST)
        video_path = now.strftime('img/%Y/%m/%d/%H-%M-%S.%f.avi')
        video_dir = os.path.dirname(video_path)
        if not os.path.exists(video_dir):
            os.makedirs(video_dir)
        height, width = frame.shape[:2]
        # 可変フレームレートで動画ファイルとして書き出すためffmpegプロセス
        process = (
            ffmpeg.input(
                'pipe:', format='rawvideo', pix_fmt='bgr24',
                s='{}x{}'.format(width, height), use_wallclock_as_timestamps=1).output(
                    video_path, vsync='vfr', r=24.0).overwrite_output().run_async(pipe_stdin=True)
        )
        print("Detected, path %s" % video_path)
    elif len(contours2) > 0:
        initial_not_detect_time = None

    # ビデオ保存
    if video_path is not None:
        process.stdin.write(frame.astype(np.uint8).tobytes())

process.stdin.close()
process.wait()

cap.release()


def mark_taken(grid, hstart, hend, vstart, vend, id):
    for a_row_idx in range(vstart, vstart+vend):
        for a_column_idx in range(hstart, hstart+hend):
            if grid[a_row_idx][a_column_idx] == 0:
                grid[a_row_idx][a_column_idx] = id
            else:
                grid[a_row_idx][a_column_idx] = 'x'

def part_1(input='day3.txt'):
    with open(input, 'r') as f:
        lines = f.readlines()
        grid = [[0]* 1000 for i in range(1000)]
        for a_line in lines:
            a_line = a_line.replace('#', '').replace(' ', '')
            id = int(a_line.split('@')[0])
            lr = a_line.split(',')
            left = int(lr[0].split('@')[1])
            top = int(lr[1].split(':')[0])
            wh = a_line.split(':')[1]
            width = int(wh.split('x')[0])
            height = int(wh.split('x')[1])
            mark_taken(grid, left, width, top, height, id)
        return sum(1 for a_row in grid for a_column in a_row if a_column == 'x')

# print(part_1())

def mark_taken2(grid, hstart, width, vstart, height, id):
    borked = set()
    for a_row_idx in range(vstart, vstart + height):
        for a_column_idx in range(hstart, hstart + width):
            if grid[a_row_idx][a_column_idx] != 0:
                borked.add(grid[a_row_idx][a_column_idx])
                borked.add(id)
            grid[a_row_idx][a_column_idx] = id
    return borked

# test_grid = [[1, 1, 1, 1],
#              [5, 2, 3, 3],
#              [6, 4, 8, 2],
#              [7, 8, 9, 3]]
#
# print(mark_taken2(test_grid, 1, 2, 1, 1, 16), test_grid)

def part_2(input='day3.txt'):
    with open(input, 'r') as f:
        lines = f.readlines()
        grid = [[0]* 1000 for i in range(1000)]
        rekt_claims = set()
        claims = set()
        count = 0
        for a_line in lines:
            a_line = a_line.replace('#', '').replace(' ', '')
            id = int(a_line.split('@')[0])
            claims.add(id)
            lr = a_line.split(',')
            left = int(lr[0].split('@')[1])
            top = int(lr[1].split(':')[0])
            wh = a_line.split(':')[1]
            width = int(wh.split('x')[0])
            height = int(wh.split('x')[1])
            claimed = mark_taken2(grid, left, width, top, height, id)
            # print(claimed)
            # count +=1
            # if count == 100:
            #     exit(1)
            for a_claim_num in claimed:
                rekt_claims.add(a_claim_num)
        return claims.difference(rekt_claims)

print(part_2())


#Importa a biblioteca
import folium

#Cria o mapa base
map = folium.Map(location=[-20.807382,-49.360453], zoom_start = 10)

#Multiplos marcadores
for coordinates in [[-20.800385, -49.367797],[-20.800154, -49.368665]]:
    folium.Marker(location=coordinates, icon=folium.Icon(color = 'green')).add_to(map)

#Salva o mapa
map.save("mapa.html")


from typing import List

class Solution:
    def flipAndInvertImage(self, image: List[List[int]]) -> List[List[int]]:
        row_size = len(image)
        col_size = len(image[0])

        for r in range(row_size):
            for c in range(col_size // 2):
                image[r][c], image[r][col_size - c - 1] = image[r][col_size - c - 1], image[r][c]

        for r in range(row_size):
            for c in range(col_size):
                image[r][c] = 0 if image[r][c] else 1

        return image

def main():
    sol = Solution()
    print(sol.flipAndInvertImage([[1,1,0],[1,0,1],[0,0,0]]))
    print(sol.flipAndInvertImage([[1,1,0,0],[1,0,0,1],[0,1,1,1],[1,0,1,0]]))

if __name__ == '__main__':
    main()


from fastapi import APIRouter, HTTPException, Depends
from utils.mongo_utils import register_user
from utils.enums import Register
from validations.register_validator import UserRegistrationRequest
from utils.rate_limiter import register_limiter, init_rate_limiter

router = APIRouter()

@router.post("/user-register", status_code=201, dependencies=[Depends(register_limiter)])
async def register_user_endpoint(user_data: UserRegistrationRequest):
    user_data_dict = user_data.dict()
    registration_status = register_user(user_data_dict)
    if(registration_status == "USERNAME_EXISTS"):
        raise HTTPException(status_code=400, detail="Username already exists")
    elif(registration_status == "EMAIL_EXISTS"):
        raise HTTPException(status_code=400, detail="Email already exists")
    elif(registration_status == "REGISTRATION_SUCCESS"):
        return {"message": "User registration successful"}
    else:
        raise HTTPException(status_code=500, detail="Someting went wrong")

    # if not register_user(user_data_dict):
    #     raise HTTPException(status_code=400, detail="Username or email already exists")

    # return {"message": "User registration successful"}