#! /usr/bin/env python3
#
# Monitor a directory recursively for updates using inotifywait.
# The directory structure below src is expected to be
#   glider/file
#
# which will then be copied to
#   tgt/glider/to-glider/file
#
# A dated copy of the file will be archived into
#   archive/glider/file.YYYYMMDD.HHMMSS
#
# This is designed to run as a service.
#
# March-2019, Pat Welch, pat@mousebrains.com

import sys
import time
import os.path
import argparse
import logging
import logging.handlers
import threading
import queue
import subprocess
import getpass
import socket
import re
import io
import smtplib
from email.message import EmailMessage

class MyBaseThread(threading.Thread):
  def __init__(self, name, logger, qExcept):
    threading.Thread.__init__(self, daemon=True)
    self.name = name
    self.logger = logger
    self.qExcept = qExcept

  def run(self): # Called on thread start. Will pass any exception to qExcept so program can exit
    q = self.qExcept
    try:
      self.runMain()
    except Exception as e:
      self.logger.exception('Unexpected exception')
      q.put(e)

  def runMain(self): # Should be overridden by main thread
    raise Exception('runMain not overridden by derived class')

class Monitor(MyBaseThread):
  def __init__(self, src, fn, logger, qExcept, doit):
    MyBaseThread.__init__(self, 'Monitor', logger, qExcept)
    self.src = src
    self.fn = fn
    self.doit = doit

  def runMain(self): # Called on thread start
    src = self.src
    if not os.path.isdir(src):
      raise Exception('Source "' + src + '" is not a directory')

    self.logger.info('Starting %s: %s', self.src, str(self.fn))

    files = set()
    for fn in self.fn:
      files.add(fn)
    self.logger.info(files)

    cmd = ['/usr/bin/inotifywait', \
		'--monitor', \
		'--quiet', \
		'--recursive', \
		'--event', 'close_write', \
		'--event', 'attrib', \
		'--event', 'moved_to', \
		'--format', '%w%f', \
                self.src]

    self.logger.debug('cmd %s', ' '.join(cmd))

    with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
      while True:
        line = proc.stdout.readline()
        if line == '' and proc.poll() is not None:
          raise Exception('inotifywait failed, ' + str(cmd))
        if not line:
          self.logger.warn('empty line')
          continue
        line = str(line.strip(), 'utf-8')
        fn = os.path.basename(line) # Filename to consider
        if fn not in files: # Not a file of interest
          self.logger.warn('%s is not in files list', line)
          continue
        prefix = os.path.dirname(line)
        if os.path.dirname(prefix) != self.src:
          self.logger.warn('Invalid directory structure for %s', line)
          continue
        gld = os.path.basename(prefix)
        self.logger.debug('file %s glider %s file %s', line, gld, fn)
        self.doit.put((time.time(), line, gld, fn))

class LineIO: # Iterable class of lines
  def __init__(self, lines):
    self.index = 0 # Next index to read
    self.lines = []
    if lines is None:
      return
    for line in lines:
      index = line.find('#') # Find a comment character
      if index >= 0:
         line = line[0:index] # Drop off trailing comment
      line = line.strip()
      if line: # non-empty line, so append
        self.lines.append(line)

  def __iter__(self):
    return self;

  def __next__(self):
    if self.index < len(self.lines): # something to return
      self.index = self.index + 1
      return self.lines[self.index - 1]
    raise StopIteration

class Doit(MyBaseThread):
  def __init__(self, logger, qExcept, args):
    MyBaseThread.__init__(self, 'Doit', logger, qExcept)
    self.archiveDir = args.archive
    self.tgt = args.tgt
    self.qDryrun = args.dryrun
    self.delay = args.delay
    self.notify = args.notify
    self.mailFrom = args.mailFrom
    self.mailHost = args.mailHost
    self.queue = queue.Queue()
    self.bargPattern = re.compile("^\w+(\w+)$", flags=re.ASCII)

  def put(self, a):
    self.queue.put(a)

  def runMain(self): # Called on thread start
    q = self.queue
    self.logger.info('Starting')
    history = {}
    delay = self.delay
    if delay <= 0:
      delay = 0.1

    while True:
      (t, src, gld, fn) = q.get()
      q.task_done()
      if src in history and (t <= (history[src] + delay)): # Ignore quick updates
        continue
      history[src] = t
      time.sleep(delay) # Wait a bit for everything to settle
      (lines, afn) = self.archiveFile(t, src, gld, fn)
      if lines is None: # Unable to open the file
        continue
      if not self.qSane(lines, afn):
        continue
      self.logger.debug('afn=%s\n', afn)
      self.syncit(afn, os.path.join(self.tgt, gld, 'to-glider', fn))
      if args.notify:
        self.notifier(lines.lines, fn, gld)

  def archiveFile(self, t, src, gld, fn):
    adir = os.path.join(self.archiveDir, gld)
    if not os.path.isdir(adir):
      os.makedirs(adir)
    afn = os.path.join(adir, str(time.strftime("%Y%m%d.%H%M%S.")) + os.path.basename(fn))
    try:
      ifp = open(src, 'r')
    except FileNotFoundError as e:
      self.logger.error("Error opening %s for reading", src)
      return (None, afn)
    lines = []
    with open(afn, 'w') as ofp, ifp:
      lines = ifp.readlines()
      ofp.writelines(lines)
    os.remove(src)
    self.logger.info('Copied and removed %s to %s', src, afn)
    return (LineIO(lines), afn)

  def qSane(self, lines, fn):
    for line in lines:
      if line == "behavior_name=goto_list": # Must be first non-blank line
        return self.qSaneGoto(lines, fn)
      self.logger.error("Unsupported behavior file '%s' %s", fn, line)
      return False
    self.logger.error('No behavior_name line found in %s', fn)
    return False

  def qSaneGoto(self, lines, fn):
    bargs = None
    waypts = -1
    for line in lines:
      if line == "<start:b_arg>":
        bargs = self.qSaneBArg(lines, fn)
        if not bargs:
          return False
      elif line == "<start:waypoints>":
        waypts = self.qSaneWaypoints(lines, fn)
        if waypts is None:
          return False
      else:
        self.logger.error('Unsupported line "%s" in %s', line, fn)
        return False
    if 'num_waypoints(nodim)' not in bargs:
      self.logger.error('num_waypoints(nodim) not in %s', fn);
      return False
    if bargs['num_waypoints(nodim)'] != waypts:
      self.logger.error('num_waypoints(nodim) (%s) and number of waypoints (%s) do not match in %s', bargs['num_waypoints(nodim)'], waypts, fn)
      return False
    if bargs['num_waypoints(nodim)'] > 8:
      self.logger.error('num_waypoints(nodim) (%s) must be 8 or less in %s', bargs['num_waypoints(nodim)'], fn)
      return False
    if bargs['num_waypoints(nodim)'] < 1:
      self.logger.error('num_waypoints(nodim) (%s) must be >= 1 in %s', bargs['num_waypoints(nodim)'], fn)
      return False
    if bargs['num_waypoints(nodim)'] > 8:
      self.logger.error('num_waypoints(nodim) (%s) must be <= 8 in %s', bargs['num_waypoints(nodim)'], fn)
      return False
    return True

  def qSaneBArg(self, lines, fn):
    info = {}
    for line in lines:
      if line == "<end:b_arg>":
         return info
      fields = line.split()
      if len(fields) != 3:
        self.logger.error('Line "%s" does not have three fields', line)
        return False
      if fields[0] != 'b_arg:':
        self.logger.error('Line "%s" does not begin with b_arg:', line)
        return {}
      if re.match(self.bargPattern, fields[1]):
        self.logger.error('Variable in "%s" is not formated properly', line)
        return {}
      try:
        val = float(fields[2])
        info[fields[1]] = val
      except ValueError:
        self.logger.error('Third field in "%s" is not numeric', line)
        return {}

    self.logger.error('No <end:b_arg> found for %s', fn)
    return {}

  def qSaneWaypoints(self, lines, fn):
    count = 0
    for line in lines:
      if not line:
        continue
      if line == "<end:waypoints>":
        if count > 0:
          return count
        self.logger.error('No waypoints found in %s', fn)
        return None
      fields = line.split()
      if len(fields) != 2:
        self.logger.error('Line "%s" in %s does not have two fields', line, fn)
        return {}
      try:
        lon = float(fields[0])
        lat = float(fields[1])
        if lon < -18000 or lon > 18000:
          self.logger.error('Longitude(%s) is out of range in %s', line, fn)
          return None
        if lat < -9000 or lat > 9000:
          self.logger.error('Latitude(%s) is out of range in %s', line, fn)
          return None
        if (abs(lon) % 100) >= 60:
          self.logger.error('Longitude(%s) minutes are out of range in %s', line, fn)
          return None
        if (abs(lat) % 100) >= 60:
          self.logger.error('Latitude(%s) minutes are out of range in %s', line, fn)
          return None
        count += 1
      except ValueError:
        self.logger.error('Unable to convert "%s" into a pair of numbers in %s', line, fn)
        return None

    self.logger.error('No <end:waypoints> found in %s', fn)
    return None

  def syncit(self, src, tgt):
    cmd = ['/usr/bin/rsync', '--quiet', '--archive', '--delay-updates', src, tgt]
    self.logger.debug(' '.join(cmd))
    if self.qDryrun:
      self.logger.info('Not syncing %s to %s due to dryrun', src, tgt)
      return True
    p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    if p.returncode == 0: # Success
      self.logger.info('Synced %s to %s', src, tgt)
      return True
    self.logger.error('Error syncing %s to %s\n, %s', src, tgt, str(p.stdout, 'utf-8'))
    return False

  def notifier(self, lines, fn, gld):
    msg = EmailMessage()
    msg['Subject'] = "{} {}".format(gld, fn)
    msg['From'] = self.mailFrom
    msg['To'] = self.notify
    a = []
    state = 0
    for line in lines:
      if state == 0 and line == '<start:waypoints>':
        state = 1
      elif state == 1:
        if line == '<end:waypoints>':
          state == 0
        else:
          a.append(line)
    msg.set_content('\n'.join(a))
    s = smtplib.SMTP(self.mailHost)
    s.send_message(msg)
    s.quit()

parser = argparse.ArgumentParser()
parser.add_argument('--src', help='Source directory to monitor', required=True, action='append')
parser.add_argument('--tgt', help='Where to send the files', required=True)
parser.add_argument('--archive', help='Where to make a dated backup copy of the file at', required=True)
parser.add_argument('--fn', help='Filename to send from src to tgt when it changes', required=True, action='append')
parser.add_argument('--log', help='Log filename, if not specified use the console')
parser.add_argument('--maxlogsize', default=10000000, type=int, \
			help='How large to let the log file grow to')
parser.add_argument('--backupcount', default=5, type=int, help='How many logfiles to keep')
parser.add_argument('--delay', default=10, type=int, help='How many logfiles to keep')
parser.add_argument('--dryrun', help='Do not actually copy the files', action='store_true')
parser.add_argument('--verbosity', help='Logging verbosity level ERROR|WARN|INFO|DEBUG', default='ERROR')
parser.add_argument('--notify', help='Who to mail notifications to', action='append')
parser.add_argument('--mailTo', help='Who to mail errors to', action='append')
parser.add_argument('--mailHost', help='SMTP hostname', default='localhost')
parser.add_argument('--mailFrom', help='Who mail is coming from',
                    default=getpass.getuser() + '@' + socket.gethostname())
parser.add_argument('--mailSubject', help='Subject of mail',
                    default="ERROR: " + socket.gethostname())
args = parser.parse_args()

logger = logging.getLogger(__name__)
logger.setLevel(args.verbosity)

if args.log is None:
  ch = logging.StreamHandler()
else:
  name = os.path.dirname(args.log)
  if not os.path.isdir(name):
    os.makedirs(name)
  ch = logging.handlers.RotatingFileHandler(args.log, \
                 maxBytes=args.maxlogsize, backupCount=args.backupcount)
ch.setLevel(args.verbosity)
ch.setFormatter(logging.Formatter('%(asctime)s: %(threadName)s:%(levelname)s - %(message)s'))
logger.addHandler(ch)

if args.mailTo: # Where to send error messages to
  ch = logging.handlers.SMTPHandler(args.mailHost, args.mailFrom, args.mailTo, args.mailSubject)
  ch.setLevel(logging.ERROR)
  ch.setFormatter(logging.Formatter('%(threadName)s:%(levelname)s - %(message)s'))
  logger.addHandler(ch)

try:
  logger.debug('Args %s', args)

  excQueue = queue.Queue() # Where thread exceptions are sent

  thrDoit = Doit(logger, excQueue, args)
  thrDoit.start()

  threads = []

  for src in args.src:
    thr = Monitor(src, args.fn, logger, excQueue, thrDoit)
    thr.start()
    threads.append(thr)

  e = excQueue.get() # Wait for an exception from a thread
  excQueue.task_done()
  raise(e)
except Exception as e:
  logger.exception('Thread exception')

sys.exit(1)


import sys

if len(sys.argv) < 3:
    print("error")
    exit()

try:
    for x in sys.argv[1:]:
        int(x)
except ValueError:
    print("error")
    exit()

def tri_selec(array):
    minus = 0
    new_array = []
    for y in range(0, len(array) - 1):

        minus = y
        for x in range(y + 1, len(array) ):
            if int(array[x]) < int(array[minus]):
                minus = x
        if minus != y:
            array[y], array[minus] = array[minus], array[y]
    return " ".join(array)

arg1 = sys.argv[1:]

print(tri_selec(arg1))


"""
define model for gp
"""
# from threading import Thread
# from queue import Queue
from multiprocessing import Pool
from random import random, randint
from math import floor
import operator
from autoprover.gp.gene import Gene
from autoprover.gp.rule import GeneRule
from autoprover.gp.action import GeneAction
from autoprover.gp.trigger import GeneTrigger
from autoprover.gp.restriction import Restriction

#TODO fox too many instant
class GPModel:
    """
    gp model
    """
    #TODO fix too many args
    def __init__(self, args=None, populationSize=None, maxGeneration=None,
                 mutateRate=None, eliteRate=None, crossRate=None,
                 crossType=None, verifyNum=None, proof=None, tactics=None,
                 limit_hyp=None, limit_goal=None):
        self.population_size = populationSize or args.populationSize
        self.max_generation = maxGeneration or args.maxGeneration
        self.mutate_rate = mutateRate or args.mutateRate
        self.elite_rate = eliteRate or args.eliteRate
        self.cross_rate = crossRate or args.crossRate
        self.cross_type = crossType or args.crossType
        self.verify_num = verifyNum or args.verifyNum
        self.limit_hyp = limit_hyp or args.limit_hyp
        self.limit_goal = limit_goal or args.limit_goal
        self.debug = args.debug
        self.proof = proof
        self.tactics = tactics
        self.population = None
        self.current_generation = 1
        self.proofs = []
        self.rules = []
        self.init_population(self.population_size)
        self.pre_process()

    def show_prop(self):
        """
        display property for model
        """
        print(self.population_size)
        print(self.max_generation)
        print(self.mutate_rate)
        print(self.elite_rate)
        print(self.cross_rate)
        print(self.cross_type)
        print(self.verify_num)
        print(self.proof)

    def init_population(self, size):
        """
        create population by size
        """
        print("Initializing population.")
        self.population = []
        for _ in range(size):
            self.population.append(Gene(self.tactics))

    def pre_process(self):
        """
        run before start
        """
        self.current_generation = 1
        self.update_fitness_for_population()
        self.fitness_sharing()
        self.sort_population()
        self.update_tactic_usage()
        self.check_proof()

    def is_proved(self):
        """
        check population has a proof
        """
        return len(self.proofs) > 0

    def start(self, gen=None):
        """
        run the model
        """
        if gen is None:
            # if gen is not set
            local_gen_limit = self.max_generation + 1
        else:
            local_gen_limit = gen

        if self.current_generation > self.max_generation:
            return

        for _ in range(local_gen_limit):
            print("Generation No.{0}".format(self.current_generation))
            if self.debug:
                self.sort_population()
                for index in range(0, 30):
                    self.population[index].print_lastest()
            self.crossover()
            self.update_fitness_for_population()
            self.apply_rules()
            self.fitness_sharing()
            self.next_generation()
            if self.current_generation > self.max_generation:
                break

        # self.printGeneByIndex(0, True)

    def fitness_sharing(self):
        """Use fitness sharing to re-evaluate fitness"""
        for gene in self.population:
            gene.fitness = gene.raw_fitness
        return
        def dist(gene1, gene2):
            """Return distence between two gene"""
            return abs(len(gene1.goal) - len(gene2.goal))
        for gene in self.population:
            raw_fitnesses = [e.raw_fitness for e in self.population if dist(e, gene) <= 5]
            gene.fitness = sum(raw_fitnesses) / len(raw_fitnesses)

    def next_generation(self):
        """
        next generation
        """
        print("Avg. fitness\tAvg. length")
        print("{0:.8f}\t{1}".format(self.average_fitness(),
                                    self.average_length_of_gene()))
        self.current_generation += 1
        self.sort_population()
        self.update_tactic_usage()
        self.check_proof()

    def check_proof(self):
        """Check if there is a proof in population
        """
        for gene in self.population:
            if gene.is_proof:
                print(gene.chromosome)
                for state in gene.coq_states:
                    print(state)
                self.proofs.append(Gene(chromosome=gene.valid_tactics))

    def update_fitness_for_population(self):
        """
        return individual if theorem is proved, o.w return None
        """
        def wrapper(func, *args, **kwargs):
            """func wrapper"""
            return func, args, kwargs

        with Pool(processes=4) as pool:
            for gene in self.population:
                func, args, kargs = wrapper(gene.update_fitness_for_proof,
                                            self.proof, self.limit_hyp,
                                            self.limit_goal)
                pool.apply_async(func(*args, **kargs))

    def apply_rules(self):
        """Perform action by rules"""
        if len(self.rules) == 0:
            return
        for gene in self.population:
            for rule in self.rules:
                if rule.type == "gene":
                    rule.check_and_apply(gene)

    def crossover(self):
        """
        the crossover operation for gp
        """
        self.sort_population()
        elite_amount = round(self.elite_rate * self.population_size)
        # preserve from the top
        new_population = [ele for ele in self.population if ele.ttl > 0]
        for individual in new_population:
            if individual.ttl > 0:
                individual.ttl -= 1
        new_population += self.population[:elite_amount]

        while len(new_population) < self.population_size:
            # newGene = self.crossBelowCrossRate()
            new_gene, new_gene2 = self.cross_on_arb_seq()
            if random() <= self.mutate_rate:
                self.mutate_append(new_gene)
            new_population.append(new_gene)
            if len(new_population) == self.population_size:
                break

            if random() <= self.mutate_rate:
                self.mutate_append(new_gene2)
            new_population.append(new_gene2)
        self.population = new_population

    def sort_population(self):
        """
        sort population by length and fitness
        """
        self.population.sort(key=lambda x: x.fitness, reverse=True)

    def update_tactic_usage(self):
        """update tactic statistic usage"""
        usage = {e: 0 for e in self.tactics.all_tactics}
        count = 0
        for gene in self.population:
            for tactic in gene.chromosome:
                count += 1
                try:
                    usage[tactic] += 1
                except KeyError:
                    usage[tactic] = 1
        for tactic in usage:
            usage[tactic] = usage[tactic]/count
        self.proof.tactics.usage = usage

    def cross_below_cross_rate(self):
        """
        select two parent by cross rate, crossover on random point
        """
        p1_index = randint(0, floor(self.population_size * self.cross_rate)-1)
        p2_index = randint(0, floor(self.population_size * self.cross_rate)-1)
        gene_of_p1 = self.population[p1_index]
        gene_of_p2 = self.population[p2_index]
        cross_point = randint(0, int_min(len(gene_of_p1), len(gene_of_p2))-1)
        new_chromosome = []
        new_chromosome += gene_of_p1.chromosome[:cross_point]
        new_chromosome += gene_of_p2.chromosome[cross_point:]
        if (self.tactics.is_unrepeatable(new_chromosome[cross_point])
                and cross_point < len(new_chromosome)-1):
            if new_chromosome[cross_point] == new_chromosome[cross_point+1]:
                del new_chromosome[cross_point]
        return Gene(chromosome=new_chromosome)

    def cross_on_arb_seq(self, slmax=6):
        """
        select two parent by cross_rate, crossover by some seqence
        """
        p1_index = randint(0, floor(self.population_size * self.cross_rate)-1)
        p2_index = randint(0, floor(self.population_size * self.cross_rate)-1)
        gene_of_p1 = self.population[p1_index]
        gene_of_p2 = self.population[p2_index]

        p1_begin = myrandint(0, len(gene_of_p1)-1)
        p1_end = p1_begin + myrandint(1, int_min(slmax, len(gene_of_p1)-p1_begin))
        p2_begin = myrandint(0, len(gene_of_p2)-1)
        p2_end = p2_begin + myrandint(1, int_min(slmax, len(gene_of_p2)-p2_begin))
        new_chromosome = []
        new_chromosome += gene_of_p1.chromosome[:p1_begin]
        new_chromosome += gene_of_p2.chromosome[p2_begin:p2_end]
        new_chromosome += gene_of_p1.chromosome[p1_end:]
        new_chromosome2 = []
        new_chromosome2 += gene_of_p2.chromosome[:p2_begin]
        new_chromosome2 += gene_of_p1.chromosome[p1_begin:p1_end]
        new_chromosome2 += gene_of_p2.chromosome[p2_end:]
        self.remove_repeatable(new_chromosome)
        self.remove_repeatable(new_chromosome2)
        return Gene(chromosome=new_chromosome), Gene(chromosome=new_chromosome2)

    def remove_repeatable(self, chromosome):
        """
        remove repeatable tactic
        """
        tactic_set = set()

        for index, tactic in enumerate(chromosome):
            if self.tactics.is_unrepeatable(tactic):
                if tactic in tactic_set:
                    del chromosome[index]
                else:
                    tactic_set.add(tactic)

    def mutate(self, gene):
        """
        the mutate operation
        """
        if len(gene) == 1:
            gene.chromosome[0] = self.tactics.mutate_select()
        else:
            index = randint(len(gene)//2, len(gene)-1)
            gene.chromosome[index] = self.tactics.mutate_select()

    def mutate_append(self, gene):
        """append a tactic  to chromosome"""
        gene.chromosome.append(self.tactics.mutate_select())

    def average_fitness(self):
        """Calculate the average fitness for population.

        Returns:
            double: avg. fitness
        """
        return sum([e.fitness for e in self.population]) / len(self.population)

    def average_length_of_gene(self):
        """Calculate the average fitness for population.

        Returns:
            double: avg. fitness
        """
        return sum([len(e) for e in self.population]) / len(self.population)

    def edit(self, index, data=None):
        """Human involved modification of some gene of the population
        """
        if self.current_generation > self.max_generation:
            return
        print("Edit Gene {} now.".format(index))
        gene = self.population[index]
        gene.modification(data=data)
        gene.update_fitness_for_proof(self.proof, self.limit_hyp,
                self.limit_goal)
        if gene.is_proof:
            self.proofs.append(Gene(chromosome=gene.valid_tactics))
        return

    def show_proofs(self):
        """Show proofs found
        """
        if self.proofs:
            for gene in self.proofs:
                print(gene.format_output(self.proof))
        else:
            print("There is not proof for now.")

    def list(self, argv):
        """List property of some individual.

        Args:
            argv(list): sub command of list function.
        """
        def get_interval(interval):
            """Get begin and end of interval
            """
            interval_list = interval.split("-")
            if len(interval_list) == 1:
                return (int(interval_list[0]), int(interval_list[0])+1)
            else:
                return (int(interval_list[0]), int(interval_list[1])+1)
        if not argv or not argv[0]:
            return

        (begin, end) = get_interval(argv[0])
        if len(argv) == 1:
            for index, gene in enumerate(self.population[begin:end]):
                print("{0}: {1:.8f}".format(index, gene.fitness))
                gene.print_progress()
        elif argv[1] == "fitness":
            for index, gene in enumerate(self.population[begin:end]):
                print("{0}: {1:.8f}".format(index, gene.fitness))
        elif argv[1] == "chromosome":
            for index, gene in enumerate(self.population[begin:end]):
                print("{0}: {1}".format(index, gene.chromosome))
        elif argv[1] == "ttl":
            for index, gene in enumerate(self.population[begin:end]):
                print("{0}: {1}".format(index, gene.ttl))

    def read_rule_from_file(self, file_name):
        """Read a rule from a JSON file"""
        try:
            self.rules.append(GeneRule(file_name=file_name, proof=self.proof))
        except FileNotFoundError:
            return

    def delete_rule(self, index):
        """delete a rule from rule_list"""
        del self.rules[index]

    def remove_tactic(self):
        """remove a tactic in tactic_set and population"""
        tactic_removed = input("Enter a tactic to be removed: ")
        self.proof.tactics.remove(tactic_removed)
        for gene in self.population:
            gene.chromosome = [e for e in gene.chromosome if e != tactic_removed]

    def set_attributes(self, argv):
        """Set attributes of population
        """
        if argv[0] == "population" or argv[0] == "pop":
            if argv[1] == "ttl":
                self.population[int(argv[2])].ttl = int(argv[3])

    def defrag(self, index_list):
        """Defrag some gene"""
        for index in index_list:
            self.population[index].defrag(self.proof)

    def print_stats(self):
        """print tactic usage"""
        sorted_stats = sorted(self.proof.tactics.usage.items(),
                              key=operator.itemgetter(1), reverse=True)
        for tactic, usage in sorted_stats:
            print("{0}: {1:.4f}%".format(tactic, usage*100))

def myrandint(begin, end):
    """
    randint warrper for begin == end
    """
    if begin == end:
        return begin
    else:
        return randint(begin, end)

def int_max(int_a, int_b):
    """
    max(a, b)
    """
    if int_a > int_b:
        return int_a
    else:
        return int_b

def int_min(int_a, int_b):
    """
    min(a, b)
    """
    if int_a < int_b:
        return int_a
    else:
        return int_b


# https://www.codewars.com/kata/52b757663a95b11b3d00062d
# Write a function toWeirdCase (weirdcase in Ruby) that accepts a string, and returns the same string with all even
# indexed characters in each word upper cased, and all odd indexed characters in each word lower cased.
# The indexing just explained is zero based, so the zero-ith index is even, therefore that character should be upper
# cased and you need to start over for each word.
#
# The passed in string will only consist of alphabetical characters and spaces(' '). Spaces will only be present if
# there are multiple words. Words will be separated by a single space(' ').
#
# Examples:
# to_weird_case('String'); # => returns 'StRiNg'
# to_weird_case('Weird string case') # => returns 'WeIrD StRiNg CaSe'

def to_weird_case(words: str) -> str:
    word_list = words.split()
    for i, word in enumerate(word_list):
        word_list[i] = ''.join([char.upper() if j % 2 == 0 else char.lower() for j, char in enumerate(word)])
    return ' '.join(word_list)

print(to_weird_case('This'))
print(to_weird_case('is'))
print(to_weird_case('THIs iS a TEST'))


# Q03(a)

# Initialise variables
alphabet = ["a","b","c","d","e","f","g","h","i","j",
            "k","l","m","n","o","p","q","r","s","t",
            "u","v","w","x","y","z"]

plaintext = ""
ciphertext = ""
key = 0
plaintextLength = 0
count = 0

# Add your code to get the plaintext and convert it to lowercase
plaintext = str(input("write the plaintext:").lower())

# Add your code to get the key and make sure the key is between 1 and 25
key = int(input("1-25 for the key: "))
while key < 1 or key > 25:
    key = int(input("1-25 for the key: "))

# Ciphertext is generated
plaintextLength = len(plaintext)

# Each plaintext letter is convereted into a ciphertext letter
while count <  plaintextLength:
    found = False
    alphabetCount = 0
    while alphabetCount < 26 and found == False:
        if alphabet[alphabetCount] == plaintext[count]:
            found = True
            if alphabetCount + key - 26 < 0:
                ciphertext += alphabet[alphabetCount + key]
            else:
                ciphertext += alphabet[alphabetCount + key - 26]
        alphabetCount = alphabetCount + 1
    count = count + 1

# Add your code to write the ciphertext to a text file
f = open("ciphertext.txt", 'w')
f.write(ciphertext)
f.close()

# Add your code to display the ciphertext
print(ciphertext)


from ten_to_zero_game import ten_to_zero

def no_turn_solver(game, position, prev_positions = {}, remoteness = []):
    """"Determines whether the current position is
    a winning position for the person who plays this
    move"""
    GenerateMoves = game.GenerateMoves
    DoMove = game.DoMove
    PrimitiveValue = game.PrimitiveValue
    my_remoteness = []

    if position in prev_positions:
        return prev_positions[position]

    result = PrimitiveValue(position)

    if result == "undecided":
        has_tie = False
        reset_remoteness = False
        for move in GenerateMoves(position):
            after_move_result = Solve(game, DoMove(position,  move), remoteness = my_remoteness)
            if after_move_result == "lose":
                prev_positions[position] = "win"
                result = "win"

                if len(remoteness) >= 1:
                    my_remoteness[0] = min(my_remoteness[0], remoteness[0] - 1)
                    remoteness[0] = my_remoteness[0] + 1
                else:
                    remoteness.append(my_remoteness[0] + 1)

            elif after_move_result == "tie":
                has_tie = True
        if has_tie and result != "win":
            prev_positions[position] = "tie"
            result = "tie"
            if len(remoteness) >= 1:
                remoteness[0] = min(my_remoteness[0] + 1, remoteness[0])
            else:
                remoteness.append(my_remoteness[0] + 1)
        else:
            prev_positions[position] = "lose"
            result = "lose"
            if len(remoteness) >= 1:
                remoteness[0] = min(my_remoteness[0] + 1, remoteness[0])
            else:
                print("I shouldn't be here")
                remoteness.append(my_remoteness[0] + 1)
        return result
    else:
        if result == "lose" and len(remoteness) == 0:
            remoteness.append(0)
        else:
            remoteness[0] = 0
        return result

def turn_based_solver(game, position, turn, prev_positions = {}, save_result_to = None, remoteness = []):
    print(position)
    """"Determines whether the current position is
    a winning position for the person who plays this
    move
    wins must be formatted as such: \"[turn] won\" """
    assert turn != None, "Turn cannot be left blank"
    GenerateMoves = game.GenerateMoves
    DoMove = game.DoMove
    PrimitiveValue = game.PrimitiveValue
    players = game.players
    opponent = game.opponent
    hashencode = game.hashencode
    hashdecode = game.hashdecode
    convert_piece = game.convert_piece
    my_remoteness = []

    if type(turn) is not str:
        turn = convert_piece(turn)
    if save_result_to == None:
        save_result_to = {"Primitive " + turn + " won": 0, "Primitive " + opponent(turn) + " won":  0,\
        "Primitive tie": 0, "X won" : 0, "O won": 0, "tie": 0}

    # print("=============")
    # game.print_board(position)
    # print("=============")
    a = prev_positions
    hash_position = hashencode(position)
    if hash_position in prev_positions:
        return prev_positions[hash_position]

    result = PrimitiveValue(position)

    if result == "undecided":
        has_tie = False
        has_win = False
        for moves in GenerateMoves(position, turn):
            after_move_result = Solve(game, DoMove(position, moves), \
                prev_positions, opponent(turn), save_result_to)

            if type(turn) is str:
                if after_move_result == turn + " won":
                    has_win = True
                elif after_move_result == "tie":
                    has_tie = True
            else:
                if after_move_result == convert_piece(turn) + " won":
                    has_win = True
                elif after_move_result == "tie":
                    has_tie = True

        if has_win:
            if type(turn) is str:
                prev_positions[hash_position] = turn + " won"
                save_result_to[turn + " won"] += 1
                return turn + " won"
            else:
                prev_positions[hash_position] = convert_piece(turn) + " won"
                save_result_to[convert_piece(turn) + " won"] += 1
                return convert_piece(turn) + " won"

        elif has_tie:
            prev_positions[hash_position] = "tie"
            save_result_to["tie"] += 1
            return "tie"

        else:
            if type(turn) is int:
                prev_positions[hash_position] = convert_piece(opponent(turn)) + " won"
                save_result_to[convert_piece(opponent(turn)) + " won"] += 1
                return convert_piece(opponent(turn)) + " won"
            else:
                prev_positions[hash_position] = opponent(turn) + " won"
                save_result_to[opponent(turn) + " won"] += 1
                return opponent(turn) + " won"
    else:
        prev_positions[hash_position] = result
        save_result_to["Primitive " + result] += 1
        return result

def Solve(game, position, prev_positions = {}, turn = None, save_result_to = None, remoteness = []):
    try:
        game.piece_based
    except AttributeError:
        game.piece_based = False

    if turn == None and not game.piece_based:
        return no_turn_solver(game, position, prev_positions, remoteness)
    else:
        return turn_based_solver(game, position, turn, prev_positions, save_result_to, remoteness)


from ..problem import Problem
from ..datastructures.queue import Queue

def get_solver_mapping():
    return dict(dfs=DFS)

class DFS(object):
    def solve(self, problem: Problem):

        stk = [problem.get_start_node()]

        visited = set()
        visited.add(problem.get_start_node())

        while stk:
            node = stk.pop()

            if problem.is_end(node):
                return node

            unvisited = [child for child in problem.successors(node) if child not in visited]
            unvisited.reverse()

            for child in unvisited:
                visited.add(child)
                stk.append(child)

        return None


import pandas as pd
import matplotlib.pyplot as plt

trainingfile = 'HW_1_training.txt'
data = pd.read_csv(trainingfile, delimiter = "\t", header = None, skiprows = 1)

testingfile = 'HW_1_testing.txt'
test = pd.read_csv(testingfile, delimiter = "\t", header = None, skiprows = 1)
import numpy as np
from sklearn.neighbors.kde import KernelDensity

X = test.loc[:,[0,1]].values
Y = test.loc[:,[2]].values

x1 = data.loc[lambda f : f[2] == 0,[0,1]].values
x2 = data.loc[lambda f : f[2] == 1,[0,1]].values

bandwidth = 0.1
k1 = KernelDensity(kernel='gaussian', bandwidth=bandwidth).fit(x1)
den1 = 10**(k1.score_samples(X))

k2 = KernelDensity(kernel='gaussian', bandwidth=bandwidth).fit(x2)
den2 = 10**(k2.score_samples(X))

#predict
s = X.shape[0]
Y_pred = np.zeros((s,1))
for i in range(s):
    sum = den1[i]*0.5+den2[i]*0.5
    k = []
    k.append(den1[i]*0.5/float(sum))
    k.append(den2[i]*0.5/float(sum))
    Y_pred[i] = np.argmax(k)

countWrong = 0
error_rate = 0
for i in range(len(Y)):
	if (Y[i] != Y_pred[i]):
		countWrong += 1

error_rate = countWrong / float(len(Y))

print('Bandwidth={:f}, \nError rate = {:f} \n'.format(bandwidth, error_rate))

wrong0 = []
wrong1 = []
right0 = []
right1 = []
for i in range(len(Y)):
    if (Y[i] != Y_pred[i] and Y_pred[i] == 0):
        wrong0.append(X[i,:])
    elif (Y[i] != Y_pred[i] and Y_pred[i] == 1):
        wrong1.append(X[i,:])
    elif (Y[i] == Y_pred[i] and Y_pred[i] == 0):
        right0.append(X[i,:])
    else:
        right1.append(X[i,:])

wrong0 = np.matrix(wrong0)
right0 = np.matrix(right0)
wrong1 = np.matrix(wrong1)
right1 = np.matrix(right1)

plt.scatter([right0[:,0]],[right0[:,1]],30, color='blue', marker='o', label='Y = 0')
plt.scatter([right1[:,0]],[right1[:,1]],30, color='blue', marker='X', label='Y = 1')
plt.scatter([wrong0[:,0]],[wrong0[:,1]],30, color='yellow', marker='o', label='Y = 0,misclassified')
plt.scatter([wrong1[:,0]],[wrong1[:,1]],30, color='yellow', marker='X', label='Y = 1,misclassified')

plt.xlabel('x1')
plt.ylabel('x2')
plt.legend(loc='lower right',
       fontsize = 7)
plt.savefig('Bandwidth={:f}.eps'.format(bandwidth),format='eps')
plt.show()


alphaLower = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"]
alpha = alphaLower + list("".join(alphaLower).upper())

itemPrios = {}
rucksack = []

prio = 0
prioSum = 0

for char in alpha:
    prio += 1
    itemPrios[char] = prio

file = open("input.txt", "r")
lines = file.readlines()

for line in lines:
    line = list(line.strip())
    halfRucksack = int((len(line) / 2))
    rucksack.append(line[0:halfRucksack])
    rucksack.append(line[halfRucksack:])
    for char in alpha:
        if char in rucksack[0] and char in rucksack[1]:
            prioSum += itemPrios[char]
    rucksack = []
print(prioSum)


# https://leetcode-cn.com/explore/interview/card/bytedance/244/linked-list-and-tree/1027/
#
# 二叉树的锯齿形层次遍历
# 给定一个二叉树，返回其节点值的锯齿形层次遍历。（即先从左往右，再从右往左进行下一层遍历，以此类推，层与层之间交替进行）。
#
# 例如：
# 给定二叉树 [3,9,20,null,null,15,7],
#
#     3
#    / \
#   9  20
#     /  \
#    15   7
# 返回锯齿形层次遍历如下：
#
# [
#   [3],
#   [20,9],
#   [15,7]
# ]

# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class Solution(object):
    def zigzagLevelOrder(self, root):
        """
        :type root: TreeNode
        :rtype: List[List[int]]
        """
        """
        参考这个博客，二叉树的按层遍历可以用队列实现
        https://blog.csdn.net/sinat_20177327/article/details/78285495
        锯齿遍历使用两个栈来完成
        ps，数组的pop是pop最后一个，可以用来当栈
        """
        from collections import deque
        stack1 = deque([root])
        stack2 = []
        left_to_right = True

        result = []
        while stack1 or stack2:
            breadth = []
            if left_to_right:
                while stack1:
                    node = stack1.pop()
                    if node is not None:
                        breadth.append(node.val)
                        stack2.append(node.left)
                        stack2.append(node.right)
                left_to_right = False
            else:
                while stack2:
                    node = stack2.pop()
                    if node is not None:
                        breadth.append(node.val)
                        stack1.append(node.right)
                        stack1.append(node.left)
                left_to_right = True
            result.append(breadth)

        return result

训练/二叉树/103.二叉树的锯齿形层次遍历.py

103.二叉树的锯齿形层次遍历.py


#!/bin/env python

from vmc_postproc import proc,load,stagflux,sfpnxphz
import numpy as np
import numpy.fft as fft
import re
import six

def get_stat_spin_struct(filenames,nsamp):
    """
    Gets the static structure factor flatened.
    The q-vectors are given by:
    q=arange(L)
    qx,qy=meshgrid(q,q)
    qx=qx.flatten()
    qy=qy.flatten()
    """
    if type(filenames)!=list:
        filenames=[filenames]
    Sq=load.get_quantity(filenames,nsamp)
    params=load.get_attr(filenames[0])
    Lx=int(params['L'])
    Ly=int(params['L'])
    hLx=int(Lx/2)
    hLy=int(Ly/2)
    N=Lx*Ly
    if Sq.shape[2]==N:
        # old file format, struct stored in Fourier components
        Sqxx=np.reshape(0.25*(Sq[:,1,:]+Sq[:,2,:]+Sq[:,3,:]+Sq[:,4,:]),(Sq.shape[0],Lx,Ly))
        Sqyy=np.reshape(0.25*(Sq[:,1,:]+Sq[:,2,:]-Sq[:,3,:]-Sq[:,4,:]),(Sq.shape[0],Lx,Ly))
        Sqzz=np.reshape(Sq[:,0,:],(Sq.shape[0],Lx.Ly))
        Srxx=fft.fftshift(fft.fft2(Sqxx,axes=(1,2)),axes=(1,2))/N
        Sryy=fft.fftshift(fft.fft2(Sqyy,axes=(1,2)),axes=(1,2))/N
        Srzz=fft.fftshift(fft.fft2(Sqzz,axes=(1,2)),axes=(1,2))/N
    else :
        # new file format, struct stored as real space site pairs.
        rx,ry=np.meshgrid(np.arange(Lx,dtype=int),np.arange(Ly,dtype=int))
        rx=rx.ravel()
        ry=ry.ravel()
        rix,rjx=np.meshgrid(rx,rx)
        riy,rjy=np.meshgrid(ry,ry)
        rijx=rjx-rix
        rijy=rjy-riy
        rijx[rijx>=hLx]-=Lx
        rijx[rijx<-hLx]+=Lx
        rijy[rijy>=hLy]-=Ly
        rijy[rijy<-hLy]+=Ly
        rijx=rijx.ravel()
        rijy=rijy.ravel()
        Sr=np.zeros((Sq.shape[0],5,N),dtype=complex)
        for samp in range(Sq.shape[0]):
            for t in range(N):
                Sr[samp,:,t]=np.sum(Sq[samp,:,np.where((rijy+hLy)*Lx+rijx+hLx==t)[0]],axis=0)/N
        Srxx=np.zeros((Sq.shape[0],Lx,Ly),dtype=complex)
        Sryy=np.zeros((Sq.shape[0],Lx,Ly),dtype=complex)
        Srzz=np.zeros((Sq.shape[0],Lx,Ly),dtype=complex)
        for samp in range(Sq.shape[0]):
            Srxx[samp,:,:]=np.reshape(0.25*np.sum(Sr[samp,1:,:],axis=0),(Lx,Ly))
            Sryy[samp,:,:]=np.reshape(0.25*(np.sum(Sr[samp,1:3,:],axis=0)-np.sum(Sr[samp,3:,:],axis=0)),(Lx,Ly))
            Srzz[samp,:,:]=np.reshape(Sr[samp,0,:],(Lx,Ly))
        Sqxx=fft.ifft2(fft.fftshift(Srxx,axes=(1,2)),axes=(1,2))*N
        Sqyy=fft.ifft2(fft.fftshift(Sryy,axes=(1,2)),axes=(1,2))*N
        Sqzz=fft.ifft2(fft.fftshift(Srzz,axes=(1,2)),axes=(1,2))*N
    return (Sqxx,Sqyy,Sqzz),(Srxx,Sryy,Srzz)

def get_eig_sys(filenames,nsamp,wav=None,statstruct=None,tol=1e-12):
    if type(filenames)!=list:
        filenames=[filenames]
    HO=load.get_quantity(filenames,nsamp)
    H=HO[:,:HO.shape[1]/2,:]
    O=HO[:,HO.shape[1]/2:,:]
    if not wav:
        mo=re.search(r'(.*/?[0-9]+)-ProjHeis\.h5',filenames[0])
        wav=mo.groups()[0]+'-WaveFunction.h5'
    wav_st=load.get_wav(wav)
    fs=np.diag(proc.fermisigns(wav_st,load.get_attr(filenames[0])))
    renorm=1.0;
    for k in range(nsamp):
        H[k,:,:]=np.dot(fs,np.dot(H[k,:,:],fs))
        O[k,:,:]=np.dot(fs,np.dot(O[k,:,:],fs))
    evals=np.zeros(H.shape[:2])
    evecs=np.zeros(H.shape,dtype=complex)
    for s in range(nsamp):
        evals[s,:],evecs[s,:,:]=proc.geneigh(H[s,:,:],O[s,:,:],tol)
    if statstruct:
        params=load.get_attr(filenames[0])
        qs=[]
        for qx in range(int(params['L'])):
            for qy in range(int(params['L'])):
                qs.append((qx,qy))
        idx=qs.index((int(params['qx']),int(params['qy'])))
        ssqx=statstruct[0]
        ssqz=statstruct[2]
        if len(ssqx.shape) != 2:
            ssqx=ssqx[0,...].ravel()
            ssqz=ssqz[0,...].ravel()
        if params['stagflux_wav']:
            pkq=stagflux.spinops(params)
            if params['channel']=='trans':
                sqpm=np.einsum('i,sij,j->s',np.conj(pkq[1]),O,pkq[1])
                for s in range(nsamp):
                    O[s,:,:]*=ssqx[idx]*2/sqpm
            elif params['channel']=='long':
                sqzz=np.einsum('i,sij,j->s',np.conj(pkq[2]),O,pkq[2])
                for s in range(nsamp):
                    O[s,:,:]*=ssqz[idx]*2/sqzz
            else:
                raise RuntimeError('Channel not undertood')
        elif params['sfpnxphz_wav']:
            pkq=sfpnxphz.spinops(params)
            sqzz=np.einsum('i,sij,j->s',np.conj(pkq[2]),O,pkq[2])
            for s in range(nsamp):
                O[s,:,:]*=ssqz[idx]*2/sqzz
        else:
            raise RuntimeError('Not implemented')
    return H,O,evals,evecs

def get_sq_ampl(filenames,nsamp,wav=None,tol=1e-12,O=None,evecs=None):
    if type(filenames)!=list:
        filenames=[filenames]
    if O==None or evecs==None:
        H,O,evals,evecs=get_eig_sys(filenames,nsamp,wav,tol=tol)
    qp_mod=None
    params=load.get_attr(filenames[0])
    if params['sfpnxphz_wav']:
        qp_mod=sfpnxphz
    elif params['stagflux_wav']:
        qp_mod=stagflux
    else:
        raise RuntimeError('sfpnzphx_wav not implemented yet')
    pkq=qp_mod.spinops(params)
    if params['stagflux_wav']:
        if params['channel']=='trans':
            return abs(np.einsum('i,sij,sjk->sk',np.conj(pkq[1]),O,evecs))**2
        elif params['channel']=='long':
            return abs(np.einsum('i,sij,sjk->sk',np.conj(pkq[2]),O,evecs))**2
        else:
            raise RuntimeError('channel not understood.')
    elif params['sfpnxphz_wav']:
        out=[None]*3
        for a in range(3):
            out[a]=np.zeros((nsamp,O.shape[1]))
            for k in range(nsamp):
                out[a][k,:]=abs(np.dot(np.conj(pkq[a]),np.dot(O[k,:,:],evecs[k,:,:])))**2
        return out


class User:

    def __init__(self, user_id, user_name):
        self.id = user_id
        self.name = user_name
        self.followers = 0
        self.following = 0

    def follow(self,user):
        user.following += 1
        self.followers += 1

user_1 = User("001", "Nandy")
user_2 = User("002","Akash")
user_2.follow(user_1)
print(user_1.followers)
print(user_1.following)
print(user_2.followers)
print(user_2.following)


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import cv2 as cv
import numpy as np
import tensorflow as tf
import time, random, shutil
from multiprocessing import Process
from tqdm import tqdm
import skimage

'''
介绍      --->    数据增强方法
1、原始图像有4736张
2、使用随机上下翻转，设置阈值为0.3，随机产生一个0-1内的小数，大于阈值时，进行上下翻转
3、使用随机左右翻转，设置阈值为0.3，随机产生一个0-1内的小数，大于阈值时，进行左右翻转
4、随机尺度缩放+随机copy
    对标签进行分析: 若标签中建筑物的占比大于20%，不处理；小于20%记为待处理样本
    在待处理样本中: 若标签中建筑物的占比小于20%、大于7.5%记为优样本；
                 若标签中建筑物的占比小于7.5%记为劣样本；
        随机在优样本和劣样本中各取一张图像，进行随机尺度缩放，然后把处理过的优样本中的建筑物单独提取出来，copy到劣样本中。
'''

class MyProcess(Process):   # 多进程函数
    def __init__(self):
        super(MyProcess, self).__init__()

    def run(self) -> None:
        cv.imwrite('{}'.format(self.img_path), self.img)
        cv.imwrite('{}'.format(self.lab_path), self.lab)

    def getPara(self, img, lab, img_path, lab_path):
        self.img = img
        self.lab = lab
        self.img_path = img_path
        self.lab_path = lab_path

class Data_Enhance():
    def __init__(self):
        self.img_w = self.img_h = 512
        self.read_img_path = r'D:\dataset\train\image'
        self.read_lab_path = r'D:\dataset\train\label'
        self.save_img_path = r'D:\dataset\new_train1\image'
        self.save_lab_path = r'D:\dataset\new_train1\label'
        self.save_img_path_random_copy = r'D:\dataset\random_train\image'
        self.save_lab_path_random_copy = r'D:\dataset\random_train\label'
        self.test_img_path = r'D:\dataset\test\image'
        self.test_lab_path = r'D:\dataset\test\label'

        if not os.path.exists(self.read_img_path) or not os.path.exists(self.read_lab_path):
            raise FileNotFoundError('路径不存在')
        self.mkdirs_path(self.save_img_path)
        self.mkdirs_path(self.save_lab_path)
        self.mkdirs_path(self.save_img_path_random_copy)
        self.mkdirs_path(self.save_lab_path_random_copy)
        random.seed()
        self.ct = 0
        self.save_format = '.png'

    def run(self, use_process=True):
        img_names = os.listdir(self.read_img_path)
        for i in tqdm(range(len(img_names)), ncols = 80, desc='process', mininterval = 0.1):
        # for i in range(5):
            #
            img = cv.imread(os.path.join(self.read_img_path, img_names[i]))
            lab = cv.imread(os.path.join(self.read_lab_path, img_names[i]))
            img_path = os.path.join(self.save_img_path, img_names[i].split('.')[0] + self.save_format)
            lab_path = os.path.join(self.save_lab_path, img_names[i].split('.')[0] + self.save_format)
            self.start_process(img, lab, img_path, lab_path, use_process)

            if random.random() > 0.2:   # 随机上下翻转
                img_1 = tf.image.flip_up_down(img).numpy()
                lab_1 = tf.image.flip_up_down(lab).numpy()
                img_path = os.path.join(self.save_img_path, img_names[i].split('.')[0] + '_1' + self.save_format)
                lab_path = os.path.join(self.save_lab_path, img_names[i].split('.')[0] + '_1' + self.save_format)
                self.start_process(img_1, lab_1, img_path, lab_path, use_process)

            if random.random() > 0.2:   # 随机左右旋转
                img_2 = tf.image.flip_left_right(img).numpy()
                lab_2 = tf.image.flip_left_right(lab).numpy()
                img_path = os.path.join(self.save_img_path, img_names[i].split('.')[0] + '_2' + self.save_format)
                lab_path = os.path.join(self.save_lab_path, img_names[i].split('.')[0] + '_2' + self.save_format)
                self.start_process(img_2, lab_2, img_path, lab_path, use_process)

            if random.random() > 0.2:   # 随机0.6-2尺度缩放
                scale_factor = random.randint(6,20) / 10
                img_3, lab_3 = self.random_scale_resize(img, lab, scale_factor)
                img_path = os.path.join(self.save_img_path, img_names[i].split('.')[0] + '_3' + self.save_format)
                lab_path = os.path.join(self.save_lab_path, img_names[i].split('.')[0] + '_3' + self.save_format)
                self.start_process(img_3, lab_3, img_path, lab_path, use_process)

            if random.random() > 0.7:   # 随机 颜色变换
                img_4 = cv.cvtColor(img, cv.COLOR_BGR2RGB)
                img_path = os.path.join(self.save_img_path, img_names[i].split('.')[0] + '_4' + self.save_format)
                lab_path = os.path.join(self.save_lab_path, img_names[i].split('.')[0] + '_4' + self.save_format)
                self.start_process(img_4, lab, img_path, lab_path, use_process)

            # if random.random() > 0.5:   # 随机增加噪声
            #     img_5 = skimage.util.random_noise(img, mode='s&p')
            #     img_path = os.path.join(self.save_img_path, img_names[i].split('.')[0] + '_5' + self.save_format)
            #     lab_path = os.path.join(self.save_lab_path, img_names[i].split('.')[0] + '_5' + self.save_format)
            #     self.start_process(img_5, lab, img_path, lab_path, use_process)

        print('处理完成！')

    def copy_func(self):    # 复制+重命名
        img_names = os.listdir(r'D:\dataset\test\label')
        new_img_p = r'D:\dataset\new_test\image'
        new_lab_p = r'D:\dataset\new_test\label'
        self.mkdirs_path(new_img_p)
        self.mkdirs_path(new_lab_p)
        for i in tqdm(range(len(img_names)), ncols = 80, desc='process', mininterval = 0.1):
            old_img_path = os.path.join(r'D:\dataset\test\image', img_names[i])
            old_lab_path = os.path.join(r'D:\dataset\test\label', img_names[i])
            if os.path.exists(old_img_path) and os.path.exists(old_lab_path):
                new_img_path = os.path.join(new_img_p,
                                            img_names[i].split('.')[0] + self.save_format)
                new_lab_path = os.path.join(new_lab_p,
                                            img_names[i].split('.')[0] + self.save_format)
                shutil.copy(old_img_path, new_img_path)
                shutil.copy(old_lab_path, new_lab_path)
        print('处理完成')

    def transform_format(self): # 转图片格式
        img_names = os.listdir(r'D:\dataset\test\label')
        new_img_p = r'D:\dataset\new_tests\image'
        new_lab_p = r'D:\dataset\new_tests\label'
        self.mkdirs_path(new_img_p)
        self.mkdirs_path(new_lab_p)
        for i in tqdm(range(len(img_names)), ncols=80, desc='process', mininterval=0.1):
            img_path = os.path.join(self.test_img_path, img_names[i])
            lab_path = os.path.join(self.test_lab_path, img_names[i])
            save_img_path = os.path.join(new_img_p, img_names[i].split('.')[0] + self.save_format)
            save_lab_path = os.path.join(new_lab_p, img_names[i].split('.')[0] + self.save_format)
            img = cv.imread(img_path)
            lab = cv.imread(lab_path)
            self.start_process(img, lab, save_img_path, save_lab_path, use_process=False)

    def random_copy(self, use_process=False):
        # 尺度缩放(0.8->1.2) + copy
        img_size = self.img_h * self.img_w
        img_names = os.listdir(self.save_img_path)
        good_lab = []   # 占比小于20%、大于7.5%
        bad_lab = []    # 占比小于7.5%
        best_lab = []   # 占比大于20%

        # for i in tqdm(range(30), ncols=80, desc='process', mininterval=0.1):
        for i in tqdm(range(len(img_names)), ncols=80, desc='process', mininterval=0.1):
            lab = cv.imread(os.path.join(self.save_lab_path, img_names[i]))
            if np.sum(lab==255) > img_size//4 and np.sum(lab==255) < img_size:
                good_lab.append(img_names[i])
            elif np.sum(lab==255) >= img_size:
                best_lab.append(img_names[i])
            else:
                bad_lab.append(img_names[i])

        times = 2   # 随机缩放与copy次数

        for t in range(times):
            # 第一次是0.7 - 1.2倍缩放， 第二次是0.5 - 1.7倍缩放
            scale_factor = 0.7**t
            random.shuffle(good_lab)
            random.shuffle(bad_lab)
            print('\ngood_lab', len(good_lab))
            print('bad_lab', len(bad_lab))
            print('best_lab', len(best_lab))

            fusion_num = min(len(bad_lab), len(good_lab))
            for j in tqdm(range(fusion_num), ncols=80, desc='process', mininterval=0.1):
                first_img = cv.imread(os.path.join(self.save_img_path, good_lab[j]))
                last_img = cv.imread(os.path.join(self.save_img_path, bad_lab[j]))
                first_lab = cv.imread(os.path.join(self.save_lab_path, good_lab[j]))
                last_lab = cv.imread(os.path.join(self.save_lab_path, bad_lab[j]))

                # 小尺度随机尺度缩放
                # ------------------------------------->>>>>>
                scale1 = random.randint(int(7*scale_factor + 0.5), int(12/scale_factor + 0.5)) / 10
                first_img, first_lab = self.random_scale_resize(first_img, first_lab, scale1)

                scale2 = random.randint(7, 12) / 10
                last_img, last_lab = self.random_scale_resize(last_img, last_lab, scale2)
                # -------------------------------------<<<<<<

                # 融合
                # ------------------------------------->>>>>>
                new_img, new_lab = self.two_img_fusion(first_img, last_img, first_lab, last_lab)
                img_path = os.path.join(self.save_img_path_random_copy, 'r_{}_'.format(t) + img_names[j].split('.')[0] + '_0' + self.save_format)
                lab_path = os.path.join(self.save_lab_path_random_copy, 'r_{}_'.format(t) + img_names[j].split('.')[0] + '_0' + self.save_format)
                self.start_process(new_img, new_lab, img_path, lab_path, use_process)

                # -------------------------------------<<<<<<
                # 用于查看融合效果
                # img_path = os.path.join(self.save_img_path_random_copy, 'r_' + img_names[j].split('.')[0] + '_1' + self.save_format)
                # lab_path = os.path.join(self.save_lab_path_random_copy, 'r_' + img_names[j].split('.')[0] + '_1' + self.save_format)
                # self.start_process(first_img, first_lab, img_path, lab_path, use_process)
                #
                # img_path = os.path.join(self.save_img_path_random_copy, 'r_' + img_names[j].split('.')[0] + '_2' + self.save_format)
                # lab_path = os.path.join(self.save_lab_path_random_copy, 'r_' + img_names[j].split('.')[0] + '_2' + self.save_format)
                # self.start_process(last_img, last_lab, img_path, lab_path, use_process)

    def random_scale_resize(self, img, lab, random_scale):  # 随机尺度缩放
        img_h, img_w, _ = img.shape
        n_h = int(img_h * random_scale)
        n_w = int(img_w * random_scale)
        x = (img_w - n_w) // 2
        y = (img_h - n_h) // 2
        image = cv.resize(img, (n_h, n_w))
        label = cv.resize(lab, (n_h, n_w))
        label = self.label_(label)
        # self.ct += 1
        # cv.imwrite('D:/dataset/{}_img.jpg'.format(self.ct), image)
        # cv.imwrite('D:/dataset/{}_lab.jpg'.format(self.ct), label)
        # print('img.shape', img.shape)
        # print('lab.shape', lab.shape)
        # print('image.shape', image.shape)
        # print('label.shape', label.shape)
        if random_scale < 1:
            new_image = np.ones((img_h,img_w,3)) * 128
            new_label = np.zeros((img_h,img_w,3))
            new_image[y:y+n_h, x:x+n_w, :] = image
            new_label[y:y+n_h, x:x+n_w, :] = label
        else:
            x = (n_w - img_w) // 2
            y = (n_h - img_h) // 2
            x,y = x-1, y-1
            x = np.maximum(x, 0)
            y = np.maximum(y, 0)
            # print('x={},y={}'.format(x,y))
            new_image = image[y:y+512,x:x+512,:]
            new_label = label[y:y+512,x:x+512,:]
        if 0.7 > random.random() >= 0.4:
            new_image = tf.image.flip_up_down(new_image).numpy()
            new_label = tf.image.flip_up_down(new_label).numpy()
        elif random.random() >= 0.7:
            new_image = tf.image.flip_left_right(new_image).numpy()
            new_label = tf.image.flip_left_right(new_label).numpy()
        return new_image, new_label

    def label_(self, lab):  # 标签resize后会，在建筑物边缘出现杂质，以125为阈值进行清除
        n_lab = np.where(lab > 125, 255, 0)
        return n_lab

    def two_img_fusion(self, img1, img2, lab1, lab2):
        # 两张图片融合， img1中建筑物占比大，img2中建筑物占比小，每张经过随机缩放后，img1 -> copy -> img2
        self.ct += 1
        quality = [int(cv.IMWRITE_PNG_COMPRESSION), 0]
        # cv.imwrite('D:/dataset/fusion/img/img_{}_1.png'.format(self.ct), img1, quality)
        # cv.imwrite('D:/dataset/fusion/img/img_{}_2.png'.format(self.ct), img2, quality)

        new_lab = lab1//255 + lab2//255
        new_lab = np.where(new_lab > 0, 255, 0)

        # lab2 = np.where(lab2>0,1,0)
        lab1 = np.array(lab1, dtype=np.uint8)
        lab2 = np.array(lab2, dtype=np.uint8)
        img1 *= (lab1//255)

        lab1 = np.where(lab1 > 0, 0, 1)
        lab1 = np.array(lab1, dtype=np.uint8)

        img2 *= lab1
        new_img = img1 + img2
        # cv.imwrite('D:/dataset/fusion/img/img_{}_3.png'.format(self.ct), img1, quality)
        # cv.imwrite('D:/dataset/fusion/img/img_{}_4.png'.format(self.ct), img2, quality)
        # cv.imwrite('D:/dataset/fusion/img/img_{}_5.png'.format(self.ct), new_img, quality)
        # cv.imwrite('D:/dataset/fusion/lab/lab_{}.png'.format(self.ct), new_lab, quality)

        return new_img, new_lab

    def mkdirs_path(self, new_path):    # 检查目录是否存在，若不存在则创建
        if not os.path.exists(new_path):
            os.makedirs(new_path)
            print('成功创建：{}'.format(new_path))

    def start_process(self, img, lab, img_path, lab_path, use_process=True):
        # 根据需求，选择是否启用多进程处理
        if use_process:
            myProcess = MyProcess()
            myProcess.getPara(img, lab, img_path, lab_path)
            myProcess.start()
        else:
            '''
            1.保存png图像，图像后缀必须为.png，图像质量0-9，默认为3，0质量最好，9最差。
                cv2.imwrite("123.png", img, [int(cv2.IMWRITE_PNG_COMPRESSION), 0])
            2.保存jpg图像，图像后缀必须为.jpg，图像质量0-100，默认为95,100最好，0最差。
                cv2.imwrite("123.jpg", img, [int(cv2.IMWRITE_JPEG_QUALITY), 100])
            3.使用摄像头时默认的图像尺寸为640*480，设置摄像头分辨率的方法如下。
                cap = cv2.VideoCapture(0)
                cap.set(3,1920)
                cap.set(4,1080)
                _, frame = cap.read()
            '''
            cv.imwrite('{}'.format(img_path), img, [int(cv.IMWRITE_PNG_COMPRESSION), 0])
            cv.imwrite('{}'.format(lab_path), lab, [int(cv.IMWRITE_PNG_COMPRESSION), 9])

if __name__ == '__main__':
    data = Data_Enhance()
    run_status = [0,0,1]
    # data.copy_func()
    t1 = time.time()

    if run_status[1]:
        data.run()
        t2 = time.time()
        print('使用多进程耗时：{}'.format(t2-t1))

    if run_status[2]:
        data.run(use_process=False)
        t3 = time.time()
        print('不使用多进程耗时：{}'.format(t3-t1))

    if run_status[1]:
        data.random_copy()

    if run_status[1]:
        data.transform_format()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2018-04-13 00:41:51
# @Author  : skyhiter

import argparse
import pickle

import nltk
import torch
from ignite.contrib.handlers import ProgressBar
from ignite.engine import (Events, create_supervised_evaluator,
                           create_supervised_trainer)
from ignite.engine.engine import Engine, Events, State
from ignite.handlers import EarlyStopping, ModelCheckpoint
from ignite.metrics import Accuracy, Loss, Precision, Recall, RunningAverage
from ignite.utils import convert_tensor
from torchtext import data, vocab
from torchtext.data import BucketIterator, Iterator

class BatchGenerator:
    """BucketIterator/Iterator对象的进一步封装,
    这里的目的是为了封装每条原始数据的index
    """

    def __init__(self, data_iter: Iterator, index: str, x_var: str, y_var: str):
        """param data_iter: data iterator, BucketIterator/Iterator
        param index: 数据原始index, int
        param x_var: 自变量, str
        param y_var: 因变量（标签）, str
        """
        self.data_iter, self.index, self.x_var, self.y_var = data_iter, index, x_var, y_var

    def __len__(self):
        return len(self.data_iter)

    def __iter__(self):
        for batch in self.data_iter:
            index = getattr(batch, self.index)  # 获取原始的index
            # 因为TEXT的include_lengths=True, 所以X是(indexed_sents, lengths)
            X = getattr(batch, self.x_var)  # 获取自变量x_var的数据
            Y = getattr(batch, self.y_var)  # 获取标签y_var的数据
            yield (index, X, Y)  # index保留的是原始的顺序，X是元组，Y是标签列表

def load_dataset(args):
    tokenizer = lambda sent: nltk.tokenize.word_tokenize(sent)  # NLTK提供的默认英语的tokenizer(就是分词的意思)，返回的是word的list
    # use_vocab默认是true, 这说明后面需要加入到vocab索引中；如果数据已经是数字形式标签(0和1)，则不需要再vocab化，所以LABEL需要use_vocab=False
    TEXT = data.Field(
        sequential=True, tokenize=tokenizer, lower=True, batch_first=False,
        include_lengths=True)  # include_lengths很有用，后面获取batch时会顺便返回实际长度
    LABEL = data.Field(sequential=False, use_vocab=False, is_target=True)  # 或者直接使用data.LabelField(), is_target其实没啥大用
    fields = [('label', LABEL), ('review', TEXT)]  # label和review都是自己起的名字，对应数据文件的第一列和第二列
    test = data.TabularDataset(path='./data/_valid_saved.txt', format='TSV', fields=fields, skip_header=False)

    def add_index_attr(dataset):
        """给dataset强制添加index Field; 遍历每个Example对象，为index Field分配数值
        目的是后续shuffle时方便恢复到原顺序。函数顺便返回[[1, ['i', 'love', 'you']], ...]
        """
        _dataset_list = []
        dataset.fields['index'] = data.Field(sequential=False, use_vocab=False)  # 先给dataset的fields属性强加'index'字段
        for index, example in enumerate(dataset):  # 再给dataset的每个Example添加index值
            setattr(example, 'index', index)
            _dataset_list.append([example.label, example.review])
        return _dataset_list

    test_list = add_index_attr(test)  # 给test的每个example添加index，并返回list形式的dataset

    # 加载预存的stoi映射
    TEXT.build_vocab()
    with open('./data/pkl/stoi.pkl', 'rb') as f:
        stoi = pickle.load(f)
    TEXT.vocab.stoi = stoi

    _test_iter = Iterator(
        test,
        batch_size=args.test_batch,
        train=False,
        shuffle=False,
        sort=False,
        device=args.device,
        sort_within_batch=True,
        sort_key=lambda x: len(x.review))

    test_iter = BatchGenerator(_test_iter, 'index', 'review', 'label')

    return test_iter, test_list

def load_model(args):
    with open('./data/saved_models/best_models_info.json', 'r') as f:
        best_models_info = eval(f.read())
    print("best model path: {}".format(best_models_info['best_model_path']))
    model = torch.load(best_models_info['best_model_path'], map_location=args.device)

    return model

def prepare_test_batch(batch, device=None, non_blocking=False):
    """Prepare batch for training: pass to a device with options.

    """
    index, x, y = batch

    # index 不参与模型计算，所以不用convert
    return (index, convert_tensor(x, device=device, non_blocking=non_blocking),
            convert_tensor(y, device=device, non_blocking=non_blocking))

def predict(args):
    test_iter, test_list = load_dataset(args)
    model = load_model(args)
    model.eval()

    pred_results = []
    with torch.no_grad():
        for batch in test_iter:
            index, x, y = prepare_test_batch(batch, device=device, non_blocking=False)
            y_probs = model(x)
            max_probs, preds = torch.max(y_probs, dim=1)
            index, max_probs, preds, y = map(lambda tensors: tensors.tolist(), \
                                            [index, max_probs, preds, y])
            pred_results.extend(list(zip(index, zip(max_probs, preds, y))))
    pred_results.sort(key=lambda item: item[0])
    assert len(pred_results) == len(test_list)
    print(pred_results[:10])

    with open('./results.txt', 'w') as f:
        f.write("prob\tpred\ttrue\ttext\n")
        for index, (prob, pred, true) in pred_results:
            # print(index, prob, pred)
            f.write("{:.3f}\t{}\t{}\t{}\n".format(prob, pred, true, ' '.join(test_list[index][1])))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Prediction by pretraind model (Python 3.6 & Pytorch 1.1)")
    parser.add_argument("--test_batch", type=int, default=4, help="test batch size")
    parser.add_argument("--gpu", type=int, default=0, help="use GPU number, begin index 0")
    args = parser.parse_args()

    device = torch.device("cuda:{}".format(args.gpu) if torch.cuda.is_available() else "cpu")
    setattr(args, "device", device)  # args 对象本没有device属性，这里强加进去，这样args就包含了更多的配置信息

    predict(args)


# -*- coding:utf-8 -*-
# @author	: zjun
# @time		: 2017/06/26

import tensorflow as tf
import numpy as np
import lstm
from lstm import config as conf
import importdata as dataset
from time import strftime,gmtime
import time

#配置参数
CONFIG = {
	'hidden_size':30,
	'embedding_size':150,
	'batch_size':30,
	'dropout_rate':0.4,
	'layes':6,
	'num_epochs':1000,
	'vocab_size':8000,
	'number_of_classes':9,#17
	'sentence_max_len':36,
	'gradient_clipping_value':5,
	'learning_rate':0.005,
	'optimizer':'adadelta',
	'UNK':'UNK'
}

def run():
	'''
	主函数
	'''
	#训练数据文件
	fname = 'E:/Desktop/NeuroNER/data/conll2003/en/train_compatible_with_brat.txt'
	split = ' '
	data = dataset.readf(fname,split)
	config = conf(CONFIG)

	train_input,word_to_index,tag_to_index = dataset.load_data(data,config=config)

	# train_input = train_input[:100]
	batch_iter = dataset.batch_iter(train_input,config.batch_size,config.num_epochs)

	#每个epoch所进行batch的次数
	batch_nums = len(train_input)//config.batch_size

	with tf.Graph().as_default():
		sess = tf.Session()
		with sess.as_default():
			#模型初始化
			model = lstm.BiLSTM_CRF(config)

			#tensorboard信息保存地址
			log_dir = 'E:/Desktop/NeuroNER/contrib/log'
			merged = tf.summary.merge_all()
			train_writer = tf.summary.FileWriter(log_dir + '/train', sess.graph)

			#参数初始化
			sess.run(tf.global_variables_initializer())

			for epoch in range(config.num_epochs*batch_nums):
				#获取当前batch的训练数据
				batch_set = batch_iter.__next__()
				batch_x = [x[0] for x in batch_set]
				batch_y = [x[1] for x in batch_set]
				length = [len(x[0]) for x in batch_set]
				#补齐batch_x
				for x in batch_x:
					x.extend([0]*(config.sentence_max_len-len(x)))
				for x in batch_y:
					x.extend([[0]*config.number_of_classes]*(config.sentence_max_len-len(x)))
				#运行一次
				# print(np.array(batch_x).shape)
				summary,acc,loss,_ = sess.run([merged,model.accuracy,model.loss,model.train_op],
					feed_dict={
					model.input_words:batch_x,
					model.input_labels:batch_y,
					model.sequence_length:length,
					model.dropout_rate:config.dropout_rate
					})

				#每次训练后写入文件
				train_writer.add_summary(summary, epoch)
				#打印输出训练信息
				if epoch%10==0:
					print('{0} epochs:{1},steps:{2},accuracy:{3},loss:{4}'.format(
						time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())),
						epoch//batch_nums,epoch,acc,loss))

if __name__ == '__main__':
	run()


import os
import argparse
from gtts import gTTS

def create_story_folder():
    # Vérifier si le dossier "stories" existe, sinon le créer
    if not os.path.exists('./stories'):
        os.makedirs('./stories')

    # Récupérer le numéro de l'histoire en comptant les dossiers existants dans "./stories"
    story_number = len([name for name in os.listdir(
        './stories') if os.path.isdir(os.path.join('./stories', name))]) + 1

    # Créer le dossier pour l'histoire
    folder_name = f'story_{story_number:02d}'
    folder_path = os.path.join('./stories', folder_name)
    os.makedirs(folder_path)

    print(f"Folder '{folder_name}' created with success.")

    return folder_path


"""
Use this file to download "documents" form ForceManager.
All the links have been previously scraped from aiohttp requests and saved into a file.
- Paolo Frazzetto poalofraz@gmail.com

Useful references:
https://pawelmhm.github.io/asyncio/python/aiohttp/2016/04/22/asyncio-aiohttp.html
"""
import asyncio
import os
import time
from pathlib import Path

import aiofiles
import aiohttp
import nest_asyncio
import pandas as pd
from asyncio_throttle import Throttler
from tqdm.asyncio import tqdm

nest_asyncio.apply()
# Use private libraries to load API tokens
import credentials

token, response = credentials()

PARALLEL_REQUESTS = 240
DOWNLOAD_DIR = Path("./Attachments")
DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)

if __name__ == '__main__':

    # Get data, set up headers, connections and storing structure
    headers = {'Content-Type': 'application/json', 'Accept': '*/*', 'X-Session-Key': token}
    Docs = pd.read_excel("./data/test_attach.xlsx", index_col=0)  # File exported from FM and processed with Docs links

    start = time.perf_counter()

    async def download_with_throttler(n):
        throttler = Throttler(rate_limit=n, period=65, retry_interval=5)  # Limit API calls every 65 seconds
        session = aiohttp.ClientSession(raise_for_status=True, headers=headers)

        async def fetch_account(index, row):
            async with throttler:
                async def download_file(url: str, filename: object = None,
                                        download_path: object = DOWNLOAD_DIR) -> object:
                    if url:  # Check whether there is something to download
                        local_filename = url.split('/')[-1].split('?')[0]
                        if filename:
                            _, extension = os.path.splitext(local_filename)
                            filename += extension
                        else:
                            filename = local_filename
                        download_path = Path(download_path, filename)
                        if download_path.exists():
                            # print("File exists")
                            return
                        async with session.get(url) as resp:
                            async with aiofiles.open(download_path, mode='wb') as f:
                                data = await resp.read()
                                await f.write(data)

                await download_file(row["link"], str(index))

        await tqdm.gather(*(fetch_account(index, row) for index, row in Docs.iterrows()))
        await session.close()

    loop = asyncio.get_event_loop()
    loop.run_until_complete(download_with_throttler(PARALLEL_REQUESTS))
    end = time.perf_counter()

    time_elapsed = end - start
    print("Downloaded {} documents.".format(len(Docs)))
    print(
        "Download Time Elapsed: {:02d}:{:02d}:{:02d}".format(int(time_elapsed // 3600), int(time_elapsed % 3600 // 60),
                                                             int(time_elapsed % 60 // 1)))


import pytest
from ape._cli import cli as ape_cli
from click.testing import CliRunner

EXPECTED_OUTPUT = """
optimism
├── mainnet
│   └── geth  (default)
├── goerli
│   └── geth  (default)
├── sepolia
│   └── geth  (default)
└── local  (default)
    └── test  (default)
""".strip()

@pytest.fixture
def runner():
    return CliRunner()

@pytest.fixture
def cli():
    return ape_cli

def assert_rich_text(actual: str, expected: str):
    """
    The output from `rich` causes a bunch of extra spaces to
    appear at the end of each line. For easier testing, we remove those here.
    Also, we ignore whether the expected line is at the end or in the middle
    of the output to handle cases when the test-runner has additional plugins
    installed.
    """
    expected_lines = [
        x.replace("└", "").replace("├", "").replace("│", "").strip()
        for x in expected.strip().split("\n")
    ]
    actual_lines = [
        x.replace("└", "").replace("├", "").replace("│", "").strip()
        for x in actual.strip().split("\n")
    ]

    for expected_line in expected_lines:
        assert expected_line in actual_lines

def test_networks(runner, cli, optimism):
    optimism.mainnet.set_default_provider("geth")
    optimism.goerli.set_default_provider("geth")
    optimism.sepolia.set_default_provider("geth")

    result = runner.invoke(cli, ["networks", "list"])
    assert_rich_text(result.output, EXPECTED_OUTPUT)


#GPL-3.0-or-later

from ProfX import *

import datetime
import config
import datetime
import logging
import time

from utils import process_whitespaced_table, extract_column_widths

setRipperLogger = logging.getLogger(__name__)

""" Contains data about a Set of tests, containing their Authorisation Group and Autocomment Group. If set was unavailable (locked by other user), Success will be False. """
class SetDefinition():
    def __init__(self, SetCode, AuthorisationGroup:str, AutoCommentGroup:str, AccessedSuccessfully:bool):
        self.Code = SetCode
        self.AuthGroup = AuthorisationGroup
        self.AutoCommGroup = AutoCommentGroup
        self.Success = AccessedSuccessfully
        self.AuthGrpObj = []
        self.AutocomObj = None

    def __str__(self):
        return f"SetDefinition(SetCode={self.Code}, AuthorisationGroup={self.AuthGroup}, AutoCommentGroup={self.AutoCommGroup})"

    def __repr__(self):
        return f"SetDefinition(SetCode={self.Code}, AuthorisationGroup={self.AuthGroup}, AutoCommentGroup={self.AutoCommGroup})"

    def file_dump(self):
        with open(f"./SetData/{self.Code}.txt", "w") as Output:
            Output.write(f"Begin Set '{self.Code}'\n")
            Output.write(f"Extracted {datetime.datetime.now().ctime()}\n\n")
            Output.write(f"{self.Code} is a member of Authorization group '{self.AuthGroup}'\n")
            Output.write(f"{self.Code} belongs to Autocomment group '{self.AutoCommGroup}'\n\n")
            Output.flush()

            Output.write("Authorization Group Details:\n")
            if self.AuthGrpObj:

                Output.write(f"Authorization Group {self.AuthGroup} contains [{len(self.AuthGrpObj)}] possible NPCL lists:\n")
                for AuthGrp in self.AuthGrpObj:
                    Output.write(f"\tNPCL list {AuthGrp.AuthCode}:\n")
                    Output.write("\t\tItems are added to this list if they fulfill the following filter logic:\n")
                    Output.write("\t\t\t")
                    Output.write("\n\t\t\t".join(AuthGrp.LogicTree))
                    Output.write("\n\n")

                    Output.write("\t\tItems on this list are automatically authorized, unless *any* of the following are to True:\n")
                    Output.write(f"\t\t\tChecks:\n")
                    for Line in AuthGrp.Checks:
                        Output.write(f"\t\t\t\t{Line}\n")
                    Output.write("\n")
                    if AuthGrp.StyleSets:
                        for ListFilter in AuthGrp.StyleSets:
                            ValidEntries = [' '.join(x) for x in ListFilter.Value if x[0].strip(".")]
                            nEntries = len(ValidEntries)
                            if nEntries > 5:
                                nSpaces = " "*(len(ListFilter.Type)+1)
                                nCycles = nEntries // 5
                                Output.write(f"\t\t\t{ListFilter.Type} : {ValidEntries[0:5]}\n")
                                for i in range(1, nCycles):
                                    Output.write(f"\t\t\t{nSpaces}: {ValidEntries[i*5:(i*5)+5]}\n")
                                if nEntries % 5 != 0:
                                    Output.write(f"\t\t\t{nSpaces}: {ValidEntries[5*(nCycles):]}\n")
                            else:
                                Output.write(f"\t\t\t{ListFilter.Type} : {ValidEntries}\n")
                        Output.write("\n")
                    else:
                        Output.write("\t\t\tNo associated filters.\n")
            else:
                if self.AuthGroup:
                    Output.write("ERROR: Authorization group(s) could not be retrieved.\n")
                else:
                    Output.write(f"Authorization group(s) do not appear to be assigned to Set {self.Code}.\n")
            Output.write("End of Authorization Group Details\n\n\n")
            Output.flush()
            if self.AutocomObj:
                Output.write(f"Autocomment group {self.AutocomObj.Code} contains [{len(self.AutocomObj.Pages)}] possible sets of Autocomments:\n")
                for AutoCom in self.AutocomObj.Pages:
                    Output.write(f"\t\tDescription: {AutoCom.Description}\n")
                    Output.write(f"\t\tThese autocomments are applied if:\n")
                    Output.write("\t\t\t")
                    Output.write("\n\t\t\t".join(AutoCom.LogicTree))
                    Output.write("\n\n")
                    Output.write("\t\tAutocomments are:\n")
                    for Com in AutoCom.Comments:
                        Output.write(f"\t\t\t{Com}\n")
                    Output.write(f"\t\tPromote to Phone List: {str(AutoCom.PromoteTelList)}\n")
                    Output.write(f"\t\tFurther Set / Superset: {str(AutoCom.FurtherSetSuperset)}\n")
                    Output.write(f"\n\n")
                Output.write(f"End of Autocomments\n\n")
                Output.flush()
            else:
                if self.AutoCommGroup:
                    Output.write("ERROR: AutoComment logic could not be retrieved.\n")
                else:
                    Output.write(f"No AutoComments exist for Set {self.Code}.\n\n")
            Output.write("End of Set")
            Output.flush()

""" Supercontainer for Autocomment structure, containing multiple AutoCommentSet instances in its Pages list.
AutoCommentSet instances list the logic of whether to apply this group of comment(s), and the comment(s) text """
class AutoCommentStructure():
    def __init__(self, AUTOCOM_Code):
        self.Code   = AUTOCOM_Code
        self.Pages  = []

""" Data container for an Autocomment page, containing its description, further set/superset, the actual comments, and the logic tree of whether to apply this AutoCommentSet it """
class AutoCommentSet():
    def __init__(self, lines, treelines):
        self.Description = lines[0][15:].strip() # Removes 1) Description
        self.PromoteTelList = len(lines[1][-1:].strip("."))>0
        self.FurtherSetSuperset = lines[2][27:].strip(".") # Removes "3) Further set or superset "
        self.Comments = []
        for line in lines[3:]:
            self.Comments.append(line[line.find(" ")+9:]) # Removes "[0-99]) Comment: "
        self.LogicTree = treelines

""" NPCL Styles, as defined in NPSET, filter samples to essential data for an NPCL list, e.g. relevant sets/tests or locations. """
class NPCLAuthIntervention():
    def __init__(self, ListName, SetIndex, SetType, SetValues):
        self.Name = ListName
        self.Index = SetIndex
        self.Type = SetType
        self.Value = SetValues

    def __repr__(self):
        return f"NPCLAuthIntervention(ListName={self.Name},SetIndex={self.Index}, SetType={self.Type}, SetValues=...)"

""" Contains an NPCLList, to which samples are assigned, and their associated tests/data filtered(?) based on NPCLAuthIntervention(s)"""
class NPCLList():
    def __init__(self, AuthGroup, AuthCode):
        self.AuthGroup = AuthGroup
        self.AuthCode = AuthCode
        self.LogicTree = []
        self.StyleSets = []
        self.Checks = []

    def __repr__(self):
        return f"NPCLList(AuthGroup={self.AuthGroup}, AuthCode={self.AuthCode}, CheckSettings=..., Tree=..., StyleSets=...)"

    """ Captures data from SETM (Set Maintenance), currently only a Set's Authorisation group and Autocomment group
    Could be extended to capture sample type, etc"""

""" Using SETM, retrieves autocomment and authorisation list assignments for sets and/or supersets. Could also get e.g. sample types."""
def get_Set_definitions(Sets:list, reckless=False):

    def check_access():
        if (TelePath.screenLength >= 24):
            if (TelePath.Lines[23].strip() == "Set accessed by another user at present. Press ESC"):
                setRipperLogger.warning(f"Cannot access Set {Set}, being used by another user. Dropping.")
                #SetDefinitions.append(SetDefinition(SetCode=Set, AuthorisationGroup="", AutoCommentGroup="", AccessedSuccessfully=False))
                TelePath.send_raw(b'\x1b', readEcho=False)
                TelePath.read_data()
                return False
        return True

    return_to_main_menu()
    TelePath.send(config.LOCALISATION.SETMAINTENANCE)
    TelePath.read_data()
    TelePath.send('A')
    TelePath.read_data()
    SetDefinitions = []
    for Set in Sets:
        AUCOM_Code = ""
        AuthGroup_Code = ""
        TelePath.send(Set)
        if not reckless:
            time.sleep(1)   # Seems to take about 1 second to check for access and return error
        else:
            time.sleep(0.1)
        TelePath.read_data()
        if not check_access():
                continue

        if (TelePath.screenLength >= 24):
            if (TelePath.Lines[23].strip() == "WARNING: Exists as a superset, with priority at request entry - OK <N>"):
                setRipperLogger.warning(f"Set {Set} is a 'superset', retrieving anyway...")
                TelePath.send_raw('Y'.encode('ASCII')) #In this instance, the system instantly reacts to the keypress
                # sending it with the \r added by regular send() will then immediately skip to 'acceptable sample types'
                time.sleep(0.1) # Seems if we read too fast we miss the correction.
                if check_access(): #Still possible to get access denied here if another user is in here.
                    TelePath.read_data()
                else:
                     continue

        AuthGroupLine = [x for x in TelePath.ParsedANSI if x.line == 19 and x.column == 68 and x.text.strip(".")] #[x for x in TelePath.Lines[14].split(" ") if x]
        AUCOMLine = [x for x in TelePath.ParsedANSI if x.line == 14 and x.column == 33 and x.text.strip(".")] #[x for x in TelePath.Lines[19].split(" ") if x]

        if AUCOMLine:
            AUCOM_Code = AUCOMLine[0].text
        if AuthGroupLine:
            AuthGroup_Code = AuthGroupLine[0].text

        SetDefinitions.append(SetDefinition(SetCode=Set, AuthorisationGroup=AuthGroup_Code, AutoCommentGroup=AUCOM_Code, AccessedSuccessfully=True))

        setRipperLogger.info(f"get_Set_definitions(): Retrieved Set {Set}: AUCOM {AuthGroup_Code}, SNPCL {AUCOM_Code}")
        TelePath.send(config.LOCALISATION.CANCEL_ACTION)
        TelePath.read_data()
    setRipperLogger.info(f"get_Set_definitions(): Loop complete. Returning {len(SetDefinitions)} identifiers.")
    return SetDefinitions

""" Retrieves details of Authorisation Interventions -
    Being sifted into an NPCL list does not mean you -have- to be examined by a clinical scientist.
    Only when any of these equate to TRUE is the result held for authorisation. """
def get_NPCL_Interventions(NPCL_Lists:list):
    #FIX for 'XT'
    TelePath.tn.set_debuglevel(0)
    if not isinstance(NPCL_Lists, list):
        if isinstance(NPCL_Lists, NPCLList):
            NPCL_Lists = [NPCL_Lists]
    for NPCL_List in NPCL_Lists:
        return_to_main_menu()

        TelePath.send(config.LOCALISATION.NPCLSETS)
        TelePath.read_data()
        setRipperLogger.info(f"Retrieving auto-Authorization Interventions for NPCL list '{NPCL_List.AuthCode}'...")
        TelePath.send(NPCL_List.AuthCode)
        TelePath.read_data()
        if (TelePath.screenLength <= 23): # The list exists
            NPCL_List.Checks = [ x[5:] for x in TelePath.Lines[10:14] ]
            TelePath.send('^L')
            TelePath.read_data()
            if (TelePath.screenLength <= 23): #List exists AND has sets
                if (TelePath.hasErrors):
                    setRipperLogger.warning(f"'{NPCL_List.AuthCode}': No codes are within the given specification")
                    continue
                AuthSets =  [x for x in TelePath.Lines[4:-1] if x] #grab lines with sets
                for line in AuthSets:
                    line = [x.strip() for x in line.split(" ") if x]
                    index = line[0].rstrip(")")
                    type  = line[-1]
                    TelePath.send(index)
                    TelePath.read_data()
                    #There does not seem to be an N option on this screen, ever; max is 28 items. No check / page turning code needed.
                    CodeLines = [x for x in TelePath.Lines[5:-1] if x]
                    CodeTable = process_whitespaced_table(tableLines = CodeLines, headerWidths = [8, 14, 44, 47, 54, 86])
                    assert len(CodeTable) % 2 == 0
                    cleanTable = [x[0:3] for x in CodeTable]
                    for y in [x[3:6] for x in CodeTable]:
                        cleanTable.append(y)
                    del CodeTable
                    cleanTable = [x[1:] for x in cleanTable if x[2]]
                    NPCL_List.StyleSets.append( NPCLAuthIntervention(NPCL_List.AuthCode, index, type, cleanTable) )
                    TelePath.send('A')
                    TelePath.read_data() #Probably won't start with an ANSI code?
                    TelePath.send('^L') # ESC is not echoed, but the L is?!
                    TelePath.read_data()
            else: #List does not have sets. Press ESC.
                setRipperLogger.warning(f"'{NPCL_List.AuthCode}' does not appear to have any filters specified.")
                TelePath.send_raw(b'\x1B')
                #NPCL_List.StyleSets.append( NPCLAuthIntervention(NPCL_List.AuthCode, None, None, None, OptLines) )
        else:
            # Error "<CODE> - Not in code list for Authorisation list code. Press ESC"
            setRipperLogger.warning(f"'{NPCL_List.AuthCode}' does not appear to be an existing NPCL list.")

""" Captures data for an authorisation group, derived from a Set via SETM"""
def get_authorisation_group_structure(SNPCLLists:set):
    NPCLObjs = []
    Page_Has_Blanks = None

    def process_SNPCL_screen():
        nonlocal Page_Has_Blanks
        nonlocal NPCL_Item_Tuples
        _tmpTable = process_whitespaced_table(tableLines=TelePath.Lines[5:21], headerWidths=[7, 15, 44, 47, 55])
        for row in _tmpTable:
            if row[1]:
                NPCL_Item_Tuples.append((row[0].rstrip(")"), row[1], row[2]))
            else:
                Page_Has_Blanks = True
            if row[4]:
                NPCL_Item_Tuples.append((row[3].rstrip(")"), row[4], row[5]))
            else:
                Page_Has_Blanks = True

    return_to_main_menu()
    TelePath.send(config.LOCALISATION.SNPCL)
    TelePath.read_data()

    for SNPCLList in SNPCLLists:
        if not SNPCLList: # Some assays do not have pre-processing
            continue
        NPCL_Item_Tuples = []
        setRipperLogger.info(f"get_authorisation_group_structure(): Retrieving NPCL lists(s) for {SNPCLList}.")
        TelePath.send(SNPCLList)
        TelePath.read_data()
        if (TelePath.screenLength == 24):  # Set does not exist and we entered creation mode
            setRipperLogger.warning(f"Authorisation Group {SNPCLList} is being accessed. Must be retried later.")
            TelePath.send_raw(b'\x1B')
            TelePath.read_data()
            return_to_main_menu()    # There's no graceful return option - B makes a set called B, ^ and '' both yeet one to the main menu.
            TelePath.send('SNPCL')            # So we just need to return...
            TelePath.read_data()
            continue

        if (TelePath.Lines[4].strip() != "Auth code"):  # Set does not exist and we entered creation mode
            setRipperLogger.warning(f"Authorisation Group {SNPCLList} does not appear to exist. This shouldn't happen if fed from set_ripper().")
            TelePath.send(config.LOCALISATION.CANCEL_ACTION)        # There's no graceful return option - B makes a set called B, ^ and '' both yeet one back to the main menu.
            TelePath.read_data()
            TelePath.send(config.LOCALISATION.SNPCL)
            TelePath.read_data()
            continue

        process_SNPCL_screen() # Parses the whitespaced 'table' pf options, checks if we see any blanks, adds options to tuple thing.
        while Page_Has_Blanks == False:
            TelePath.send('N')
            TelePath.read_data()
            process_SNPCL_screen()
        #while Page_Has_Blanks == False

        for NPCLListIndex in NPCL_Item_Tuples:
            StarFreeAuthCode = NPCLListIndex[1].lstrip("*") #The * is prepended by TelePath if there is no log
            tmp = NPCLList(AuthGroup=SNPCLList, AuthCode=StarFreeAuthCode)
            if (NPCLListIndex[1]==StarFreeAuthCode):
                TelePath.send('U')  # The only reason to call U without the index is that indices _could_ be over 99.
                TelePath.read_data()
                TelePath.send(NPCLListIndex[0])#, readEcho=False) # Indices here are not per page but unique, and can be called from any page.
                TelePath.read_data()
                tmp.LogicTree = [x for x in TelePath.Lines[2:22] if x]
                TelePath.send('A')
                TelePath.read_data()
            else:
                tmp.LogicTree = ["No sift specified."]
            NPCLObjs.append( tmp )
        TelePath.send('B')    # Unlisted options, can still go (B)ack to overview
        TelePath.read_data()
    # For SNPCLList in SNPCLLists
    return NPCLObjs

""" Retrieves autocomment structure - the logic that is used for testing, and the resulting comments to be applied """
def get_autocomment_structure(AUCOMSets:set):
    AUCOM_Data = []
    return_to_main_menu()
    TelePath.send(config.LOCALISATION.AUTOCOMMENTS)
    TelePath.read_data()
    for AUCOM_Routine in AUCOMSets:
        if not AUCOM_Routine:
            setRipperLogger.info(f"get_autocomment_structure(): Skipping malformed/empty autocomment routine '{AUCOM_Routine}'")
            continue
        setRipperLogger.info(f"get_autocomment_structure(): Retrieving Autocomment routines for {AUCOM_Routine}")
        Autocomment_Obj = AutoCommentStructure(AUCOM_Routine)
        TelePath.send(AUCOM_Routine)
        TelePath.read_data()
        if (TelePath.screenLength >= 24):
            if (TelePath.Lines[-1] == " Unable to allocate spec. Press ESC"):  # If the set is being accessed and thus locked.
                TelePath.send_raw(b'\x1B')
                continue
        else:
            AUCOM_Chunks = [x for x in TelePath.ParsedANSI if x.line == 3 and x.deleteMode == 0]
            if AUCOM_Chunks:
                AUCOM_Chunks = AUCOM_Chunks[1:] # First line is a header which we would like to ignore
                try:
                    assert len(AUCOM_Chunks) % 2 == 0
                    for ChunkIndex in range(0, len(AUCOM_Chunks))[::2]:
                        Index = AUCOM_Chunks[ChunkIndex].text.strip().rstrip(")")
                        TelePath.send('E')    # Enter (E)dit mode (no other way to see contents)
                        TelePath.read_data()
                        TelePath.send(Index)
                        #time.sleep(0.2)         # Safety measure, gives the server time to collate its data
                        TelePath.read_data()
                        CommStr = [x for x in TelePath.Lines[3:-1] if x]
                        FinalComms=[]
                        for line in CommStr:
                            line = line.split("\r\n")
                            line = [x.strip() for x in line if x]
                            for comm in line:
                                if comm:
                                    FinalComms.append(comm)
                        TelePath.send('A')    # (A)ccept, switches to logic view
                        TelePath.read_data()
                        CommLogic = [x for x in TelePath.Lines[3:-1] if x]
                        tmpACP = AutoCommentSet(FinalComms, CommLogic)
                        Autocomment_Obj.Pages.append(tmpACP)    # Creates data structure and appends to AutoCommentStructure container
                        TelePath.send('A')    # (A)ccept, goes back to root for next index
                        TelePath.read_data()
                except AssertionError:
                    setRipperLogger.debug(f"get_autocomment_structure(): '{AUCOM_Routine}' returns uneven number of AUCOM_Chunks. Adding 'None' and continuing.")
            else:
                setRipperLogger.debug(f"get_autocomment_structure(): '{AUCOM_Routine}' returns no AUCOM_Chunks.")
            TelePath.send('A')    # (A)ccept, returns to root screen
            TelePath.read_data()
            try:
                FullLines = [x for x in TelePath.Lines if x]
                assert len(FullLines) < 5 # If we successfully returned to the main screen, there is only System Header, Screen Type, Routine / Title line. Not even options.
            except AssertionError:
                setRipperLogger.debug(f"get_autocomment_structure(): Unclear whether return to base screen was successful:")
                setRipperLogger.debug(TelePath.Text)
        #if TelePath.screenLength < 24
        AUCOM_Data.append( Autocomment_Obj )
    #for AUCOM_Routine
    return AUCOM_Data

""" For a list of sets, retrieves autocomment and NPCL list categories, then recursively retrieves those details and writes each to file as a report.
'reckless' deactivates a few time.sleep() options which otherwise await notification that other users are blocking entries. May get stuck if turned off!"""
def set_ripper(SetsToRip, reckless=True):
    setRipperLogger.info("set_ripper(): Start up")
    setRipperLogger.info(f"set_ripper(): Received {len(SetsToRip)} sets to grab.")

    #Gather data
    Sets = get_Set_definitions(SetsToRip, reckless)
    SNPCLs = sorted(set([x.AuthGroup for x in Sets if x.Success and x.AuthGroup]))
    AUCOMs = sorted(set([x.AutoCommGroup for x in Sets if x.Success and x.AutoCommGroup]))
    if SNPCLs:
        setRipperLogger.info(f"set_ripper(): Retrieving Authorisation groups for {len(SNPCLs)} SNPCL objects.")
        SNPCL_Objs = get_authorisation_group_structure(SNPCLs)
        setRipperLogger.info(f"set_ripper(): Retrieving NPCL autorisation interventions for {len(SNPCL_Objs)} NPCL lists.")
        get_NPCL_Interventions(SNPCL_Objs)
    if AUCOMs:
        setRipperLogger.info(f"set_ripper(): Retrieving Autocomment logics for {len(AUCOMs)} autocomment sets.")
        AUCOM_Objs = get_autocomment_structure(AUCOMs)

    setRipperLogger.info("set_ripper(): Data collection complete. Collating...")
    #Assemble / collate into 'long form'
    SetObjCounter = 0
    nSamples = len(Sets)
    ReportInterval = max(min(250, int(nSamples*0.1)), 1)
    for SetObj in Sets:
        #setRipperLogger.info(f"set_ripper(): Assembling Set {SetObj.Code}...")
        Matched_Authorisation = [x for x in SNPCL_Objs if x.AuthGroup == SetObj.AuthGroup]
        if Matched_Authorisation:
            SetObj.AuthGrpObj = Matched_Authorisation
        Matched_AutoComment = [x for x in AUCOM_Objs if x.Code == SetObj.AutoCommGroup]
        if len(Matched_AutoComment)==1:
            SetObj.AutocomObj = Matched_AutoComment[0]
        SetObjCounter = SetObjCounter + 1
        Pct = (SetObjCounter / nSamples) * 100

        # Output
        #setRipperLogger.info(f"set_ripper(): Writing Set {SetObj.Code} to file...")
        SetObj.file_dump() # lol

        if( SetObjCounter % ReportInterval == 0): setRipperLogger.info("Collated %d of %d SetObjs (%.2f%%)."% (SetObjCounter, nSamples, Pct))
    setRipperLogger.info("set_ripper(): Complete.")

SetsIO = open("./SetsToExtract.txt", "r")
SetsToRip = SetsIO.readlines()
SetsIO.close()
SetsToRip = [x.strip() for x in SetsToRip][1:5]

try:
    connect_to_LIMS()
    set_ripper(SetsToRip, reckless=True)

except Exception as e:
    setRipperLogger.error(e)

finally:
    disconnect()