import torch
from torchvision import datasets
from torch import nn, optim
import torch.nn.functional as F
import torchvision.transforms as transforms
from torch.utils.data.sampler import SubsetRandomSampler
from torch.utils.data import DataLoader
import numpy as np

train_on_gpu = torch.cuda.is_available()

if not train_on_gpu:
    print('CUDA is not available.  Training on CPU ...')
else:
    print('CUDA is available!  Training on GPU ...')

batch_size = 100
valid_size = 0.2

transform = transforms.Compose([
    transforms.RandomHorizontalFlip(),
    transforms.RandomRotation(10),
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])

train_data = datasets.CIFAR10('data', train=True,
                              download=True, transform=transform)
test_data = datasets.CIFAR10('data', train=False,
                             download=True, transform=transform)

num_train = len(train_data)
indices = list(range(num_train))
np.random.shuffle(indices)
split = int(np.floor(valid_size * num_train))
train_idx, valid_idx = indices[split:], indices[:split]

train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)

train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,
    sampler=train_sampler)
valid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,
    sampler=valid_sampler)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)

classes = ['airplane', 'automobile', 'bird', 'cat', 'deer',
           'dog', 'frog', 'horse', 'ship', 'truck']

class CNN(nn.Module):
   def __init__(self):
       super(CNN, self).__init__()
       self.conv1 = nn.Conv2d(3, 16, 5, padding=2)
       self.conv2 = nn.Conv2d(16, 32, 5, padding=2)
       self.conv3 = nn.Conv2d(32, 64, 5, padding=2)
       self.pool = nn.MaxPool2d(2, 2)
       self.fc1 = nn.Linear(64 * 4 * 4, 500)
       self.fc2 = nn.Linear(500, 10)
       self.dropout = nn.Dropout(0.25)

   def forward(self, x):
       x = self.pool(F.relu(self.conv1(x)))
       x = self.pool(F.relu(self.conv2(x)))
       x = self.pool(F.relu(self.conv3(x)))
       x = x.view(-1, 64 * 4 * 4)
       x = self.dropout(x)
       x = F.relu(self.fc1(x))
       x = self.dropout(x)
       x = self.fc2(x)
       return x

model = CNN()
print(model)

if train_on_gpu:
    model.cuda()

criterion = nn.CrossEntropyLoss()
learning_rate = 0.001
optimizer = optim.Adam(model.parameters(), lr = learning_rate)

num_epochs = 5

valid_loss_min = np.Inf

for epoch in range(num_epochs):
    train_loss = 0.0
    valid_loss = 0.0

    model.train()
    for batch_idx, (images, labels) in enumerate(train_loader):
        if train_on_gpu:
            images, labels = images.cuda(), labels.cuda()
        optimizer.zero_grad()
        prediction = model(images)
        loss = criterion(prediction, labels)
        loss.backward()
        optimizer.step()
        train_loss += loss.item()*images.size(0)

    model.eval()
    for batch_idx, (images, labels) in enumerate(valid_loader):
        if train_on_gpu:
            images, labels = images.cuda(), labels.cuda()
        prediction = model(images)
        loss = criterion(prediction, labels)
        valid_loss += loss.item()*images.size(0)

    train_loss = train_loss/len(train_loader.dataset)
    valid_loss = valid_loss/len(valid_loader.dataset)

    print('Epoch: {} \tTraining Loss: {:.6f} \tValidation Loss: {:.6f}'.format(
        epoch+1, train_loss, valid_loss))

    if valid_loss <= valid_loss_min:
        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(
        valid_loss_min,
        valid_loss))
        torch.save(model.state_dict(), 'trained_model.pt')
        valid_loss_min = valid_loss

model.load_state_dict(torch.load('trained_model.pt'))

test_loss = 0.0
class_correct = list(0. for i in range(10))
class_total = list(0. for i in range(10))

model.eval()
for batch_idx, (data, target) in enumerate(test_loader):
    if train_on_gpu:
        data, target = data.cuda(), target.cuda()
    output = model(data)
    loss = criterion(output, target)
    test_loss += loss.item()*data.size(0)
    _, pred = torch.max(output, 1)
    correct_tensor = pred.eq(target.data.view_as(pred))
    correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())
    for i in range(batch_size):
        label = target.data[i]
        class_correct[label] += correct[i].item()
        class_total[label] += 1

test_loss = test_loss/len(test_loader.dataset)
print('Test Loss: {:.6f}\n'.format(test_loss))

for i in range(10):
    if class_total[i] > 0:
        print('Test Accuracy of %5s: %2d%% (%2d/%2d)' % (
            classes[i], 100 * class_correct[i] / class_total[i],
            np.sum(class_correct[i]), np.sum(class_total[i])))
    else:
        print('Test Accuracy of %5s: N/A (no training examples)' % (classes[i]))

print('\nTest Accuracy (Overall): %2d%% (%2d/%2d)' % (
    100. * np.sum(class_correct) / np.sum(class_total),
    np.sum(class_correct), np.sum(class_total)))


import glob
import logging
import os
from collections import OrderedDict
from subprocess import Popen, PIPE, CalledProcessError

import cyvcf2
import pandas as pd
from django.conf import settings
from django.utils import timezone

from library.django_utils.django_file_utils import get_import_processing_filename
from library.utils.file_utils import name_from_filename
from upload.models import ModifiedImportedVariants, ToolVersion, UploadStep, \
    UploadStepTaskType, VCFSkippedContigs, VCFSkippedContig, UploadStepMultiFileOutput, VCFPipelineStage, \
    SimpleVCFImportInfo
from upload.tasks.vcf.unknown_variants_task import SeparateUnknownVariantsTask, AnnotateImportedVCFTask

def get_vt_tool_version(vt_command):
    p = Popen([vt_command, "--version"], stderr=PIPE)
    _, stderr = p.communicate()
    vt_version = stderr.decode().split('\n')[0]
    tool_version, _ = ToolVersion.objects.get_or_create(name='vt', version=vt_version)
    return tool_version

def create_sub_step(upload_step, sub_step_name, sub_step_commands):
    start_date = timezone.now()
    tool_version = get_vt_tool_version(settings.VCF_IMPORT_VT_COMMAND)
    command_line = ' '.join(sub_step_commands)
    return UploadStep.objects.create(name=sub_step_name,
                                     script=command_line,
                                     upload_pipeline=upload_step.upload_pipeline,
                                     sort_order=upload_step.sort_order,
                                     parent_upload_step=upload_step,
                                     tool_version=tool_version,
                                     start_date=start_date,
                                     task_type=UploadStepTaskType.TOOL)

def _write_split_headers(vcf_filename: str, remove_info: bool, split_headers_filename: str):
    """ Add VT normalize INFO to header, strip if remove_info = True """
    VT_HEADERS = [
        '##INFO=<ID=OLD_MULTIALLELIC,Number=1,Type=String,Description="Original chr:pos:ref:alt encoding">',
        '##INFO=<ID=OLD_VARIANT,Number=.,Type=String,Description="Original chr:pos:ref:alt encoding">',
    ]
    with open(split_headers_filename, "w") as f:
        written_vt_headers = False
        for line in cyvcf2.Reader(vcf_filename).raw_header.split("\n"):
            info_line = line.startswith("##INFO")
            if info_line or line.startswith("#CHROM"):
                if not written_vt_headers:
                    for vt_line in VT_HEADERS:
                        f.write(vt_line + "\n")
                    written_vt_headers = True

                if info_line and remove_info:
                    continue
            if line:
                f.write(line + "\n")

def preprocess_vcf(upload_step, remove_info=False, annotate_gnomad_af=False):
    MAX_STDERR_OUTPUT = 5000  # How much stderr output per process to store in DB

    VCF_CLEAN_AND_FILTER_SUB_STEP = "vcf_clean_and_filter"
    DECOMPOSE_SUB_STEP = "decompose"
    NORMALIZE_SUB_STEP = "normalize"
    UNIQ_SUB_STEP = "uniq"
    REMOVE_HEADER_SUB_STEP = "remove_header"
    SPLIT_VCF_SUB_STEP = "split_vcf"

    genome_build = upload_step.genome_build
    _ = genome_build.reference_fasta  # Fails if not available.

    vcf_filename = upload_step.input_filename
    if not os.path.exists(vcf_filename):
        raise FileNotFoundError(f"Can't access vcf: '{vcf_filename}'")

    pipe_commands = OrderedDict()  # Insert in pipeline order
    sub_steps = {}
    if vcf_filename.endswith(".gz"):
        pipe_commands["zcat"] = [settings.BASH_ZCAT, vcf_filename]
    else:
        pipe_commands["cat"] = ["cat", vcf_filename]

    upload_pipeline = upload_step.upload_pipeline
    vcf_name = name_from_filename(vcf_filename, remove_gz=True)
    skipped_contigs_stats_file = get_import_processing_filename(upload_pipeline.pk, f"{vcf_name}.skipped_contigs.tsv")
    skipped_records_stats_file = get_import_processing_filename(upload_pipeline.pk, f"{vcf_name}.skipped_records.tsv")
    skipped_filters_stats_file = get_import_processing_filename(upload_pipeline.pk, f"{vcf_name}.skipped_filters.tsv")

    manage_command = settings.MANAGE_COMMAND
    read_variants_cmd = manage_command + ["vcf_clean_and_filter",
                                          "--genome-build",
                                          genome_build.name,
                                          "--skipped-contigs-stats-file",
                                          skipped_contigs_stats_file,
                                          "--skipped-records-stats-file",
                                          skipped_records_stats_file,
                                          "--skipped-filters-stats-file",
                                          skipped_filters_stats_file]
    if remove_info:
        read_variants_cmd.append("--remove-info")
    pipe_commands[VCF_CLEAN_AND_FILTER_SUB_STEP] = read_variants_cmd
    sub_steps[VCF_CLEAN_AND_FILTER_SUB_STEP] = create_sub_step(upload_step, VCF_CLEAN_AND_FILTER_SUB_STEP, read_variants_cmd)

    # VT isn't the bottleneck here, it's my programs - so no speed advantage to using "+" for Uncompressed BCF streams
    pipe_commands[DECOMPOSE_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "decompose", "-s", "-"]
    pipe_commands[NORMALIZE_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "normalize", "-n", "-r", genome_build.reference_fasta, "-"]
    pipe_commands[UNIQ_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "uniq", "-"]

    # Split up the VCF
    split_vcf_dir = upload_pipeline.get_pipeline_processing_subdir("split_vcf")
    # We'll cat split_headers_filename on top of every split VCF
    split_headers_filename = os.path.join(split_vcf_dir, name_from_filename(vcf_filename, remove_gz=True))
    _write_split_headers(vcf_filename, remove_info, split_headers_filename)

    pipe_commands[REMOVE_HEADER_SUB_STEP] = [settings.VCF_IMPORT_VT_COMMAND, "view", "-"]
    pipe_commands[SPLIT_VCF_SUB_STEP] = ["split", "-", vcf_name, "--additional-suffix=.vcf.gz", "--numeric-suffixes",
                                         "--lines", str(settings.VCF_IMPORT_FILE_SPLIT_ROWS),
                                         f"--filter='sh -c \"{{ cat {split_headers_filename}; cat; }} | bgzip -c > {split_vcf_dir}/$FILE\"'"]

    for sub_step_name in [DECOMPOSE_SUB_STEP, NORMALIZE_SUB_STEP, UNIQ_SUB_STEP]:
        sub_step_commands = pipe_commands[sub_step_name]
        sub_steps[sub_step_name] = create_sub_step(upload_step, sub_step_name, sub_step_commands)

    piped_command = ' | '.join([' '.join(command_with_args) for command_with_args in pipe_commands.values()])

    # if POPEN_SHELL=True we're running this all as one command as native shell text
    # this is not recommended, less portable and if commands have errors
    if settings.POPEN_SHELL:
        print("single_commands: %s" % " | ".join([' '.join(x) for x in pipe_commands.values()]))
        p = Popen(
            piped_command,
            shell=True
        )
        _, p_stderr = p.communicate()
        if p_stderr:
            logging.error("%s stderr:", piped_command)
            logging.error(p_stderr)

        if p.returncode:
            raise CalledProcessError(p.returncode, piped_command, output=p_stderr)

    else:
        pipes = {}
        stderr_filenames = {}
        p = None
        for sub_step_name, cmd in pipe_commands.items():
            stderr_filename = get_import_processing_filename(upload_pipeline.pk, f"stderr_out_{sub_step_name}.log")
            stderr_f = open(stderr_filename, "wb")
            kwargs = {"stdout": PIPE,
                      "stderr": stderr_f}
            if p:
                kwargs["stdin"] = p.stdout
            p = Popen(cmd, **kwargs)
            pipes[sub_step_name] = p
            stderr_filenames[sub_step_name] = stderr_filename

        print("pipe_commands: %s" % " | ".join([' '.join(x) for x in pipe_commands.values()]))

        for sub_step_name, p_cmd in reversed(pipe_commands.items()):
            p = pipes[sub_step_name]
            p.communicate()
            stderr_output = None
            stderr_filename = stderr_filenames.get(sub_step_name)
            if stderr_filename:
                with open(stderr_filename, "r") as sf:
                    stderr_output = sf.read()
                if stderr_output and len(stderr_output) > MAX_STDERR_OUTPUT:
                    half = MAX_STDERR_OUTPUT // 2
                    stderr_output = stderr_output[:half] + "\n ... \n" + stderr_output[-half:]
                sub_step = sub_steps.get(sub_step_name)
                if sub_step:
                    sub_step.output_text = stderr_output
                    sub_step.save()
                else:
                    logging.warning("%s stderr:", p_cmd)
                    logging.warning(stderr_output)

            if p.returncode:
                raise CalledProcessError(p.returncode, p_cmd, output=stderr_output)

    clean_sub_step = sub_steps[VCF_CLEAN_AND_FILTER_SUB_STEP]
    if os.path.exists(skipped_contigs_stats_file):
        df = pd.read_csv(skipped_contigs_stats_file, header=None, sep='\t', index_col=0)
        if not df.empty:
            import_info = VCFSkippedContigs.objects.create(upload_step=clean_sub_step)

            for contig, count in df.iloc[:, 0].items():
                VCFSkippedContig.objects.create(import_info=import_info,
                                                contig=contig,
                                                num_skipped=count)

    _store_vcf_skip_stats(skipped_records_stats_file, clean_sub_step, "records")
    _store_vcf_skip_stats(skipped_filters_stats_file, clean_sub_step, "FILTER")

    # Create this here so downstream tasks can add modified imported variant messages
    import_info, _ = ModifiedImportedVariants.objects.get_or_create(upload_step=upload_step)
    vcf_import_annotate_dir = upload_pipeline.get_pipeline_processing_subdir("vcf_import_annotate")
    sort_order = upload_pipeline.get_max_step_sort_order()
    for split_vcf_filename in glob.glob(f"{split_vcf_dir}/*.vcf.gz"):
        sort_order += 1
        separate_upload_step = UploadStep.objects.create(upload_pipeline=upload_pipeline,
                                                         name="Separate Unknown Variants Task",
                                                         sort_order=sort_order,
                                                         task_type=UploadStepTaskType.CELERY,
                                                         pipeline_stage=VCFPipelineStage.INSERT_UNKNOWN_VARIANTS,
                                                         input_filename=split_vcf_filename)
        separate_upload_step.launch_task(SeparateUnknownVariantsTask)
        output_filename = split_vcf_filename

        # If we annotate, that file will be processed in UploadStepMultiFileOutput
        if annotate_gnomad_af:
            sort_order += 1
            name = name_from_filename(split_vcf_filename, remove_gz=True)
            output_filename = os.path.join(vcf_import_annotate_dir, f"{name}.annotated.vcf.gz")
            gnomad_step = UploadStep.objects.create(upload_pipeline=upload_pipeline,
                                                    name="Annotate gnomAD AF",
                                                    sort_order=sort_order,
                                                    task_type=UploadStepTaskType.CELERY,
                                                    pipeline_stage=VCFPipelineStage.INSERT_UNKNOWN_VARIANTS,
                                                    input_filename=split_vcf_filename,
                                                    output_filename=output_filename)
            gnomad_step.launch_task(AnnotateImportedVCFTask)

        # We don't know how big the last split file is, so leave items_to_process as null so no check
        UploadStepMultiFileOutput.objects.create(upload_step=upload_step,
                                                 output_filename=output_filename)

def _store_vcf_skip_stats(filename, upload_set, description):
    if os.path.exists(filename):
        df = pd.read_csv(filename, header=None, sep='\t', index_col=0)
        if not df.empty:
            for name, count in df.iloc[:, 0].items():
                message_string = f"Skipped {count} '{name}' {description}"
                SimpleVCFImportInfo.objects.create(upload_step=upload_set, message_string=message_string)


from flask import g, request, redirect, url_for, session
from functools import wraps

def login_required(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        if getattr(g, 'user', None) is None:
            session['next'] = request.url
            return redirect(url_for('login'))
        return f(*args, **kwargs)
    return decorated_function


#!/Users/aholik/tools/miniconda2/bin/python
import sys

scorefile = str(sys.argv[1])

import pyBigWig
bw = pyBigWig.open(scorefile)
scores = open("scores.txt", "w")
for line in open("positions.bed"):
	cols = line.strip().split()
	vals = bw.values(cols[0], int(cols[1])-1, int(cols[2]))
	print >> scores, vals
scores.close()
bw.close()


from torch import Tensor, rand
from torchvision.transforms import RandomPerspective
import torchvision.transforms.functional as F

class RandomPerspective(RandomPerspective):

    def __init__(self, distortion_scale=0.5, interpolation=F.InterpolationMode.BILINEAR, fill=0):

        super().__init__(distortion_scale=distortion_scale, p=1.0, interpolation=interpolation, fill=fill)

    def forward(self, img):

        fill = self.fill
        channels, height, width = F.get_dimensions(img)
        if isinstance(img, Tensor):
            if isinstance(fill, (int, float)):
                fill = [float(fill)] * channels
            else:
                fill = [float(f) for f in fill]

        startpoints, endpoints = self.get_params(width, height, self.distortion_scale)

        return F.perspective(img, startpoints, endpoints, self.interpolation, fill), startpoints, endpoints

if __name__ == '__main__':

    import os
    import argparse
    from random import randint

    from torchvision.datasets import MNIST
    from torchvision.transforms import ToTensor
    import matplotlib.pyplot as plt

    os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'

    parser = argparse.ArgumentParser()
    parser.add_argument('--root', default='.', type=str, help='root directory of dataset where MNIST/raw/* exist')
    parser.add_argument('--distortion_scale', default=0.5, type=str, help='argument to control degree of distortion')
    args = parser.parse_args()

    assert os.path.exists(f'{args.root}/MNIST/raw'), f'{args.root}/MNIST/raw does not exist, check the root path'
    mnist = MNIST(args.root, download=False, train=True, transform=ToTensor())

    fig = plt.figure(figsize=(8, 8))

    img, label = mnist[randint(0, len(mnist)+1)]
    fig.add_subplot(1, 2, 1)
    plt.imshow(img.squeeze(), cmap='gray')
    plt.title(f'Original, label = {label}')
    plt.axis('off')

    transformed_img, startpoints, endpoints = RandomPerspective(distortion_scale=args.distortion_scale)(img)
    fig.add_subplot(1, 2, 2)
    plt.imshow(transformed_img.squeeze(), cmap='gray')
    plt.title(f'End points: {endpoints}')
    plt.axis('off')

    plt.show()


import sys

from dcim.models import Manufacturer
from startup_script_utils import load_yaml

manufacturers = load_yaml("/opt/netbox/initializers/manufacturers.yml")

if manufacturers is None:
    sys.exit()

for params in manufacturers:
    manufacturer, created = Manufacturer.objects.get_or_create(**params)

    if created:
        print("🏭 Created Manufacturer", manufacturer.name)


s = input()
ans = ""
for i in s:
    temp = int(i)
    if temp >= 5:
        ans += str(9-temp)
    else:
        ans += str(temp)
if ans[0] == '0':
    ans = s[0] + ans[1:]
print(ans)


from collections import Counter
import os
import numpy as np
from tqdm import tqdm
from medvqa.datasets.chest_imagenome.chest_imagenome_dataset_management import (
    load_gold_standard_dicom_ids,
    load_chest_imagenome_label_names_and_templates,
    load_chest_imagenome_dicom_ids_and_labels_as_numpy_matrix,
    load_chest_imagenome_silver_bboxes_as_numpy_array,
)
from medvqa.datasets.utils import deduplicate_indices
from medvqa.datasets.vqa import VQA_Evaluator, VQA_Trainer
from medvqa.datasets.mimiccxr import (
    MIMICCXR_CACHE_DIR,
    MIMICCXR_IMAGE_ORIENTATIONS,
    MIMICCXR_STUDY_REGEX,
    MIMICCXR_EvalViewModes,
    MIMICCXR_ViewModes,
    get_dicom_id_and_orientation_list,
    get_broken_images,
    get_image_views_dict,
    get_mimiccxr_small_image_path,
    get_split_dict,
)
from medvqa.datasets.image_processing import (
    classify_and_rank_questions,
    get_nearest_neighbors,
)
from medvqa.metrics.classification.multilabel_prf1 import MultiLabelPRF1
from medvqa.utils.files import (
    get_cached_json_file,
    get_file_path_with_hashing_if_too_long,
    load_pickle,
    save_to_pickle,
)
from medvqa.utils.constants import CHEXPERT_LABELS, VINBIG_DISEASES, ReportEvalMode
from medvqa.datasets.preprocessing import (
    get_average_question_positions,
    get_question_frequencies,
)
from medvqa.utils.logging import print_red

def _get_train_preprocessing_save_path(qa_adapted_reports_filename, tokenizer, use_chest_imagenome_compatible_data):
    tokenizer_string = f'{tokenizer.vocab_size},{tokenizer.hash[0]},{tokenizer.hash[1]}'
    if tokenizer.medical_tokenization:
        tokenizer_string += f',{tokenizer.medical_terms_frequency_filename}'
    strings = [
        f'dataset={qa_adapted_reports_filename}',
        f'tokenizer={tokenizer_string}',
        f'use_chest_imagenome_compatible_data={use_chest_imagenome_compatible_data}',
    ]
    return get_file_path_with_hashing_if_too_long(MIMICCXR_CACHE_DIR, 'mimiccxr_preprocessed_train_data__', strings, 'pkl')

def get_test_preprocessing_save_path(qa_adapted_reports_filename, tokenizer, report_eval_mode=None,
                                    pretrained_checkpoint_path=None, precomputed_question_probs_path=None,
                                    precomputed_question_thresholds_path=None,
                                    n_questions_per_report=None, qclass_threshold=None, use_random_image=False,
                                    eval_view_mode=None):
    assert eval_view_mode is not None
    strings = [
        f'dataset={qa_adapted_reports_filename}',
        f'tokenizer={tokenizer.vocab_size},{tokenizer.hash[0]},{tokenizer.hash[1]}',
        f'eval_view_mode={eval_view_mode}',
    ]
    if report_eval_mode is not None:
        strings.append(f'report_eval_mode={report_eval_mode}')

        if report_eval_mode == ReportEvalMode.QUESTION_CLASSIFICATION or\
           report_eval_mode == ReportEvalMode.CHEXPERT_AND_QUESTION_CLASSIFICATION:
            assert pretrained_checkpoint_path is not None or\
                   precomputed_question_probs_path is not None
            if precomputed_question_probs_path is not None:
                strings.append(f'precomputed_question_probs_path={precomputed_question_probs_path}')
                strings.append(f'timestamp={os.path.getmtime(precomputed_question_probs_path)}')
            else:
                strings.append(f'pretrained_checkpoint_path={pretrained_checkpoint_path}')

        if report_eval_mode == ReportEvalMode.NEAREST_NEIGHBOR:
            assert pretrained_checkpoint_path is not None
            strings.append(f'pretrained_checkpoint_path={pretrained_checkpoint_path}')

        if report_eval_mode == ReportEvalMode.QUESTION_CLASSIFICATION or\
           report_eval_mode == ReportEvalMode.CHEXPERT_AND_QUESTION_CLASSIFICATION or\
           report_eval_mode == ReportEvalMode.MOST_POPULAR:
            assert n_questions_per_report is not None
            strings.append(f'n_questions_per_report={n_questions_per_report}')

        if report_eval_mode == ReportEvalMode.QUESTION_CLASSIFICATION or\
           report_eval_mode == ReportEvalMode.CHEXPERT_AND_QUESTION_CLASSIFICATION:
            assert (qclass_threshold is not None) != (precomputed_question_thresholds_path is not None)
            if qclass_threshold:
                strings.append(f'qclass_threshold={qclass_threshold}')
            else:
                strings.append(f'precomputed_question_thresholds_path={precomputed_question_thresholds_path}')

        if use_random_image: strings.append('rand_img')

    file_path = get_file_path_with_hashing_if_too_long(MIMICCXR_CACHE_DIR, 'mimiccxr_preprocessed_test_data__', strings)
    return file_path

def _get_orientation_id(orientation):
    try:
        return MIMICCXR_IMAGE_ORIENTATIONS.index(orientation)
    except ValueError:
        return 0

def _rank_questions(report_ids, precomputed_question_probs_path, n_questions, top_k,
        min_num_q_per_report=5, threshold=None, precomputed_question_thresholds_path=None):
    assert (threshold is None) != (precomputed_question_thresholds_path is None)
    print_red('_rank_questions(): loading question probabilities from', precomputed_question_probs_path)
    question_probs = load_pickle(precomputed_question_probs_path)
    if precomputed_question_thresholds_path is not None:
        print_red('_rank_questions(): loading question thresholds from', precomputed_question_thresholds_path)
        thresholds = load_pickle(precomputed_question_thresholds_path)
    question_ids = list(range(n_questions))
    questions = [None] * len(report_ids)
    K = min(min_num_q_per_report, top_k)
    for i, rid in enumerate(report_ids):
        probs = question_probs[rid]
        if threshold:
            probs -= threshold
        else:
            probs -= thresholds
        question_ids.sort(key=lambda k:probs[k], reverse=True)
        questions[i] = [qid for k, qid in enumerate(question_ids) if k < top_k and probs[qid] >= 0]
        if len(questions[i]) < K:
            questions[i] = question_ids[:K]
            assert 0 < len(questions[i]) <= top_k
    print_red('_rank_questions(): average num of questions per report:', sum(len(q) for q in questions) / len(questions))
    return questions

def _sanity_check_questions(questions, report_ids, mimiccxr_qa_reports):
    n_q = len(mimiccxr_qa_reports['questions'])
    gt = np.zeros((len(report_ids), n_q))
    pred = np.zeros((len(report_ids), n_q))
    for i, ri in enumerate(report_ids):
        for j in mimiccxr_qa_reports['reports'][ri]['question_ids']:
            gt[i][j] = 1
        for j in questions[i]:
            pred[i][j] = 1
    met = MultiLabelPRF1(device='cpu')
    met.update((pred, gt))
    res = met.compute()
    print_red(f'_sanity_check_questions(): f1(macro)={res["f1_macro_avg"]}, f1(micro)={res["f1_micro_avg"]}')

def _precompute_questions_per_report(basic_data, report_eval_mode,
        n_questions_per_report=None, qclass_threshold=None,
        qa_adapted_reports_filename=None, image_transform=None,
        image_local_feat_size=None, n_questions_aux_task=None, pretrained_weights=None,
        pretrained_checkpoint_path=None, precomputed_question_probs_path=None,
        precomputed_question_thresholds_path=None, batch_size=None):

    print('_precompute_questions_per_report():')

    # Filepath
    strings = [
        f'report_eval_mode={report_eval_mode}',
        f'dataset={qa_adapted_reports_filename}',
    ]
    if n_questions_per_report is not None:
        strings.append(f'n_questions_per_report={n_questions_per_report}')
    if report_eval_mode == ReportEvalMode.QUESTION_CLASSIFICATION or\
            report_eval_mode == ReportEvalMode.CHEXPERT_AND_QUESTION_CLASSIFICATION:
        assert pretrained_checkpoint_path is not None or precomputed_question_probs_path is not None
        assert qclass_threshold is not None or precomputed_question_thresholds_path is not None
        if precomputed_question_probs_path is not None:
            strings.append(f'precomputed_question_probs_path={precomputed_question_probs_path}')
            strings.append(f'timestamp={os.path.getmtime(precomputed_question_probs_path)}')
        else:
            strings.append(f'pretrained_checkpoint_path={pretrained_checkpoint_path}')
        if qclass_threshold:
            strings.append(f'qclass_threshold={qclass_threshold}')
        else:
            strings.append(f'precomputed_question_thresholds_path={precomputed_question_thresholds_path}')
    file_path = get_file_path_with_hashing_if_too_long(MIMICCXR_CACHE_DIR, 'questions_per_report', strings)

    data = load_pickle(file_path)
    if data is not None:
        print('   questions per report data loaded from', file_path)
        print('   len(data) =', len(data))
        return data

    mimiccxr_qa_reports = get_cached_json_file(os.path.join(MIMICCXR_CACHE_DIR, qa_adapted_reports_filename))

    test_report_ids = [rid for rid, split in zip(basic_data['report_ids'], basic_data['splits']) if split == 'test']
    assert len(test_report_ids) > 0

    data = {}

    if report_eval_mode == ReportEvalMode.GROUND_TRUTH:
        for ri in test_report_ids:
            data[ri] = mimiccxr_qa_reports['reports'][ri]['question_ids']

    elif report_eval_mode == ReportEvalMode.QUESTION_CLASSIFICATION or\
         report_eval_mode == ReportEvalMode.CHEXPERT_AND_QUESTION_CLASSIFICATION:
        assert n_questions_per_report is not None
        assert qclass_threshold is not None or precomputed_question_thresholds_path is not None
        train_report_ids = [rid for rid, split in zip(basic_data['report_ids'], basic_data['splits']) if split == 'train']
        assert len(train_report_ids) > 0
        question_scores = get_average_question_positions(MIMICCXR_CACHE_DIR, qa_adapted_reports_filename, train_report_ids)
        if precomputed_question_probs_path is not None:
            questions = _rank_questions(test_report_ids,
                                        precomputed_question_probs_path,
                                        n_questions_aux_task,
                                        n_questions_per_report,
                                        threshold=qclass_threshold,
                                        precomputed_question_thresholds_path=precomputed_question_thresholds_path)
        elif pretrained_weights is not None:
            test_image_paths = [image_path for image_path, split in zip(basic_data['image_paths'], basic_data['splits']) if split == 'test']
            questions = classify_and_rank_questions(test_image_paths,
                                        image_transform,
                                        image_local_feat_size,
                                        n_questions_aux_task,
                                        pretrained_weights,
                                        batch_size,
                                        n_questions_per_report,
                                        qclass_threshold)
        else: assert False
        assert len(questions) == len(test_report_ids)
        _sanity_check_questions(questions, test_report_ids, mimiccxr_qa_reports)
        question_scorer = lambda j : question_scores[j]

        if report_eval_mode == ReportEvalMode.QUESTION_CLASSIFICATION:
            for i, ri in enumerate(test_report_ids):
                classified_question_ids = sorted(questions[i], key=question_scorer)
                data[ri] = classified_question_ids
        else: # hybrid method
            chexpert_question_ids = [x for x in range(len(CHEXPERT_LABELS))]
            for i, ri in enumerate(test_report_ids):
                classified_question_ids = sorted(questions[i], key=question_scorer)
                assert all(x not in chexpert_question_ids for x in classified_question_ids)
                data[ri] = chexpert_question_ids + classified_question_ids

    elif report_eval_mode == ReportEvalMode.MOST_POPULAR:
        assert n_questions_per_report != None
        test_report_ids = set(test_report_ids)
        train_report_ids = [i for i in range(len(mimiccxr_qa_reports['reports'])) if i not in test_report_ids]
        question_scores = get_question_frequencies(MIMICCXR_CACHE_DIR, qa_adapted_reports_filename, train_report_ids)
        question_ids = list(range(len(mimiccxr_qa_reports['questions'])))
        assert len(question_scores) == len(question_ids)
        question_ids.sort(key=lambda i : question_scores[i], reverse=True)
        question_ids = question_ids[:n_questions_per_report]
        for ri in test_report_ids:
            data[ri] = question_ids

    elif report_eval_mode == ReportEvalMode.NEAREST_NEIGHBOR:
        assert pretrained_weights is not None
        assert pretrained_checkpoint_path is not None
        test_image_paths = [image_path for image_path, split in zip(basic_data['image_paths'], basic_data['splits']) if split == 'test']
        train_image_paths = [image_path for image_path, split in zip(basic_data['image_paths'], basic_data['splits']) if split == 'train']
        nearest_neighbors = get_nearest_neighbors(target_images = test_image_paths,
                                                  reference_images = train_image_paths,
                                                  transform = image_transform,
                                                  pretrained_weights = pretrained_weights,
                                                  batch_size = batch_size,
                                                  cache_dir = MIMICCXR_CACHE_DIR,
                                                  pretrained_checkpoint_path = pretrained_checkpoint_path,
                                                  suffix = 'split=test')
        for i, ri in enumerate(test_report_ids):
            nearest_ri = train_report_ids[nearest_neighbors[i]]
            assert ri != nearest_ri
            data[ri] = mimiccxr_qa_reports['reports'][nearest_ri]['question_ids']

    elif report_eval_mode == ReportEvalMode.CHEXPERT_LABELS:
        question_ids = list(range(len(CHEXPERT_LABELS)))
        for ri in test_report_ids:
            data[ri] = question_ids
    elif report_eval_mode == ReportEvalMode.VINBIG_DISEASES:
        question_ids = list(range(len(VINBIG_DISEASES)))
        for ri in test_report_ids:
            data[ri] = question_ids
    else:
        assert False, f'Unknown report_eval_mode = {report_eval_mode}'

    save_to_pickle(data, file_path)
    print('   len(data) =', len(data))
    print('   questions per report data saved to', file_path)

    return data

def _get_basic_data(qa_adapted_reports_filename, image_views_dict, split_dict,
                    view_mode=MIMICCXR_ViewModes.ANY_SINGLE,
                    use_chest_imagenome_compatible_data=False, chest_imagenome_dicom_ids=None):

    if use_chest_imagenome_compatible_data:
        assert chest_imagenome_dicom_ids is not None
        assert type(chest_imagenome_dicom_ids) == set
        assert view_mode == MIMICCXR_ViewModes.CHEST_IMAGENOME

    print(f'Obtaining basic data ...')

    file_path = os.path.join(MIMICCXR_CACHE_DIR,
        f'mimiccxr_basic_data(dataset={qa_adapted_reports_filename},view_mode={view_mode}).pkl')
    data = load_pickle(file_path)
    if data is not None:
        print(f'   basic data loaded from {file_path}')
        return data

    broken_images = get_broken_images()

    mimiccxr_qa_reports = get_cached_json_file(os.path.join(MIMICCXR_CACHE_DIR, qa_adapted_reports_filename))

    data = dict(
        report_ids = [],
        dicom_ids = [],
        image_paths = [],
        orientation_ids = [],
        splits = [],
    )

    for ri, report in tqdm(enumerate(mimiccxr_qa_reports['reports'])):

        part_id, subject_id, study_id = map(int, MIMICCXR_STUDY_REGEX.findall(report['filepath'])[0])
        views = image_views_dict[(subject_id, study_id)]
        # images = glob.glob(f'/mnt/workspace/mimic-cxr-jpg/images-small/p{part_id}/p{subject_id}/s{study_id}/*.jpg')
        # assert len(views) == len(images)

        dicom_id_orientation_pairs = get_dicom_id_and_orientation_list(views, view_mode, chest_imagenome_dicom_ids)

        for dicom_id, orientation in dicom_id_orientation_pairs:
            assert dicom_id is not None

            if (subject_id, study_id, dicom_id) not in broken_images:
                image_path = get_mimiccxr_small_image_path(part_id, subject_id, study_id, dicom_id)
                orientation_id = _get_orientation_id(orientation)
                data['report_ids'].append(ri)
                data['dicom_ids'].append(dicom_id)
                data['image_paths'].append(image_path)
                data['orientation_ids'].append(orientation_id)
                data['splits'].append(split_dict[(subject_id, study_id, dicom_id)])

    save_to_pickle(data, file_path)
    print(f'   len(report_ids) =', len(data['report_ids']))
    print(f'   len(set(report_ids)) =', len(set(data['report_ids'])))
    # Print the distribution of orientations
    counter = Counter(data["orientation_ids"])
    for oid, c in counter.items():
        print(f'   {MIMICCXR_IMAGE_ORIENTATIONS[oid]}, count = {c}')
    print(f'   basic data saved to {file_path}')

    return data

def _filter_report_orientation_pairs_for_test(report_ids, orientation_ids, view_mode):

    print(f'Filtering report orientation pairs for test ...')

    if view_mode == MIMICCXR_EvalViewModes.ALL:
        output = list(range(len(report_ids)))
    elif view_mode == MIMICCXR_EvalViewModes.FRONT_ALL:
        idxs = []
        PA_ID = MIMICCXR_IMAGE_ORIENTATIONS.index('PA')
        AP_ID = MIMICCXR_IMAGE_ORIENTATIONS.index('AP')
        valid_oids = [PA_ID, AP_ID]
        for i, (ri, oid) in enumerate(zip(report_ids, orientation_ids)):
            if oid in valid_oids:
                idxs.append(i)
        output = idxs
    elif view_mode == MIMICCXR_EvalViewModes.FRONT_SINGLE:
        rid2idx = {}
        PA_ID = MIMICCXR_IMAGE_ORIENTATIONS.index('PA')
        AP_ID = MIMICCXR_IMAGE_ORIENTATIONS.index('AP')
        valid_oids = [PA_ID, AP_ID]
        for i, (ri, oid) in enumerate(zip(report_ids, orientation_ids)):
            if oid in valid_oids:
                if ri not in rid2idx:
                    rid2idx[ri] = i
                else:
                    if oid == PA_ID:
                        rid2idx[ri] = i
        output = list(rid2idx.values())
    elif view_mode == MIMICCXR_EvalViewModes.ANY_SINGLE:
        rid2idx = {}
        PA_ID = MIMICCXR_IMAGE_ORIENTATIONS.index('PA')
        AP_ID = MIMICCXR_IMAGE_ORIENTATIONS.index('AP')
        for i, (ri, oid) in enumerate(zip(report_ids, orientation_ids)):
            if ri not in rid2idx:
                rid2idx[ri] = i
            else:
                if oid == PA_ID:
                    rid2idx[ri] = i
                elif oid == AP_ID:
                    if rid2idx[ri] != PA_ID:
                        rid2idx[ri] = i
        output = list(rid2idx.values())
    else:
        raise ValueError(f'Unknown view_mode = {view_mode}')

    print(f'   len(report_ids) = {len(report_ids)}')
    print(f'   len(output) = {len(output)}')
    return output

def _preprocess_data(self, split_name):

    assert split_name in ('train_val', 'test'), f'Unknown split_name = {split_name}'

    is_train_val = split_name == 'train_val'

    tokenizer = self.tokenizer
    mimiccxr_qa_reports = self.mimiccxr_qa_reports

    if tokenizer.medical_tokenization and split_name != 'test':
        answer_string2ids_func = tokenizer.string2medical_tag_ids
    else:
        answer_string2ids_func = tokenizer.string2ids

    if mimiccxr_qa_reports is None:
        file_path = os.path.join(MIMICCXR_CACHE_DIR, self.qa_adapted_reports_filename)
        print(f'Loading {file_path}')
        mimiccxr_qa_reports = get_cached_json_file(file_path)

    split_dict = get_split_dict()
    image_views_dict = get_image_views_dict()
    basic_data = _get_basic_data(self.qa_adapted_reports_filename, image_views_dict, split_dict,
                                 view_mode=self.view_mode, use_chest_imagenome_compatible_data=self.use_chest_imagenome_compatible_data,
                                 chest_imagenome_dicom_ids=self.chest_imagenome_dicom_ids)

    if split_name == 'test' and self.report_eval_mode is not None:
        questions_per_report = _precompute_questions_per_report(basic_data, self.report_eval_mode,
                        qa_adapted_reports_filename=self.qa_adapted_reports_filename,
                        image_transform=self.transform,
                        image_local_feat_size=self.image_local_feat_size,
                        n_questions_aux_task=self.n_questions_aux_task,
                        pretrained_weights=self.pretrained_weights,
                        batch_size=self.batch_size,
                        n_questions_per_report=self.n_questions_per_report,
                        qclass_threshold=self.qclass_threshold,
                        pretrained_checkpoint_path=self.pretrained_checkpoint_path,
                        precomputed_question_probs_path=self.precomputed_question_probs_path,
                        precomputed_question_thresholds_path=self.precomputed_question_thresholds_path)
    else:
        questions_per_report = None

    print('Preprocessing MIMIC-CXR vqa dataset ...')

    # Define question_list to use
    question_list = mimiccxr_qa_reports['questions'] # default
    if split_name == 'test':
        if self.report_eval_mode == ReportEvalMode.CHEXPERT_LABELS:
            question_list = CHEXPERT_LABELS
        elif self.report_eval_mode == ReportEvalMode.CHEXPERT_AND_QUESTION_CLASSIFICATION:
            question_list = CHEXPERT_LABELS + mimiccxr_qa_reports['questions']

    # Collect multiple fields considerng the split and mode
    if is_train_val:
        if self.use_chest_imagenome_compatible_data:
            # Use only instances whose dicom_id is not in Chest-Imagenome gold standard
            gold_dicom_ids = self.chest_imagenome_gold_dicom_ids
            assert type(gold_dicom_ids) is set
            report_ids = [rid for rid, s, did in zip(basic_data['report_ids'], basic_data['splits'], basic_data['dicom_ids'])\
                 if s != 'test' and did not in gold_dicom_ids]
            image_paths = [ip for ip, s, did in zip(basic_data['image_paths'], basic_data['splits'], basic_data['dicom_ids'])\
                    if s != 'test' and did not in gold_dicom_ids]
            orientation_ids = [oid for oid, s, did in zip(basic_data['orientation_ids'], basic_data['splits'], basic_data['dicom_ids'])\
                    if s != 'test' and did not in gold_dicom_ids]
            dicom_ids = [did for s, did in zip(basic_data['splits'], basic_data['dicom_ids']) if s != 'test' and did not in gold_dicom_ids]
            splits = [s for s, did in zip(basic_data['splits'], basic_data['dicom_ids']) if s != 'test' and did not in gold_dicom_ids]
        else:
            report_ids = [rid for rid, s in zip(basic_data['report_ids'], basic_data['splits']) if s != 'test']
            image_paths = [ip for ip, s in zip(basic_data['image_paths'], basic_data['splits']) if s != 'test']
            orientation_ids = [oid for oid, s in zip(basic_data['orientation_ids'], basic_data['splits']) if s != 'test']
            dicom_ids = [did for s, did in zip(basic_data['splits'], basic_data['dicom_ids']) if s != 'test']
            splits = [s for s in basic_data['splits'] if s != 'test']
    else:
        report_ids = [rid for rid, s in zip(basic_data['report_ids'], basic_data['splits']) if s == 'test']
        image_paths = [ip for ip, s in zip(basic_data['image_paths'], basic_data['splits']) if s == 'test']
        orientation_ids = [oid for oid, s in zip(basic_data['orientation_ids'], basic_data['splits']) if s == 'test']
        dicom_ids = [did for s, did in zip(basic_data['splits'], basic_data['dicom_ids']) if s == 'test']
    assert len(report_ids) == len(image_paths) == len(orientation_ids) == len(dicom_ids)
    assert len(report_ids) > 0

    # Collect data in VQA format (multiple questions per report)
    self.report_ids = []
    self.question_ids = []
    self.dicom_ids = []
    self.images = []
    self.questions = []
    self.orientations = []
    if is_train_val:
        self.train_indices = []
        self.val_indices = []

    if questions_per_report is None: # Normal method
        self.answers = []
        idx = 0

        for i in tqdm(range(len(report_ids))):

            ri = report_ids[i]
            report = mimiccxr_qa_reports['reports'][ri]
            sentences = report['sentences']
            image_path = image_paths[i]
            orientation_id = orientation_ids[i]
            dicom_id = dicom_ids[i]
            split = splits[i]

            for qid, a_ids in report['qa'].items():
                qid = int(qid)
                question = question_list[qid]
                answer = '. '.join(sentences[i] for i in a_ids)
                self.report_ids.append(ri)
                self.question_ids.append(qid)
                self.dicom_ids.append(dicom_id)
                self.images.append(image_path)
                self.questions.append(tokenizer.string2ids(question.lower()))
                self.answers.append(answer_string2ids_func(answer.lower()))
                self.orientations.append(orientation_id)
                if is_train_val:
                    if split == 'train':
                        self.train_indices.append(idx)
                    elif split == 'validate':
                        self.val_indices.append(idx)
                    else:
                        raise ValueError(f'Unknown split: {split}')
                    idx += 1

    else: # Report evaluation method -> we ignore the answers

        filtered_idxs = _filter_report_orientation_pairs_for_test(report_ids, orientation_ids, self.eval_view_mode)

        for i in tqdm(filtered_idxs):
            ri = report_ids[i]
            image_path = image_paths[i]
            orientation_id = orientation_ids[i]
            dicom_id = dicom_ids[i]
            question_ids = questions_per_report[ri]

            # assert len(question_ids) > 0, mimiccxr_qa_reports['reports'][ri]

            for qid in question_ids:
                question = question_list[qid]
                self.report_ids.append(ri)
                self.question_ids.append(qid)
                self.dicom_ids.append(dicom_id)
                self.images.append(image_path)
                self.questions.append(tokenizer.string2ids(question.lower()))
                self.orientations.append(orientation_id)

    self.report_ids = np.array(self.report_ids, dtype=int)
    self.question_ids = np.array(self.question_ids, dtype=int)
    self.dicom_ids = np.array(self.dicom_ids, dtype=str)
    self.images = np.array(self.images, dtype=str)
    self.questions = np.array(self.questions, dtype=object)
    self.orientations = np.array(self.orientations, dtype=int)
    if is_train_val:
        self.train_indices = np.array(self.train_indices, dtype=int)
        self.val_indices = np.array(self.val_indices, dtype=int)
    if questions_per_report is None:
        self.answers = np.array(self.answers, dtype=object)

class MIMICCXR_VQA_Trainer(VQA_Trainer):

    def __init__(self, train_image_transform, val_image_transform,
                batch_size, collate_batch_fn,
                num_workers,
                qa_adapted_reports_filename,
                tokenizer,
                view_mode = MIMICCXR_ViewModes.ANY_SINGLE,
                collate_batch_fn_chexpert_mode = None,
                verbose_question = True,
                classify_tags = False,
                medical_tags_per_report_filename = None,
                classify_orientation = False,
                classify_chexpert = False,
                chexpert_labels_filename = None,
                classify_questions = False,
                classify_chest_imagenome = False,
                predict_bboxes_chest_imagenome = False,
                clamp_bboxes_chest_imagenome = False,
                question_labels_filename = None,
                mimiccxr_qa_reports = None,
                balanced_dataloading = False,
                balanced_metadata_filename = None,
                imbalance_reduction_coef = 1,
                allowed_questions = None,
                one_question_per_batch = False,
                include_mined_questions_mode = False,
                include_chexpert_mode = False,
                include_image = True,
                use_precomputed_visual_features = False,
                precomputed_visual_features_path = None,
                use_merged_findings = False,
                findings_remapper = None,
                n_findings = None,
                include_chest_imagenome_mode = False,
                chest_imagenome_labels_filename = None,
                chest_imagenome_label_names_filename = None,
                collate_batch_fn_chest_imagenome_mode = None,
                debug = False):

        self.tokenizer = tokenizer
        self.mimiccxr_qa_reports = mimiccxr_qa_reports
        self.qa_adapted_reports_filename = qa_adapted_reports_filename
        self.view_mode = view_mode
        self.include_chest_imagenome_mode = include_chest_imagenome_mode
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.classify_chest_imagenome = classify_chest_imagenome
        self.predict_bboxes_chest_imagenome = predict_bboxes_chest_imagenome
        self.chest_imagenome_label_names_filename = chest_imagenome_label_names_filename
        self.chest_imagenome_labels_filename = chest_imagenome_labels_filename
        self.use_chest_imagenome_compatible_data = (
            include_chest_imagenome_mode or classify_chest_imagenome or predict_bboxes_chest_imagenome
        )

        other_tasks = [] # Hack so that parent class can handle other tasks

        preprocessing_save_path = _get_train_preprocessing_save_path(qa_adapted_reports_filename, tokenizer, self.use_chest_imagenome_compatible_data)

        print(f'MIMICCXR_VQA_Trainer: balanced_dataloading = {balanced_dataloading}')

        super().__init__(train_image_transform, val_image_transform,
                        batch_size, collate_batch_fn,
                        preprocessing_save_path,
                        MIMICCXR_CACHE_DIR,
                        num_workers,
                        collate_batch_fn_chexpert_mode = collate_batch_fn_chexpert_mode,
                        verbose_question = verbose_question,
                        classify_tags = classify_tags,
                        rid2tags_filename = medical_tags_per_report_filename,
                        classify_orientation = classify_orientation,
                        classify_chexpert = classify_chexpert,
                        chexpert_labels_filename = chexpert_labels_filename,
                        classify_questions = classify_questions,
                        question_labels_filename = question_labels_filename,
                        balanced_dataloading = balanced_dataloading,
                        balanced_metadata_filename = balanced_metadata_filename,
                        imbalance_reduction_coef = imbalance_reduction_coef,
                        allowed_questions = allowed_questions,
                        qa_adapted_reports_filename = qa_adapted_reports_filename,
                        one_question_per_batch = one_question_per_batch,
                        include_mined_questions_mode = include_mined_questions_mode,
                        include_chexpert_mode = include_chexpert_mode,
                        include_image = include_image,
                        use_precomputed_visual_features = use_precomputed_visual_features,
                        precomputed_visual_features_path = precomputed_visual_features_path,
                        use_merged_findings = use_merged_findings,
                        findings_remapper = findings_remapper,
                        n_findings = n_findings,
                        other_tasks=other_tasks,
                        debug = debug)

        # --------- Chest-Imagenome specific logic ---------
        if include_chest_imagenome_mode:
            assert collate_batch_fn_chest_imagenome_mode is not None
            self.collate_batch_fn_chest_imagenome_mode = collate_batch_fn_chest_imagenome_mode
            # Load Chest-Imagenome label names and templates
            assert chest_imagenome_label_names_filename is not None
            self.chest_imagenome_label_names, self.chest_imagenome_templates = \
                load_chest_imagenome_label_names_and_templates(chest_imagenome_label_names_filename)

        if include_chest_imagenome_mode or classify_chest_imagenome:
            assert chest_imagenome_labels_filename is not None
            # Load Chest-Imagenome dicom_ids and labels
            self.chest_imagenome_dicom_ids, self.chest_imagenome_labels = \
                load_chest_imagenome_dicom_ids_and_labels_as_numpy_matrix(chest_imagenome_labels_filename, qa_adapted_reports_filename)
            # Load Chest-Imagenome gold standard dicom_ids (they must be removed from training and validation sets)
            self.chest_imagenome_gold_dicom_ids = set(load_gold_standard_dicom_ids())

        if classify_chest_imagenome:
            other_tasks.append(('chest_imagenome', lambda _, rid: self.chest_imagenome_labels[rid]))

        if predict_bboxes_chest_imagenome:
            self.dicom_idxs, self.bbox_coords, self.bbox_presence =\
                load_chest_imagenome_silver_bboxes_as_numpy_array(
                    self.dicom_ids, clamp_bboxes_chest_imagenome)
            other_tasks.append(('chest_imagenome_bbox_coords', lambda i, _: self.bbox_coords[self.dicom_idxs[i]]))
            other_tasks.append(('chest_imagenome_bbox_presence', lambda i, _: self.bbox_presence[self.dicom_idxs[i]]))

    def _preprocess_data(self):
        _preprocess_data(self, 'train_val')

    def _generate_dataset_and_dataloader__chest_imagenome_mode(
            self, indices, batch_size, collate_batch_fn, num_workers,
            infinite=True, n_pos_samples=None, n_neg_samples=None, min_pos_to_include=0):

        return self._create_label_based_dataset_and_dataloader(
            indices=indices,
            labels=self.chest_imagenome_labels,
            label_names=self.chest_imagenome_label_names,
            templates=self.chest_imagenome_templates,
            tokenizer=self.tokenizer,
            batch_size=batch_size,
            num_workers=num_workers,
            collate_batch_fn=collate_batch_fn,
            infinite=infinite,
            n_pos_samples=n_pos_samples,
            n_neg_samples=n_neg_samples,
            min_pos_to_include=min_pos_to_include,
            report_ids=self.report_ids,
            print_every=30,
            # break_loop_at_i=40, # for debugging
            create_dataset_kwargs=dict(
                fixed_qa_pair=True,
            ))

    def _generate_train_dataset_and_dataloader__chest_imagenome_mode(self, batch_size, collate_batch_fn, num_workers):
        print('Generating balanced train dataset in chest imagenome mode ...')
        flattened_indices = []
        for _indices in self.train_indices.values():
            flattened_indices.extend(_indices)
        dedup_indices = deduplicate_indices(flattened_indices, self.report_ids)
        dataset, dataloader = self._generate_dataset_and_dataloader__chest_imagenome_mode(
            dedup_indices, batch_size, collate_batch_fn, num_workers, min_pos_to_include=20)
        self.train_dataset__chest_imagenome_mode = dataset
        self.train_dataloader__chest_imagenome_mode = dataloader
        print('len(self.train_dataset__chest_imagenome_mode) =', len(self.train_dataset__chest_imagenome_mode))

    def _generate_val_dataset_and_dataloader__chest_imagenome_mode(self, batch_size, collate_batch_fn, num_workers):
        print('Generating balanced validation dataset in chest imagenome mode ...')
        dedup_indices = deduplicate_indices(self.val_indices, self.report_ids)
        dataset, dataloader = self._generate_dataset_and_dataloader__chest_imagenome_mode(
            dedup_indices, batch_size, collate_batch_fn, num_workers, infinite=False,
            n_pos_samples=5, n_neg_samples=5)
        self.val_dataset__chest_imagenome_mode = dataset
        self.val_dataloader__chest_imagenome_mode = dataloader
        print('len(self.val_dataset__chest_imagenome_mode) =', len(self.val_dataset__chest_imagenome_mode))

    def _load_optional_datasets_and_dataloaders(self):
        if self.include_chest_imagenome_mode:
            if not self.validation_only:
                self._generate_train_dataset_and_dataloader__chest_imagenome_mode(
                    self.batch_size, self.collate_batch_fn_chest_imagenome_mode, self.num_workers)
            if not self.train_with_all:
                self._generate_val_dataset_and_dataloader__chest_imagenome_mode(
                    self.batch_size, self.collate_batch_fn_chest_imagenome_mode, self.num_workers)

class MIMICCXR_VQA_Evaluator(VQA_Evaluator):

    def __init__(self, transform, batch_size, collate_batch_fn,
                qa_adapted_reports_filename,
                num_workers,
                verbose_question = True,
                classify_tags = False,
                medical_tags_per_report_filename = None,
                classify_orientation = False,
                classify_chexpert = False,
                chexpert_labels_filename = None,
                classify_questions = False,
                question_labels_filename = None,
                tokenizer = None,
                mimiccxr_qa_reports = None,
                report_eval_mode = None,
                image_local_feat_size = None,
                pretrained_checkpoint_path = None,
                pretrained_weights = None,
                precomputed_question_probs_path = None,
                precomputed_question_thresholds_path = None,
                n_questions_aux_task = None,
                n_questions_per_report = None,
                qclass_threshold = None,
                include_image = True,
                use_random_image = False,
                use_precomputed_visual_features = False,
                precomputed_visual_features_path = None,
                view_mode = None,
                eval_view_mode = None,
                include_chest_imagenome_mode = False,
                classify_chest_imagenome = False,
                chest_imagenome_labels_filename = None,
                **unused_kwargs):

        print('report_eval_mode =', report_eval_mode)

        self.tokenizer = tokenizer
        self.mimiccxr_qa_reports = mimiccxr_qa_reports
        self.qa_adapted_reports_filename = qa_adapted_reports_filename
        self.view_mode = view_mode
        self.include_chest_imagenome_mode = include_chest_imagenome_mode

        # Args used in report_eval_mode
        self.report_eval_mode = report_eval_mode
        self.transform = transform
        self.image_local_feat_size = image_local_feat_size
        self.n_questions_aux_task = n_questions_aux_task
        self.pretrained_weights = pretrained_weights
        self.batch_size = batch_size
        self.n_questions_per_report = n_questions_per_report
        self.qclass_threshold = qclass_threshold
        self.pretrained_checkpoint_path = pretrained_checkpoint_path
        self.precomputed_question_probs_path = precomputed_question_probs_path
        self.precomputed_question_thresholds_path = precomputed_question_thresholds_path
        self.eval_view_mode = eval_view_mode

        if pretrained_weights is not None:
            assert pretrained_checkpoint_path is not None

        if classify_chest_imagenome: # Chest-Imagenome specific logic
            assert chest_imagenome_labels_filename is not None
            # Load Chest-Imagenome dicom_ids and labels
            self.chest_imagenome_dicom_ids, self.chest_imagenome_labels = \
                load_chest_imagenome_dicom_ids_and_labels_as_numpy_matrix(chest_imagenome_labels_filename, qa_adapted_reports_filename)
            # Necessary hack so that parent classes can access chest_imagenome_labels
            other_tasks = [('chest_imagenome', lambda _, rid: self.chest_imagenome_labels[rid])]
        else:
            other_tasks = None

        preprocessing_save_path = get_test_preprocessing_save_path(
                        qa_adapted_reports_filename, tokenizer, report_eval_mode,
                        pretrained_checkpoint_path, precomputed_question_probs_path,
                        precomputed_question_thresholds_path,
                        n_questions_per_report, qclass_threshold, use_random_image,
                        eval_view_mode)

        super().__init__(transform, batch_size, collate_batch_fn,
                        preprocessing_save_path,
                        MIMICCXR_CACHE_DIR,
                        num_workers,
                        verbose_question = verbose_question,
                        include_answer = report_eval_mode == None,
                        classify_tags = classify_tags,
                        rid2tags_filename = medical_tags_per_report_filename,
                        classify_orientation = classify_orientation,
                        classify_chexpert = classify_chexpert,
                        chexpert_labels_filename = chexpert_labels_filename,
                        classify_questions = classify_questions,
                        question_labels_filename = question_labels_filename,
                        include_image = include_image,
                        use_random_image = use_random_image,
                        use_precomputed_visual_features = use_precomputed_visual_features,
                        precomputed_visual_features_path = precomputed_visual_features_path,
                        other_tasks = other_tasks)

    def _preprocess_data(self):
        _preprocess_data(self, 'test')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

#class TestDnaHelper:
def pattern_count(Text, Pattern):
    # fill in your function here
    count = 0
    patternLength = len(Pattern)
    for i in range(0, len(Text) - patternLength+1):
        if Pattern == Text[i: i + patternLength]:
            count += 1
    return count

def frequent_words(Text, k, t = 1):
    frequent_patterns = dict()
    search_len = len(Text) - k
    for i in range(0, search_len):
        pattern = Text[i: i+k] # the k-mer
        if pattern not in frequent_patterns:
            patt_count = pattern_count(Text, pattern)
            frequent_patterns[pattern] = patt_count

    max_freq_value = max(frequent_patterns.values())
    print('max freq: ' + str(max_freq_value))
    max_freq_array = [p for p, c in frequent_patterns.items() if c == max_freq_value and c >= t]

    return max_freq_array

def get_last_symbol(pattern):
    return pattern[-1]

def symbol_to_number(symbol):
    return {'A': 0, 'C': 1, 'G': 2, 'T': 3}[symbol]

def number_to_symbol(number):
    return {0:'A', 1:'C', 2:'G', 3:'T'}[number]

def get_prefix(pattern):
    #remove the last charater
    return pattern[0 : -1]

def pattern_to_number(pattern):
    if len(pattern) == 0:
        #recursion is done
        return 0

    last_symbol = get_last_symbol(pattern)
    prefix = get_prefix(pattern)
    result = 4 * pattern_to_number(prefix) + symbol_to_number(last_symbol)
   # print(str(last_symbol) + ' ' + str(result))
    return result

def number_to_pattern(index, k):
    if k == 1: return number_to_symbol(index)

    qr = divmod(index, 4)
    pattern_prefix = number_to_pattern(qr[0], k - 1)
    return pattern_prefix + number_to_symbol(qr[1])

def computing_frequencies(Text, k):

    frequencys = [0] * (4**k) # declare an array of zeros

    n = len(Text)
    for i in range(0, n - k + 1):
        pattern = Text[i:i+k]
        j = pattern_to_number(pattern)
        frequencys[j] += 1

    return frequencys

def reverse_complement(Pattern):
    result = ''
    for n in Pattern:
        if n == 'A': result += 'T'
        elif n == 'T': result += 'A'
        elif n == 'C': result += 'G'
        elif n == 'G': result += 'C'

    return result[::-1]

def pattern_matching(Pattern, Genome):
    pos_record = []
    patternLength = len(Pattern)
    search_len = len(Genome) - patternLength+1

    for i in range(0, search_len):
        if Pattern == Genome[i: i + patternLength]:
            pos_record.append(i)

    return pos_record

def clump_finding(genome, k, L, t):
    clump = [0] * (4**k) # declare an array of zeros

    text = genome[0: L]
    frequency_array = computing_frequencies(text, k)
    for i in range(0, len(frequency_array)):
        if frequency_array[i] >= t:
            clump[i] = 1

    for i in range(1, len(genome) - L):
        first_pattern = genome[i - 1: i - 1 + k]
        index = pattern_to_number(first_pattern)
        frequency_array[index] -= 1

        last_pattern = genome[i + L - k: i + L]
        index = pattern_to_number(last_pattern)
        frequency_array[index] += 1

        if frequency_array[index] >= t:
            clump[index] = 1

    frequency_patterns = []
    for i in range(0, len(clump) - 1):
        if clump[i] == 1:
            pattern = number_to_pattern(i, k)
            frequency_patterns.append(pattern)

    return frequency_patterns

    # BetterClumpFinding(Genome, k, t, L)
    #     FrequentPatterns ← an empty set
    #     for i ← 0 to 4k − 1
    #         Clump(i) ← 0
    #     Text ← Genome(0, L)
    #     FrequencyArray ← ComputingFrequencies(Text, k)
    #     for i ← 0 to 4k − 1
    #         if FrequencyArray(i) ≥ t
    #             Clump(i) ← 1
    #     for i ← 1 to |Genome| − L
    #         FirstPattern ← Genome(i − 1, k)
    #         index ← PatternToNumber(FirstPattern)
    #         FrequencyArray(index) ← FrequencyArray(index) − 1
    #         LastPattern ← Genome(i + L − k, k)
    #         index ← PatternToNumber(LastPattern)
    #         FrequencyArray(index) ← FrequencyArray(index) + 1
    #         if FrequencyArray(index) ≥ t
    #             Clump(index) ← 1
    #     for i ← 0 to 4k − 1
    #         if Clump(i) = 1
    #             Pattern ← NumberToPattern(i, k)
    #             add Pattern to the set FrequentPatterns
    #     return FrequentPatterns


# coding: utf-8
# for autocomplete
import re
import operator

from django import forms
from django.db import models
from django.utils.encoding import smart_str
from django.utils.safestring import mark_safe
from django.conf.urls.defaults import patterns
from django.db.models.query import QuerySet
from django.http import HttpResponse, HttpResponseNotFound
from django.conf import settings
from django.template.defaultfilters import striptags, force_escape

# ModelAdmin Mixin to enable autocompletion
# based and inspired by http://jannisleidel.com/2008/11/autocomplete-form-widget-foreignkey-model-fields/
class AutocompleteFKMixin(object):
    # Add an autocomplete search URL in the ModelAdmin
    def get_urls(self):
        urls = super(AutocompleteFKMixin, self).get_urls()
        search_url = patterns('', (r'^search/$', self.search))
        return search_url + urls
    # Base queryset
    def get_autocomplete_queryset(self, request, field_name):
        """ Returns the basic queryset to filter results for an autocomplete field `field_name`.
        If None is returned - all db objects are used for a queryset. """
        return None
    # Search view
    def search(self, request):
        """ Searches in the fields of the given related model and returns the
        result as a simple string to be used by the jQuery Autocomplete plugin """
        query         = request.GET.get('q', None)
        app_label     = request.GET.get('app_label', None)
        model_name    = request.GET.get('model_name', None)
        search_fields = request.GET.get('search_fields', None)
        field_name    = request.GET.get('field_name', None)
        related_fields = [field[:-3] for field in request.GET
            if field[-3:] == "_pk" and request.GET[field]]

        # remove all garbage and injections
        from django.template.defaultfilters import slugify
        search_fields = [f for f in search_fields.split(',') if f==slugify(f)]

        if search_fields and app_label and model_name and query:
            def construct_search(field_name):
                # use different lookup methods depending on the notation
                if field_name.startswith('^'):
                    return "%s__istartswith" % field_name[1:]
                elif field_name.startswith('='):
                    return "%s__iexact" % field_name[1:]
                elif field_name.startswith('@'):
                    return "%s__search" % field_name[1:]
                else:
                    return "%s__icontains" % field_name

            model = models.get_model(app_label, model_name)
            qs = self.get_autocomplete_queryset(request, field_name)
            if qs is None:
                qs = model._default_manager.all()
            for bit in query.split():
                or_queries = [models.Q(**{construct_search(
                    smart_str(field_name)): smart_str(bit)})
                        for field_name in search_fields]

                other_qs = QuerySet(model)
                other_qs.dup_select_related(qs)
                other_qs = other_qs.filter(reduce(operator.or_, or_queries))
                qs = qs & other_qs
            # filter out by related ids
            qs = qs.filter(**dict([(str(field), request.GET[field+"_pk"]) for field in related_fields]))
            # sort rows directly containing query be the first
            from django.utils.datastructures import SortedDict

            # we do not was to dig into __ relations here. Only first model fields go for smart sorting.
            smartsorted_fields = []
            for field_name in search_fields:
                # add qualifying table
                try:
                    model._meta.get_field_by_name(field_name)
                except:
                    continue
                smartsorted_fields.append(field_name)

            for field_name in smartsorted_fields:
                full_field_name = model._meta.db_table+"."+field_name
                qs = qs.extra(select=SortedDict([
                    ('_%s_match' % field_name,
                        "CASE WHEN lower("+full_field_name+")=lower(%s) THEN 1 ELSE 0 END"),
                    ('_%s_contains' % field_name,
                    # can not put percents here because of http://code.djangoproject.com/ticket/6400
                        "CASE WHEN "+full_field_name+" LIKE %s THEN 1 ELSE 0 END"),
                    ]), select_params = [query, '%'+query+'%'])
            ordering = [ ['-_%s_match' % field_name, '-_%s_contains' % field_name]
                for field_name in smartsorted_fields]
            # flatten list of sorting options
            ordering = sum(ordering, [])
            qs = qs.order_by(*ordering)
            qs = qs[:25]

            # sometimes we want to present suggestions in a special way
            def ajax_str(obj):
                if hasattr(obj, 'ajax_str') and callable(obj.ajax_str):
                    return obj.ajax_str()
                else:
                    return unicode(obj)

            data = ''.join([u'%s|%s\n' % (ajax_str(f), f.pk) for f in qs])
            return HttpResponse(data, mimetype='text/plain')
        return HttpResponseNotFound()

    # Change form field for the related search fields
    def formfield_for_dbfield(self, db_field, **kwargs):
        """
        Overrides the default widget for Foreignkey fields if they are
        specified in the related_search_fields class attribute.
        """
        if isinstance(db_field, models.ForeignKey) and hasattr(self, 'related_search_fields') \
                and db_field.name in self.related_search_fields:
            kwargs['widget'] = ForeignKeySearchInput(db_field.rel, self.related_search_fields[db_field.name])
        return super(AutocompleteFKMixin, self).formfield_for_dbfield(db_field, **kwargs)

class ForeignKeySearchInput(forms.TextInput):
    """
    A Widget for displaying ForeignKeys in an autocomplete search input
    instead in a ``select`` box.
    """
    class Media:
        css = {
            'all': ('admin-autocomplete/jquery.autocomplete.css',)
        }
        js = (
            # Breaks our current
        ''') % {
            'search_fields': ','.join(self.search_fields),
            'admin_media_prefix': admin_media_prefix,
            'model_name': self.rel.to._meta.module_name,
            'app_label': self.rel.to._meta.app_label,
            'label': label,
            'name': name,
            'extra_params': extra_params,
        }


# #Find the cube root of a perfect cube
# x = int(input('Enter an integer: '))
# ans = 0
# while ans**3 < abs(x):
#     #print("The value of the decrmenting function abs(x) - ans**3 is ", abs(x)- ans **3)
#     ans = ans +1
# if ans**3 != abs(x):
#     print(x, 'is not a perfect cube')
# else:
#     if x<0:
#         ans = -ans
#     print('Cube root of ',x, 'is', ans)

maxVal = int(input('Enter a positive integer:'))
i = 0
while i < maxVal:
    i += 1
print(i)

print('hello world')

IntroToProgramming - Guttag Book/Figures/Figure 3-1.py


def run():
   #Creating the varibles for the direction
  direction = str(input("Towards which direction should I paint (up, down, left or right)?"))
  #Creating the branches to show which direction the robot will be painting in
  if (direction == "up"):
    print("I am painting in the upward direction!")
  elif (direction == "down"):
    print("I am painting in the downward direction")
  elif (direction == "left"):
    print("I am painting in the left direction")
  elif (direction == "right"):
    print("I am painting in the right direction")
  else:
    print("You  have not selected any of the avaiable options..")

run()


# https://leetcode.com/problems/binary-tree-inorder-traversal/submissions/867446669/
from typing import Optional, List
from _structures import TreeNode

# Recursive approach
class Solution:
    def inorderTraversal(self, root: Optional[TreeNode]) -> List[int]:
        result = []
        self.inorder(root, result)
        return result

    def inorder(self, root: Optional[TreeNode], result: List[int]):
        if root is not None:
            self.inorder(root.left, result)
            result.append(root.val)
            self.inorder(root.right, result)

# Test
root = TreeNode.from_list([1,None,2,3])
result = Solution().inorderTraversal(root)
root.print()
print("result: ", result)


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import contextlib
import copy

from mozperftest.argparser import FLAVORS
from mozperftest.hooks import Hooks
from mozperftest.layers import Layers, StopRunError
from mozperftest.metrics import pick_metrics
from mozperftest.system import pick_system
from mozperftest.test import pick_test
from mozperftest.utils import MachLogger

SYSTEM, TEST, METRICS = 0, 1, 2

class MachEnvironment(MachLogger):
    def __init__(self, mach_cmd, flavor="desktop-browser", hooks=None, **kwargs):
        MachLogger.__init__(self, mach_cmd)
        self._mach_cmd = mach_cmd
        self._mach_args = dict(
            [(self._normalize(key), value) for key, value in kwargs.items()]
        )
        self.layers = []
        if flavor not in FLAVORS:
            raise NotImplementedError(flavor)
        for layer in (pick_system, pick_test, pick_metrics):
            self.add_layer(layer(self, flavor, mach_cmd))
        if hooks is None:
            # we just load an empty Hooks instance
            hooks = Hooks(mach_cmd)
        self.hooks = hooks

    @contextlib.contextmanager
    def frozen(self):
        self.freeze()
        try:
            # used to trigger __enter__/__exit__
            with self:
                yield self
        finally:
            self.unfreeze()

    def _normalize(self, name):
        if name.startswith("--"):
            name = name[2:]
        return name.replace("-", "_")

    def set_arg(self, name, value):
        """Sets the argument"""
        # see if we want to restrict to existing keys
        self._mach_args[self._normalize(name)] = value

    def get_arg(self, name, default=None, layer=None):
        name = self._normalize(name)
        marker = object()
        res = self._mach_args.get(name, marker)
        if res is marker:
            # trying with the name prefixed with the layer name
            if layer is not None and not name.startswith(layer.name):
                name = "%s_%s" % (layer.name, name)
                return self._mach_args.get(name, default)
            return default
        return res

    def get_layer(self, name):
        for layer in self.layers:
            if isinstance(layer, Layers):
                found = layer.get_layer(name)
                if found is not None:
                    return found
            elif layer.name == name:
                return layer
        return None

    def add_layer(self, layer):
        self.layers.append(layer)

    def freeze(self):
        # freeze args  (XXX do we need to freeze more?)
        self._saved_mach_args = copy.deepcopy(self._mach_args)

    def unfreeze(self):
        self._mach_args = self._saved_mach_args
        self._saved_mach_args = None

    def run(self, metadata):
        # run the system and test layers
        try:
            with self.layers[SYSTEM] as syslayer, self.layers[TEST] as testlayer:
                metadata = testlayer(syslayer(metadata))

            # then run the metrics layers
            with self.layers[METRICS] as metrics:
                metadata = metrics(metadata)
        except StopRunError:
            # ends the cycle but without bubbling up the error
            pass
        return metadata

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return


import logging
from docarray import Document
from pywebio import session, config, start_server
from pywebio.output import *
from pywebio.pin import *
import os, io, uuid
from discord import Webhook, RequestsWebhookAdapter, Embed, File
import urllib.parse

_discord_webhook_url = os.environ["DISCORD_WEBHOOK"]
_webhook = Webhook.from_url(_discord_webhook_url, adapter=RequestsWebhookAdapter())
os.environ["JINA_HUB_ROOT"] = "/tmp/.jina"

server_url = 'grpc://dalle-flow.jina.ai:51005'

file_dir = os.path.dirname(os.path.abspath(__file__))

server_error_text = 'The demo server returned an error. Please come back in 5 mins and try again.'
image_gen_text = 'Generating images...\nThanks to JinaAI for providing the image generation service to public for free. This process may take up to 10 mins. Please be patient.'
step1 = "Step 1/3: Write down what you want to draw, and hit the `Draw` button. Up to 16 candidate images will be generated."
step2 = "Step 2/3: Click on your favorite image to fine tune it."
step3 = "Step 3/3: Choose one to generate a high-res image (1024 x 1024 px)"
credits = r'''### App source code:
https://github.com/pywebio/demos/tree/main/jina_dalle-flow

### Generated Art Gallery
Arts generated here are shared with PyWebIO community on our Discord server. Come see it: https://discord.gg/MvaCcg76Z7

**User privacy is fully respected as the sharing is anonymous.**

### Credits
This app is inspired by [JinaAI's DALLÂ·E Flow project](https://github.com/jina-ai/dalle-flow): A Human-in-the-Loop workflow for creating HD images from text.

**JinaAI**
* [Dalle-flow](https://github.com/jina-ai/dalle-flow)
* [Clip as service](https://github.com/jina-ai/clip-as-service)

**Boris Dayma**
* [DALLÂ·E Mini](https://github.com/borisdayma/dalle-mini)

**Jack Qiao**
* [Glid-3-xl](https://github.com/Jack000/glid-3-xl)

**Jingyun Liang**
* [SwinIR](https://github.com/JingyunLiang/SwinIR)
'''

def send_discord_result(prompt, image_data, stage):
    filename = uuid.uuid4().hex + '.png'
    file_path = os.path.join(file_dir, 'temp/', filename)

    if stage == 'preview':
        discord_msg = 'ðŸŽ¨ New art has just been created on https://demo.pyweb.io/live/jina_dalle-flow/app/ from a text input: `' + prompt + '`\n'
    else:
        discord_msg = 'After fine-tuning, the final result of `' + prompt + ':`\n'

    msg = Embed(description=discord_msg, color=10312083)

    data = image_data.split(',')[1]
    b = urllib.parse.unquote_to_bytes(data)
    bo = io.BytesIO(b)

    with open(file_path, "wb+") as f:
        f.write(bo.getbuffer())

    with open(file_path, "rb") as f:
        my_file = File(f)

    _webhook.send(embed=msg)
    _webhook.send(file=my_file)

@use_scope('images', clear=True)
def upscale_gen(prompt, diffused, dfav_id):
    global server_url
    fav = diffused[dfav_id]
    send_discord_result(prompt, diffused[dfav_id, 'uri'], 'diffused')
    put_text(image_gen_text)
    try:
        with put_loading():
            fav = fav.post(f'{server_url}/upscale', target_executor='upscaler')
        clear()
        image_data = fav.uri
        put_image(image_data)
    except Exception as e:
        logging.exception(e)
        clear()
        popup(server_error_text)

@use_scope('images', clear=True)
def diffused_gen(prompt, da, fav_id):
    global server_url
    fav = da[fav_id]
    send_discord_result(prompt, da[fav_id, 'uri'], 'preview')
    put_text(image_gen_text)
    try:
        with put_loading():
            diffused = fav.post(f'{server_url}', parameters={'skip_rate': 0.5, 'num_images': 9},
                                target_executor='diffusion').matches
        clear()
        put_text(step3)
        for i in range(0, 9):
            image_data = diffused[i, 'uri']
            put_image(image_data).onclick(lambda x=i: upscale_gen(prompt, diffused, x))
    except Exception as e:
        logging.exception(e)
        clear()
        popup(server_error_text)

@use_scope('images', clear=True)
def preview_image_gen(prompt):
    global server_url
    if prompt.replace(" ", "") == '':
        toast('Please input a valid sentence', color='error', duration=10)
    else:
        put_text(image_gen_text)
        try:
            with put_loading():
                da = Document(text=prompt).post(server_url, parameters={'num_images': 8}).matches
            clear()
            put_text(step2)
            for i in range(0, 16):
                image_data = da[i, 'uri']
                put_image(image_data).onclick(lambda x=i: diffused_gen(prompt, da, x))
        except Exception as e:
            logging.exception(e)
            clear()
            popup(server_error_text)

css = """
#pywebio-scope-images {
    height: calc(100vh - 150px);
    overflow-y: hidden;
}
#pywebio-scope-images:hover {
    overflow-y: scroll;
}
#pywebio-scope-input {
    height: calc(100vh - 150px);
    overflow-y: hidden;
}
#pywebio-scope-input:hover {
    overflow-y: scroll;
}
/* Works on Firefox */
* {
  scrollbar-width: thin;
}
/* Works on Chrome, Edge, and Safari */
*::-webkit-scrollbar {
  width: 7px;
}
*::-webkit-scrollbar-track {
  background: transparent;
}
*::-webkit-scrollbar-thumb {
  background-color: gray;
  border-radius: 20px;
  border: 2px
}
"""

@config(theme="minty", css_style=css)
def main():
    session.set_env(title='Dall-E Flow Web App', output_max_width='100%')

    put_markdown('# Dall-E Flow Web App')
    put_row(
        [put_scope('input'), None, put_scope('images')],
        size="2fr 40px minmax(60%, 6fr)",
    )

    with use_scope('input'):
        put_textarea('description',
            placeholder='E.g., An oil painting of a humanoid robot playing chess in the style of Matisse',
            rows=5,
        ),
        put_button('Draw', onclick=lambda: preview_image_gen(pin['description']))
        put_markdown(credits).style('font-size: 12px; color: gray; background: #f6f6f6;')

    with use_scope('images'):
        put_markdown('### How it works')
        for step in [step1, step2, step3]:
            put_text(step)

if __name__ == '__main__':
    start_server(main, debug=True, port=9999)


# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
from odoo import models, fields, api, _

class JournalEntry(models.Model):
    _name = 'account.move'
    _inherit = ['account.move', 'mail.thread']

    @api.model
    def create(self, vals):
        move = super(JournalEntry, self).create(vals)
        move.log_message()
        return move

    def log_message(self):
        def _format_message(message_description, tracked_values):
            message = ''
            if message_description:
                message = '%s' % message_description
            for name, values in tracked_values.items():
                message += '
     • %s: ' % name
                message += '%s
' % values
            return message

        for line in self:
            if line:
                msg_values = {
                    'Created by': self.create_uid.name,
                    'Created on': self.create_date.strftime('%Y-%m-%dT%H:%M:%S'),
                    'Last Updated on': self.write_date.strftime('%Y-%m-%dT%H:%M:%S'),
                    'Last Updated by': self.write_uid.name}
                msg = _format_message(_('New Journal Entry Created.'), msg_values)
                line.message_post(body=msg)

class AccountMoveLine(models.Model):
    _inherit = 'account.move.line'

    partner_id = fields.Many2one('res.partner', string='Partner', ondelete='restrict', tracking=True)
    analytic_account_id = fields.Many2one('account.analytic.account', string='Analytic Account',
        index=True, compute="_compute_analytic_account_id", store=True, readonly=False, check_company=True, copy=True,tracking=True)

    amount_currency = fields.Monetary(string='Amount in Currency', store=True, copy=True,tracking=True,
        help="The amount expressed in an optional other currency if it is a multi-currency entry.")
    debit = fields.Monetary(string='Debit', default=0.0, currency_field='company_currency_id',tracking=True)

    credit = fields.Monetary(string='Credit', default=0.0, currency_field='company_currency_id',tracking=True)
    tax_ids = fields.Many2many(
    comodel_name='account.tax',
    string="Taxes",
    context={'active_test': False},
    check_company=True,
    tracking=True,
    help="Taxes that apply on the base amount")


import numpy as np

import options

"""
Compute derived variables for all the models in the order that is defined by each model
in the function "derivedVariablesDependsOn".
"""
def recurseModelDependencies(t, obj, currentDepth, models, visited):
  if obj in visited:
    return visited

  visited.append(obj)
  dependsOn = obj["model"].derivedVariablesDependsOn(models)

  for dependency in dependsOn:
    recurseModelDependencies(t, dependency, currentDepth + 1, models, visited)

  if not obj["simplifiedInit"] is None:
    mask, args = obj["simplifiedInit"]
    obj["derived"] = obj["model"].computeSimplifiedDerivedVariables(args, t)
  else:
    obj["derived"] = obj["model"].computeDerivedVariables(t, obj["state"], models)

  return visited

from solve.states import applyModelStates, collectModelStates

def applyDerivedVariablesToResult(t, y, models):

  for key in models:
    models[key]["derivedResult"] = np.zeros((len(models[key]["derived"]), len(t)))

    # Recover derived variables
  for i in range(len(t)):
    if options.printTime:
      print(t[i])
    applyModelStates(models, y[:,i])

    # Set up derived variables
    visited = []
    for key in models:
      visited = recurseModelDependencies(t[i], models[key], 0, models, visited)

    for key in models:
      models[key]["derivedResult"][:, i] = models[key]["derived"]

  applyModelStates(models, y)


from setuptools import setup
from setuptools import find_packages

PACKAGES = find_packages(where=".")

setup(
    name="clif",
    version="0.2.2",
    description="CLImate Fingerprinting methods",
    long_description="",
    url="",
    author=["Kenny Chowdhary", "Jake Nichol"],
    author_email="kchowdh@sandia.gov",
    license="BSD3",
    packages=PACKAGES,
    package_data={"clif": ["folder/*.txt"]},
    test_suite="nose.collector",
    tests_required=["nose"],
    install_requires=[
        "numpy",
        "scikit-learn",
        "tqdm",
        "xarray",
        "netCDF4",
        "statsmodels",
        "cftime",
        "matplotlib",
        "dask",
    ],
    include_package_data=True,
    zip_safe=False,
)


"""Utilities for handling language tokens."""

from tiktoken import encoding_for_model, get_encoding

from readmeai.core import logger

logger = logger.Logger(__name__)

def adjust_max_tokens(
    max_tokens: int, prompt: str, target: str = "Hello!"
) -> int:
    """Adjust the maximum number of tokens based on the specific prompt."""
    is_valid_prompt = prompt.strip().startswith(target.strip())
    adjusted_max_tokens = max_tokens if is_valid_prompt else max_tokens // 5
    return adjusted_max_tokens

def get_token_count(text: str, encoding_name: str) -> int:
    """Returns the number of tokens in a text string."""
    encoding = get_encoding(encoding_name)
    num_tokens = len(encoding.encode(text))
    return num_tokens

def get_token_encoder() -> str:
    """The token encoder to use for the model."""
    return (
        encoding_for_model("gpt-3.5-turbo")
        if "gpt"
        else get_encoding("cl100k_base")
    )

def truncate_tokens(text: str, max_tokens: int) -> str:
    """Truncate a text string to a maximum number of tokens."""
    if not text:
        return text
    try:
        encoder = get_token_encoder()

        prompt_token_total = len(encoder.encode(text))
        if prompt_token_total <= max_tokens:
            return text

        char_total = len(text)
        chars_per_token = char_total / prompt_token_total
        truncated_total = int(chars_per_token * max_tokens)
        return text[:truncated_total]

    except Exception as excinfo:
        logger.error(f"Error truncating tokens: {excinfo}")
        return text


# PROGRAMMER: Humberto Jimenez
# DATE CREATED: 21 Feb 2021
#Purpose: to create a model training object that acts as an image classifier.
#Parts of this model are inspired directly from the pytorch exercises taught in the Udacity sessions,
#I have added a note to indicate where parts of the code is similar to what was taught, any similarities with other student's work is purely coincidental
#as we have all learned from the same source (examples and videos).

#basic imports
import argparse
import numpy as np
from PIL import Image
import json
import os
import random

#torch and torchvision import
import torch
from torchvision import transforms, models
import torch.nn.functional as F

from tools import load_checkpoint, load_cat_names

#Define arguments loader
#Learning source https://pymotw.com/3/argparse/
def parse_args ():
    parser = argparse.ArgumentParser(description = "Image classifier calculations")
    parser.add_argument('checkpoint', action='store', default='checkpoint.pth')
    parser.add_argument('--topk', dest='topk', default='5', type=int)
    parser.add_argument('--filepath', dest='filepath', type=str)
    parser.add_argument('--category_names', dest='category_names', default='cat_to_name.json')
    parser.add_argument('--gpu', action='store', default='gpu') #Added gpu/cpu argument
    return parser.parse_args()

#Define image processor
#This section was inspired by browsing the web and with the aid of this learnign source: https://pillow.readthedocs.io/en/latest/reference/Image.html
def process_image(image):
    #Open the image using PIL
    pil_image = Image.open(image)

    #perform transformations to make it suitable for the model
    adjust_image = transforms.Compose([transforms.Resize(256),
                                      transforms.CenterCrop(224),
                                      transforms.ToTensor(),
                                      transforms.Normalize([0.485, 0.456, 0.406],
                                                           [0.229, 0.224, 0.225])
                                      ])

    #adjust image using the transformations
    image = adjust_image(pil_image)

    return image

#Define predictor
def predict(image_path, model, topk):
    #Invoke the arguments
    args = parse_args() #Adding to fix args not defined issue
    #move the model to the gpu environment
    device = 'cuda' if torch.cuda.is_available() and args.gpu else 'cpu' #readjusted the cuda statement
    model.to(device)

    # transform the image by running it through the process_image method,
    #unsqueeze to return the tensor with an extra dimension and convert it to float.
    # Reference for unsqueeze method: https://pytorch.org/docs/stable/generated/torch.unsqueeze.html

    image = torch.unsqueeze(process_image(image_path),0).float()

    #Enable no grad to prevent a backward step and reduce memory consumption
    with torch.no_grad():
        #Send image and label to the gpu/device
        image = image.to(device)
        #push the image into the network
        logps = model.forward(image)
    #Calculate the probability using the softmax function
    probability = F.softmax(logps.data, dim=1)
    #Create the array of top probabilities
    probs = np.array(probability.topk(topk)[0][0])

    #Determine top classes and create an array with the top possible answers
    index_to_class = {val: key for key, val in model.class_to_idx.items()}
    top_classes = [np.int(index_to_class[each]) for each in np.array(probability.topk(topk)[1][0])]

    return probs, top_classes

#Define main part of the program to run
def main():
    #invoke all arguments for the function
    args = parse_args()
    model = load_checkpoint(args.checkpoint)
    cat_to_name = load_cat_names(args.category_names)

    #Choose a random image from the test folder #10 if the user does not specify a file
    if args.filepath is None:
        test_image = random.choice(os.listdir('flowers/test/10/'))
        test_img_path= 'flowers/test/10/'+ test_image
    else:
        test_img_path = args.filepath

    #Calculate probability and classes using model
    probs, classes = predict(test_img_path, model, args.topk) #Added the topk to allow the user to choose their no. of classes
    labels = [cat_to_name[str(index)] for index in classes]

    #Print results to display file, class labels and probability
    print ('selected file:' + test_img_path)
    print('the image is likely to be a: {} with a confidence interval of: {:.3f} %'.format(cat_to_name[str(classes[0])] , probs[0]))
    print('\n other alternatives are: \n {} \n with {} probability'.format(labels, probs))

if __name__ == "__main__":
    main()