from __future__ import unicode_literals

from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.core.exceptions import ValidationError

import json
import jsonschema
import os
import time
import uuid
import psycopg2
import datetime
import re
import errno
import fcntl
import boto3

from osgeo import gdal, osr, ogr
from pyproj import Transformer

import logging
from . import settings
logging.config.dictConfig( settings.DEFAULT_LOGGING )
logger = logging.getLogger( 'debug' )
gdal.SetConfigOption( 'GDAL_PAM_ENABLED', 'NO' )
gdal.SetConfigOption("GDAL_DISABLE_READDIR_ON_OPEN","EMPTY_DIR")
METADATA_SCHEMA = os.path.join( os.path.dirname(__file__), 'metadata_loader_schema.json' )

@csrf_exempt
def get_spec(request):
    if request.method != "GET":
        return JsonResponse(status=405)
    try:
        datasetId=request.GET.get("datasetId")
    except Exception as e:
        print(e)
        return JsonResponse({"status":400,"message":"datasetId required"},status=400)

    try:
        with open(settings.DATASETS_SPEC) as json_file:
            data = json.load(json_file)[datasetId]
        return JsonResponse(data,status=200)
    except Exception as ex:
        return JsonResponse({"status":200,"message":"datasetId specification not available"},status=200)

@csrf_exempt
def upload( request ):
    settings.init_environ()
    """
    Upload raster on the system
    0) check json and data
    1) temporary store raster
    2) register raster to catalogue
    3) move raster to mwcs
    """
    logger.info( "Data upload started" )
    #check method
    files = None
    remote_files = None

    if request.method != 'POST':
        return JsonResponse( { "status":405, "error": "Method %s Allowed" %( request.method ) }, status=405 )

    #check file
    try:
        files = request.FILES
        logger.debug( "Post Raster OK" )
    except Exception as e:
        return JsonResponse( { "status":500, "error": "Raster upload error" }, status=500 )

    if not files:
        try:
            metadata_source = json.loads(request.POST.get("metadata"))
            for feature_id in metadata_source:
                remote_files = metadata_source[feature_id]["source"]
            logger.debug( "Post Raster filename OK" )
        except Exception as e:
            print(e)
            return JsonResponse( { "status":500, "error": "Raster filename upload error" }, status=500 )

    #check metadata
    try:
        if files:
            metadata = json.loads( request.POST.get( 'metadata' ) )
            logger.debug( "Post metadata json OK" )
        else:
            metadata = json.loads(request.POST.get("metadata"))
    except:
        return JsonResponse( { "status":500, "error": "Unable to load posted metadata json" }, status=500 )

    #initialize database connection
    try:
        conn = psycopg2.connect( settings.catalogue_db )
        cur = conn.cursor()
        logger.debug( "Metadata database connection OK" )
    except Exception as e:
        return JsonResponse( { "status":500, "error": "Unable to establish connection with metadata database" }, status=500 )

    response = {}
    describeCoverageTmp = {}

    if files:
        for ff in files:
            dest = None
            original_identifier = None
            logger.info( "Start management of %s" %( files[ff].name ) )
            response[ ff ] = {}
            try:
                start = time.time()
                #::::::::::: 0) verify data :::::::::::
                if not ff in metadata:
                    raise KeyError( "key %s not found in metadata dict" %( ff ) )
                json_schema = json.load( open( METADATA_SCHEMA, 'r' ) )
                validator = jsonschema.Draft7Validator( json_schema )
                for error in validator.iter_errors( metadata[ ff ] ):
                    if error.validator == 'required':
                        raise ValidationError( error.message )
                    else:
                        path = list( error.path )[0]
                        raise ValidationError( "%s - %s" %( path, error.message ) )
                try:
                    identifier = metadata[ ff ][ "identifier" ]
                except:
                    logger.exception("CREATING NEW UUID")
                    identifier = str( uuid.uuid1() )
                landsat_row, landsat_path, wrs, sentinel_mgrs, mgrs, mwcs_grid = _getGrid( metadata[ ff ], cursor=cur )
                logger.debug( "0 - %s validation OK" %( files[ff].name,  ) )

                #::::::::::: 1) store dataset :::::::::::
                dest = os.path.join( settings.TMP_STORAGE_DIR, identifier )
                with open( dest, 'wb+' ) as destination:
                    for chunk in files[ff].chunks():
                        destination.write( chunk )
                response[ ff ][ "tmp_file" ] = dest
                logger.debug( "1 - %s upload OK" %( files[ff].name ) )

                #::::::::::: 2) read dataset ::::::::::::
                raster = ImageUtils( response[ ff ][ "tmp_file" ] )
                if _getKey( metadata[ ff ], "geometry", default="none") is not "none":
                    try:
                        raster.setPolygon( _getKey( metadata[ ff ], "geometry" ) )
                    except Exception as ex :
                        raise ValidationError( "geometry - unable to convetr geojson geometry" )
                logger.debug( "2 - %s raster validity ok" %( files[ff].name ) )

                #::::::::::: 3) check identifier and update ::::::::
                cur.execute( "SELECT identifier, mwcs_path FROM records WHERE identifier='%s'" %( identifier ) )
                row = cur.fetchone()
                if row is not None:
                    response[ ff ][ "status" ] = "UPDATE"
                    if row[1] is not None:
                        absolute_mwcs_path = os.path.join( settings.WCS_DIR, row[1] )
                        if os.path.isfile( absolute_mwcs_path ):
                            logger.debug( "backup existing raster %s for revert" %( absolute_mwcs_path ) )
                            raster_backup = absolute_mwcs_path + ".BCK"
                            os.rename( absolute_mwcs_path, raster_backup )
                    backup_identifier = str( uuid.uuid1() ) + "BCK"
                    cur.execute( "UPDATE records SET identifier='%s' WHERE identifier='%s'" %( backup_identifier, identifier ) )
                    conn.commit()
                    logger.debug( "Metadata backup. New_uid='%s' - Old_uid='%s'" %( backup_identifier, identifier ) )
                    original_identifier = identifier
                else:
                    response[ ff ][ "status" ] = "INSERT"
                logger.debug( "3 - %s identifier OK" %( files[ff].name ) )

                #::::::::::: 4) MWCS store :::::::::::::::
                mwcs_path = os.path.join(
                    metadata[ ff ][ 'datasetId' ],
                    mwcs_grid,
                    "Y%s" %( metadata[ ff ][ 'date' ][ 0:4 ] ),
                    "M%s" %( metadata[ ff ][ 'date' ][ 5:7 ] ),
                    "D%s" %( metadata[ ff ][ 'date' ][ 8:10 ] ),
                    "T%s" %( metadata[ ff ][ 'date' ][ 11:19 ].replace( ":", "") ),
                    "E%s" %( raster.AuthorityCode ),
                    raster.getMwcsGeom(),
                    identifier
                )
                response[ ff ][ "mwcs_path" ] = mwcs_path
                abs_mwcs_path = os.path.join( settings.WCS_DIR, mwcs_path )
                try:
                    os.makedirs( os.path.dirname( abs_mwcs_path ) )
                except OSError as exception:
                    if exception.errno != errno.EEXIST:
                        error_mess = "Unable to create directory %s" %( os.path.dirname( abs_mwcs_path ) )
                        logger.exception( error_mess )
                        raise OSError( error_mess )
                #:::::::::::::::::::::::::::::::: 4.1) upload data on s3 :::::::::::::::::::::::::::::::
                os.rename( response[ ff ][ "tmp_file" ], abs_mwcs_path )

                describeCoverageTmp = _updateDescribeCoverage( describeCoverageTmp, raster, metadata[ ff ] )
                logger.debug( "%s describe coverage update" %( abs_mwcs_path ) )
                del( response[ ff ][ "tmp_file" ] )
                logger.debug( "4 - %s mwcs OK [ %s ]" %( files[ff].name, abs_mwcs_path ) )

                #::::::::::: 5) pycsw catalogue :::::::::::
                typename = "csw:Record"
                schema = "http://www.opengis.net/cat/csw/2.0.2"
                mdsource = "local"
                insert_date = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
                xml = ""
                _type = "http://purl.org/dc/dcmitype/Dataset"
                title = _getKey( metadata[ ff ], "datasetId" )
                title_alternate = _getKey( metadata[ ff ], "title" )
                keywords = _getKey( metadata[ ff ], "keywords" )
                try:
                    time_begin = _getKey( metadata[ ff ], "timeExtent" ).split( "/" )[0]
                    time_end = _getKey( metadata[ ff ], "timeExtent" ).split( "/" )[1]
                except:
                    time_begin = ""
                    time_end = ""
                date = _getKey( metadata[ ff ], "date" )
                date_modified = _getKey( metadata[ ff ], "creationDate" )
                _format = "image/tiff"
                sourcerastergeo= _getSourceRasterGeo( metadata[ ff ], identifier )
                source=metadata[ff]["source"]
                wkt_geometry = raster.getPolygon()
                anytext = ' '.join( [ identifier, title, title_alternate, date, date_modified, time_begin, time_end ] + re.split( "[, \-!?:_]+", title_alternate ) + re.split( "[, \-!?:_]+", title ) )
                abstract_dict = _getGrid( metadata[ ff ],  of="dict" )
                abstract_dict[ "Identifier" ] = identifier
                abstract_dict[ "Data Type" ] = raster.datatype
                abstract_dict[ "Start" ] = time_begin
                abstract_dict[ "End" ] = time_end
                abstract = _getAbstract( abstract_dict )
                metadata[ff]["identifier"]=identifier
                metadata[ff]["minValue"]=raster.min
                metadata[ff]["maxValue"]=raster.max
                metadata[ff]["noDataValue"]=raster.nodata
                metadata[ff]["minDate"]=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime(int(describeCoverageTmp["%s_%s"%(metadata[ ff ]["datasetId"],raster.coverageId())]["t_min"])))
                metadata[ff]["maxDate"]=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime(int(describeCoverageTmp["%s_%s"%(metadata[ ff ]["datasetId"],raster.coverageId())]["t_max"])))
                metadata[ff]["EPSG"]=raster.AuthorityCode
                metadata[ff]["source"]=metadata[ff]["path"]
                del(metadata[ff]["path"])

                new_record = """
                INSERT INTO records ( identifier, typename, schema, mdsource, insert_date, xml, anytext, type,title, title_alternate, abstract, keywords, time_begin, time_end, date,date_modified, format, sourcerastergeo,source, wkt_geometry,landsat_row, landsat_path, wrs, sentinel_mgrs, mgrs,mwcs_path, json_metadata )
                VALUES ( '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s','%s', '%s', %s, %s, %s, %s, %s, '%s', '%s' )
                """%(
                    identifier, typename, schema, mdsource, insert_date, xml, anytext, _type, title, title_alternate, abstract, keywords, time_begin, time_end, date, date_modified, _format, sourcerastergeo,source, wkt_geometry,
                    landsat_row, landsat_path, wrs, sentinel_mgrs, mgrs,
                    mwcs_path, json.dumps( metadata[ ff ] )
                )
                cur.execute( new_record )
                conn.commit()
                logger.debug( '5 - Image "%s" registered successfully on catalogue with id "%s"' %( abs_mwcs_path, identifier ) )
                response[ ff ][ "wcs_url" ] = sourcerastergeo
                response[ ff ][ "identifier" ] = identifier
                response[ ff ][ "wcs_server" ] = settings.WCS_BASE_PATH

                #::::::::::::::::: 6) FINISH ::::::::::::
                if response[ ff ][ "status" ] == "UPDATE":
                    try:
                        os.remove( raster_backup )
                        logger.debug( "6 - remove old raster backup %s" %( raster_backup  ) )
                    except:
                        None
                    try:
                        cur.execute( "DELETE FROM records WHERE identifier='%s'" %( backup_identifier ) )
                        conn.commit()
                        logger.debug( "6 - remove old metadata from DB. old identifier=%s" %( backup_identifier ) )
                    except:
                        None
                logger.info( 'Registration of %s completed in %f s'%( files[ff].name, time.time() - start ) )

            except Exception as ex:
                logger.exception(ex)
                try:
                    os.remove( response[ ff ][ "tmp_file" ] )
                    logger.error( "REVERT - remove uploaded file %s" %( response[ ff ][ "tmp_file" ] ) )
                except:
                    None
                try:
                    os.rename( raster_backup, raster_backup.strip( ".BCK" ) )
                    logger.error( "REVERT - restore backuped raster %s" %( raster_backup.strip( ".BCK" ) ) )
                    response[ ff ][ "restored_mwcs_path" ] = raster_backup.strip( ".BCK" )
                except:
                    None
                try:
                    cur.execute( "DELETE FROM records WHERE identifier='%s';" %( original_identifier ) )
                    cur.execute( "UPDATE records SET identifier='%s' WHERE identifier='%s';" %( original_identifier, backup_identifier ) )
                    conn.commit()
                    logger.debug( "Metadata backup restored with identifier '%s'" %( original_identifier ) )
                    response[ ff ][ "restored_identifier" ] = original_identifier
                except:
                    None
                logger.exception( "File upload failed.")
                if 'mwcs_path' in response[ ff ]:
                    del( response[ ff ][ 'mwcs_path' ] )
                if 'identifier' in response[ ff ]:
                    del( response[ ff ][ 'identifier' ] )
                if 'wcs_url' in response[ ff ]:
                    del( response[ ff ][ 'wcs_url' ] )
                response[ ff ][ "error" ] = str( ex )
            response[ ff ][ "time" ] = time.time() - start

        conn.close()
    elif remote_files is not None:
        try:
            start = time.time()
            for feature_id in metadata_source:
                dest = None
                original_identifier = None
                logger.info( "Start management of %s" %( feature_id ) )
                response[ feature_id ] = {}
                json_schema = json.load( open( METADATA_SCHEMA, 'r' ) )
                validator = jsonschema.Draft7Validator( json_schema )
                for error in validator.iter_errors( metadata[ feature_id ] ):
                    if error.validator == 'required':
                        raise ValidationError( error.message )
                    else:
                        path = list( error.path )[0]
                        raise ValidationError( "%s - %s" %( path, error.message ) )

                try:
                    identifier = feature_id
                except:
                    identifier = str( uuid.uuid1() )
                logger.debug( "0 - %s validation OK" %( feature_id  ) )
                if metadata[feature_id]["geolocated"]:
                    landsat_row, landsat_path, wrs, sentinel_mgrs, mgrs, mwcs_grid = _getGrid( metadata[ feature_id ], cursor=cur )

                    raster = ImageUtils( metadata[feature_id]["source"] )
                    raster.setPolygon( metadata[feature_id]["geometry"] )
                    #::::::::::: 1) store dataset :::::::::::
                    dest = os.path.join( settings.TMP_STORAGE_DIR, identifier+".vrt" )
                    translateopt= gdal.TranslateOptions(format="VRT")
                    gdal.Translate(dest,metadata[feature_id]["source"],options=translateopt)
                    response[ feature_id ][ "tmp_file" ] = dest
                    logger.debug( "1 - %s upload OK" %( feature_id ) )

                    #::::::::::: 3) check identifier and update ::::::::
                    cur.execute( "SELECT identifier, mwcs_path FROM records WHERE identifier='%s'" %( identifier ) )
                    row = cur.fetchone()
                    if row is not None:
                        response[ feature_id ][ "status" ] = "UPDATE"
                        if row[1] is not None:
                            absolute_mwcs_path = os.path.join( settings.WCS_DIR, row[1] )
                            if os.path.isfile( absolute_mwcs_path ):
                                logger.debug( "backup existing raster %s for revert" %( absolute_mwcs_path ) )
                                raster_backup = absolute_mwcs_path + ".BCK"
                                os.rename( absolute_mwcs_path, raster_backup )
                        backup_identifier = str( uuid.uuid1() ) + "BCK"
                        cur.execute( "UPDATE records SET identifier='%s' WHERE identifier='%s'" %( backup_identifier, identifier ) )
                        conn.commit()
                        logger.debug( "Metadata backup. New_uid='%s' - Old_uid='%s'" %( backup_identifier, identifier ) )
                        original_identifier = identifier
                    else:
                        response[ feature_id ][ "status" ] = "INSERT"
                    logger.debug( "3 - %s identifier OK" %( feature_id ) )

                    #::::::::::: 4) MWCS store :::::::::::::::
                    mwcs_path = os.path.join(
                        metadata[ feature_id ][ 'datasetId' ],
                        mwcs_grid,
                        "Y%s" %( metadata[ feature_id ][ 'date' ][ 0:4 ] ),
                        "M%s" %( metadata[ feature_id ][ 'date' ][ 5:7 ] ),
                        "D%s" %( metadata[ feature_id ][ 'date' ][ 8:10 ] ),
                        "T%s" %( metadata[ feature_id ][ 'date' ][ 11:19 ].replace( ":", "") ),
                        "E%s" %( raster.AuthorityCode ),
                        raster.getMwcsGeom(),
                        "L_%s"%(metadata[feature_id]["subDatasetId"])
                    )
                    response[ feature_id ][ "mwcs_path" ] = mwcs_path
                    abs_mwcs_path = os.path.join( settings.WCS_DIR, mwcs_path,"" )
                    try:
                        os.makedirs( os.path.dirname( abs_mwcs_path ) )
                    except OSError as exception:
                        if exception.errno != errno.EEXIST:
                            error_mess = "Unable to create directory %s" %( os.path.dirname( abs_mwcs_path ) )
                            logger.exception( error_mess )
                            raise OSError( error_mess )
                    os.rename( response[ feature_id ][ "tmp_file" ], abs_mwcs_path+identifier+".vrt" )
                    describeCoverageTmp = _updateDescribeCoverage( describeCoverageTmp, raster, metadata[ feature_id ] )
                    logger.debug( "%s describe coverage update" %( abs_mwcs_path ) )
                    logger.debug( "4 - %s mwcs OK [ %s ]" %( feature_id, abs_mwcs_path ) )
                else:
                    cur.execute( "SELECT identifier FROM records WHERE identifier='%s'" %( identifier ) )
                    row = cur.fetchone()
                    if row is not None:
                        response[ feature_id ][ "status" ] = "UPDATE"
                        if row[0] is not None:
                            backup_identifier = row[0] + "BCK"
                            cur.execute( "UPDATE records SET identifier='%s' WHERE identifier='%s'" %( backup_identifier, identifier ) )
                            conn.commit()
                            logger.debug( "Metadata backup. New_uid='%s' - Old_uid='%s'" %( backup_identifier, identifier ) )
                            original_identifier = identifier
                    else:
                        response[ feature_id ][ "status" ] = "INSERT"
                    logger.debug( "3 - %s identifier OK" %( feature_id ) )
                    landsat_row="null"
                    landsat_path="null"
                    wrs="null"
                    sentinel_mgrs="null"
                    mgrs="null"
                    mwcs_grid="null"
                    mwcs_path="null"

                #::::::::::: 5) pycsw catalogue :::::::::::
                typename = "csw:Record"
                schema = "http://www.opengis.net/cat/csw/2.0.2"
                mdsource = "local"
                insert_date = datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
                xml = ""
                _type = "http://purl.org/dc/dcmitype/Dataset"
                title = _getKey( metadata[ feature_id ], "datasetId" )
                title_alternate = _getKey( metadata[ feature_id ], "title_collection" )
                keywords = _getKey( metadata[ feature_id ], "keywords" )
                try:
                    time_begin = _getKey( metadata[ feature_id ], "timeExtent" ).split( "/" )[0]
                    time_end = _getKey( metadata[ feature_id ], "timeExtent" ).split( "/" )[1]
                except:
                    time_begin = ""
                    time_end = ""
                date = _getKey( metadata[ feature_id ], "date" )
                date_modified = _getKey( metadata[ feature_id ], "creationDate" )
                _format = "image/tiff"
                abstract_dict = _getGrid( metadata[ feature_id ],  of="dict" )
                abstract_dict[ "Identifier" ] = identifier

                abstract_dict[ "Start" ] = time_begin
                abstract_dict[ "End" ] = time_end
                if metadata[feature_id]["geolocated"]:
                    sourcerastergeo = _getSourceRasterGeo( metadata[ feature_id ], identifier ,)
                    source=metadata[feature_id]["source"]
                    wkt_geometry = raster.getPolygon()
                    abstract_dict[ "Data Type" ] = raster.datatype
                    metadata[feature_id]["minValue"]=raster.min
                    metadata[feature_id]["maxValue"]=raster.max
                    metadata[feature_id]["noDataValue"]=raster.nodata
                    metadata[feature_id]["EPSG"]=raster.AuthorityCode
                    metadata[feature_id]["minDate"]=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime(int(describeCoverageTmp["%s_%s"%(metadata[ feature_id ]["datasetId"],raster.coverageId())]["t_min"])))
                    metadata[feature_id]["maxDate"]=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime(int(describeCoverageTmp["%s_%s"%(metadata[ feature_id ]["datasetId"],raster.coverageId())]["t_max"])))
                else:
                    sourcerastergeo='null'
                    source=metadata[feature_id]["source"]
                    wkt_geometry =(ogr.CreateGeometryFromJson(metadata[feature_id]["geometry"])).ExportToWkt()
                    abstract_dict[ "Data Type" ] = None
                    metadata[feature_id]["minValue"]=None
                    metadata[feature_id]["maxValue"]=None
                    metadata[feature_id]["noDataValue"]=None
                    metadata[feature_id]["EPSG"]=None
                    metadata[feature_id]["minDate"]=metadata[feature_id]["date"]
                    metadata[feature_id]["maxDate"]=metadata[feature_id]["date"]

                anytext = ' '.join( [ identifier, title, title_alternate, date, date_modified, time_begin, time_end ] + re.split( "[, \-!?:_]+", title_alternate ) + re.split( "[, \-!?:_]+", title ) )

                abstract = _getAbstract( abstract_dict )
                del(metadata[feature_id]["meta"])
                del(metadata[feature_id]["umm"])
                if "wcs_val" in metadata[feature_id]:
                    del(metadata[feature_id]["wcs_val"])
                metadata[feature_id]["identifier"]=identifier

                new_record = """
                INSERT INTO records ( identifier, typename, schema, mdsource, insert_date, xml, anytext, type,
                                      title, title_alternate, abstract, keywords, time_begin, time_end, date,
                                      date_modified, format, sourcerastergeo, source, wkt_geometry,
                                      landsat_row, landsat_path, wrs, sentinel_mgrs, mgrs,
                                      mwcs_path, json_metadata )
                VALUES ( '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', %s, %s, %s, %s, %s, '%s', '%s' )
                """%(
                    identifier, typename, schema, mdsource, insert_date, xml, anytext, _type, title, title_alternate, abstract, keywords, time_begin, time_end, date, date_modified, _format,sourcerastergeo, source, wkt_geometry,
                    landsat_row, landsat_path, wrs, sentinel_mgrs, mgrs,
                    mwcs_path, json.dumps( metadata[ feature_id ] )
                )
                cur.execute( new_record )
                conn.commit()
                logger.debug( '5 - Image registered successfully on catalogue with id "%s"' %( identifier ) )
                response[ feature_id ][ "wcs_url" ] = sourcerastergeo
                response[ feature_id ][ "identifier" ] = identifier
                response[ feature_id ][ "wcs_server" ] = settings.WCS_BASE_PATH

                #::::::::::::::::: 6) FINISH ::::::::::::
                #remove backups
                if response[ feature_id ][ "status" ] == "UPDATE":
                    try:
                        os.remove( raster_backup )
                        logger.debug( "6 - remove old raster backup %s" %( raster_backup  ) )
                    except:
                        None
                    try:
                        cur.execute( "DELETE FROM records WHERE identifier='%s'" %( backup_identifier ) )
                        conn.commit()
                        logger.debug( "6 - remove old metadata from DB. old identifier=%s" %( backup_identifier ) )
                    except:
                        None
                logger.info( 'Registration of %s completed in %f s'%( feature_id, time.time() - start ) )

        except Exception as ex:
            logger.exception(ex)
            response[ feature_id ][ "status" ] = "FAILED"
            try:
                os.remove( response[ feature_id ][ "tmp_file" ] )
                logger.error( "REVERT - remove uploaded file %s" %( response[ feature_id ][ "tmp_file" ] ) )
            except:
                None
            try:
                os.rename( raster_backup, raster_backup.strip( ".BCK" ) )
                logger.error( "REVERT - restore backuped raster %s" %( raster_backup.strip( ".BCK" ) ) )
                response[ feature_id ][ "restored_mwcs_path" ] = raster_backup.strip( ".BCK" )
            except:
                None
            try:
                cur.execute( "DELETE FROM records WHERE identifier='%s';" %( original_identifier ) )
                cur.execute( "UPDATE records SET identifier='%s' WHERE identifier='%s';" %( original_identifier, backup_identifier ) )
                conn.commit()
                logger.debug( "Metadata backup restored with identifier '%s'" %( original_identifier ) )
                response[ feature_id ][ "restored_identifier" ] = original_identifier
            except:
                None
            logger.exception( "File upload failed.")
            if 'mwcs_path' in response[ feature_id ]:
                del( response[ feature_id ][ 'mwcs_path' ] )
            if 'identifier' in response[ feature_id ]:
                del( response[ feature_id ][ 'identifier' ] )
            if 'wcs_url' in response[ feature_id ]:
                del( response[ feature_id ][ 'wcs_url' ] )
            response[ feature_id ][ "error" ] = str( ex )
        response[ feature_id ][ "time" ] = time.time() - start

    conn.close()

    _commitDescribeCoverage( describeCoverageTmp )

    response[ "status" ] = 200
    return JsonResponse( response )

def _getKey( metadata_json, key, default="" ):
    """
    try to extract key from jspn or return default"""
    try:
        return metadata_json[ key ]
    except:
        return default

def _getGrid( metadata_json, of = "string", cursor=None ):
    """
    extract mgrs or patwor grid for mwcs
    """
    landsat_row = "null"
    landsat_path = "null"
    wrs = "null"
    sentinel_mgrs = "null"
    mgrs = "null"
    mwcs_grid = ""

    if "tile" in metadata_json and metadata_json[ "tile" ].strip():
        if re.match( r'^[A-Z]{1}[0-9]{2}[A-Z]{3}$', metadata_json[ "tile" ] ):
            sentinel_mgrs = metadata_json[ "tile" ]
            zone = sentinel_mgrs[ 0:3 ]
            q    = sentinel_mgrs[ 3 ]
            x    = sentinel_mgrs[ 4 ]
            y    = sentinel_mgrs[ 5 ]
            if of == "dict":
                return { 'mgrs_tile': sentinel_mgrs }
            mwcs_grid = ( "%s/%s/%s/%s" %( zone, q, x, y ) ).lower()
            sentinel_mgrs="'%s'" %( sentinel_mgrs )
            cursor.execute( "SELECT id FROM sentinel_mgrs WHERE mgrs=%s" %( sentinel_mgrs ) )
            row = cursor.fetchone()
            if row is None:
                raise ValidationError( "tile - %s not found in DB. Is it a valid Sentinel tile?" %( metadata_json[ "tile" ] ) )
            else:
                mgrs = row[0]

        elif re.match( r'^[0-9]{6}$', metadata_json[ "tile" ] ):
            pathrow = metadata_json[ "tile" ]
            landsat_path = int( pathrow[ 0:3 ] )
            landsat_row = int( pathrow[ 3: ] )
            if of == "dict":
                return{ 'Path': landsat_path, 'Row': landsat_row }
            mwcs_grid = "%d/%d" %( landsat_path, landsat_row )
            cursor.execute( "SELECT id FROM landsat_wrs WHERE path=%d and row=%d" %( landsat_row, landsat_path ) )
            row = cursor.fetchone()
            if row is None:
                raise ValidationError( "tile - %s not found in DB. Is it a valid Landsat tile?" %( metadata_json[ "tile" ] ) )
            else:
                wrs = row[0]
        else:
            raise ValidationError( "tile - %s is not a supported tiling schema" %( metadata_json[ "tile" ] ) )
    else:
        if metadata_json["grid"]:
            if "plan_value" in metadata_json and "image_value" in metadata_json:
                landsat_row = metadata_json["plan_value"]
                landsat_path = metadata_json["image_value"]
                mwcs_grid = "%d/%d" %( int(landsat_path), int(landsat_row ))
                if of == "dict":
                    return {"subset":"gfix(%s,%s)"%(landsat_row,landsat_path)}
            if "wcs_val" in metadata_json and "SubRegion_value" in metadata_json and "Product_value" in metadata_json:
                mwcs_grid = "%s/%s/%s" %( metadata_json["wcs_val"],metadata_json["SubRegion_value"],metadata_json["Product_value"])
                if of == "dict":
                    return {"subset":"gfix(%s,%s,%s)"%(metadata_json["wcs_val"],metadata_json["SubRegion_value"],metadata_json["Product_value"])}
    if of=="dict":
        return {}
    return landsat_row, landsat_path, wrs, sentinel_mgrs, mgrs, mwcs_grid

def _getSourceRasterGeo( metadata_json, identifier ):
    """
    return the wcs url without the host and the base path
    """
    src = 'service=WCS&Request=GetCoverage&version=2.0.0&subset=t(%d)&CoverageId=%s&format=image/tiff&filename=%s&subdataset=%s' %(
        _date_to_unix( metadata_json[ "date" ] ),
        metadata_json[ "datasetId" ],
        identifier,
        metadata_json["subDatasetId"]
    )
    grid_parameters = _getGrid( metadata_json,  of="dict" )
    if grid_parameters:
        for key in grid_parameters:
            new_query_attr = "&%s=%s" %( key.lower(), str( grid_parameters[ key ] ) )
            src += new_query_attr
    return src

def _getAbstract( abstract_dict ):
    table = '<table><tbody>'
    for key in abstract_dict:
        table += '<tr><td><strong>%s</strong></td><td>%s</td></tr>' %( key, str( abstract_dict[ key ] ) )
    table += '</tbody></table>'
    return table

def _date_to_unix( raster_date_string ):
    """
    convert string date to unix time
    """
    raster_date = datetime.datetime.strptime( raster_date_string, '%Y-%m-%dT%H:%M:%SZ')
    origin      = datetime.datetime( 1970, 1, 1, 0, 0, 0, 0 )
    unix_date   = raster_date - origin
    return unix_date.days * 24 * 60 * 60 + unix_date.seconds

def _updateDescribeCoverage( describeCoverageTmp, image_utils, metadata_json ):
    coverageName = "%s_%s" %( metadata_json[ 'datasetId' ], image_utils.coverageId() )
    if not coverageName in describeCoverageTmp:
        describeCoverageTmp[ coverageName ] = {
            "GeoX_lr": image_utils.lower_right_x,
            "GeoY_lr": image_utils.lower_right_y,
            "GeoX_ul": image_utils.up_left_x,
            "GeoY_ul": image_utils.up_left_y,
            "pxAOISizeX": image_utils.size_x,
            "pxAOISizeY": image_utils.size_y,
            "t_min": _date_to_unix( metadata_json[ 'date' ] ),
            "t_max": _date_to_unix( metadata_json[ 'date' ] ),
            "hit_number": 1,
            "coverageName": coverageName,
            "collectionName": metadata_json[ 'datasetId' ],
            "coverageId": image_utils.coverageId(),
            "EPSG": image_utils.AuthorityCode,
            "size_z": image_utils.size_z,
            "resolution": round( image_utils.resolution, 12 ),
            "resolution2": round( image_utils.resolution2, 12 ),
            "datatype_number": image_utils.datatype_number,
            "size": image_utils.getSize()
    }
    else:
        describeCoverageTmp[ coverageName ][ "GeoX_lr" ] = max( describeCoverageTmp[ coverageName ][ "GeoX_lr" ], image_utils.lower_right_x )
        describeCoverageTmp[ coverageName ][ "GeoY_lr" ] = min( describeCoverageTmp[ coverageName ][ "GeoY_lr" ], image_utils.lower_right_y )
        describeCoverageTmp[ coverageName ][ "GeoX_ul" ] = min( describeCoverageTmp[ coverageName ][ "GeoX_ul" ], image_utils.up_left_x )
        describeCoverageTmp[ coverageName ][ "GeoY_ul" ] = max( describeCoverageTmp[ coverageName ][ "GeoY_ul" ], image_utils.up_left_y )
        describeCoverageTmp[ coverageName ][ "pxAOISizeX" ] = int( ( describeCoverageTmp[ coverageName ][ "GeoX_lr" ] - describeCoverageTmp[ coverageName ][ "GeoX_ul" ] ) / image_utils.resolution )
        describeCoverageTmp[ coverageName ][ "pxAOISizeY" ] = int( ( describeCoverageTmp[ coverageName ][ "GeoY_ul" ] - describeCoverageTmp[ coverageName ][ "GeoY_lr" ] ) / image_utils.resolution )
        describeCoverageTmp[ coverageName ][ "t_min" ] = min( describeCoverageTmp[ coverageName ][ "t_min" ], _date_to_unix( metadata_json[ 'date' ] ) )
        describeCoverageTmp[ coverageName ][ "t_max" ] = max( describeCoverageTmp[ coverageName ][ "t_max" ], _date_to_unix( metadata_json[ 'date' ] ) )
        describeCoverageTmp[ coverageName ][ "hit_number" ] += 1
        try:
            describeCoverageTmp[ coverageName ][ "size" ] += image_utils.getSize()
        except Exception as ex:
            describeCoverageTmp[ coverageName ][ "size" ] = image_utils.getSize()
    return describeCoverageTmp

def _commitDescribeCoverage( describeCoverageTmp ):
    """
    update DescribeCoverage.json
    """
    for key in describeCoverageTmp:
        commit_start = time.time()
        dc = describeCoverageTmp[ key ]

        json_describe_coverage_file = os.path.join( settings.WCS_DIR, dc[ "collectionName" ], "DescribeCoverage.json" )

        lock_fd = _acquireLock( json_describe_coverage_file )
        try:
            infomess = "Try to update existing DC file %s" %( json_describe_coverage_file )
            logger.debug( infomess )
            dc_json = json.load( lock_fd )
        except:
            #create new dc
            infomess = "Valid DC file %s not found, create new dc file" %( json_describe_coverage_file )
            logger.debug( infomess )
            dc_json = { "Collections": [] }

        updated = False
        for coverage in dc_json[ "Collections" ]:
            if coverage[ "name" ] == dc[ "coverageName" ]:
                infomess = "Coverage %s found in DC file %s" %( dc[ "coverageName" ], json_describe_coverage_file )
                logger.debug( infomess )
                coverage[ "GeoX_lr" ] = max( coverage[ "GeoX_lr" ], dc[ "GeoX_lr" ]  )
                coverage[ "GeoY_lr" ] = min( coverage[ "GeoY_lr" ], dc[ "GeoY_lr" ]  )
                coverage[ "GeoX_ul" ] = min( coverage[ "GeoX_ul" ], dc[ "GeoX_ul" ] )
                coverage[ "GeoY_ul" ] = max( coverage[ "GeoY_ul" ], dc[ "GeoY_ul" ] )
                coverage[ "pxAOISizeX" ] = max( coverage[ "pxAOISizeX" ], dc[ "pxAOISizeX" ] )
                coverage[ "pxAOISizeY" ] = max( coverage[ "pxAOISizeY" ], dc[ "pxAOISizeY" ] )
                coverage[ "t_min" ] = min( coverage[ "t_min" ], dc[ "t_min" ] )
                coverage[ "t_max" ] = max( coverage[ "t_max" ], dc[ "t_max" ] )
                coverage[ "hit_number" ] += dc[ "hit_number" ]
                updated = True
                break
        if not updated:
            coverage = {}
            infomess = "Coverage %s not found in DC file %s" %( dc[ "coverageName" ], json_describe_coverage_file )
            coverage[ "name" ] = dc[ "coverageName" ]
            coverage[ "id" ]   = dc[ "coverageId" ]
            coverage[ "GeoX_lr" ] = dc[ "GeoX_lr" ]
            coverage[ "GeoY_lr" ] = dc[ "GeoY_lr" ]
            coverage[ "GeoX_ul" ] = dc[ "GeoX_ul" ]
            coverage[ "GeoY_ul" ] = dc[ "GeoY_ul" ]
            coverage[ "pxAOISizeX" ] = dc[ "pxAOISizeX" ]
            coverage[ "pxAOISizeY" ] = dc[ "pxAOISizeY" ]
            coverage[ "t_min" ] = dc[ "t_min" ]
            coverage[ "t_max" ] = dc[ "t_max" ]
            coverage[ "hit_number" ] = dc[ "hit_number" ]
            coverage[ "nband" ] = dc[ "size_z" ]
            coverage[ "EPSG" ] = dc[ "EPSG" ]
            coverage[ "x_res" ] = "%.12f".rstrip("0") %( round( dc[ "resolution" ], 12 ) )
            coverage[ "y_res" ] = "%.12f".rstrip("0") %( round( dc[ "resolution2" ], 12 ) )
            coverage[ "type" ] = dc[ "datatype_number" ]
            coverage[ "bands" ] = []
            for i in range( 0, dc[ "size_z" ] ):
                new_band = {}
                new_band[ "color_interpretation" ] = "Undefined"
                new_band[ "data_type" ] = dc[ "datatype_number" ]
                new_band[ "definition" ] = ""
                new_band[ "identifier" ] = ""
                new_band[ "name" ] = ""
                new_band[ "nil_values" ] = { "reason": "http:\/\/www.opengis.net\/def\/nil\/OGC\/0\/unknown", "value": "0" }
                new_band[ "uom" ] = ""
                new_band[ "value_max" ] = ""
                new_band[ "value_min" ] = ""
                coverage[ "bands" ].append( new_band )
            dc_json[ "Collections" ].append( coverage )

        lock_fd.seek(0)
        json.dump( dc_json, lock_fd )
        lock_fd.truncate()
        _releaseLock( lock_fd )

        _updateVirtualDc( dc[ "collectionName" ] )
        commit_stop = time.time()
        logger.info( 'DescribeCoverage "%s" updated in %f seconds' %( json_describe_coverage_file, commit_stop - commit_start))

def _updateVirtualDc( collectionName ):
    """
    remove describe coverage file for virtual collection
    """
    for coll in os.listdir( settings.WCS_DIR ):
        abs_coll = os.path.join( settings.WCS_DIR, coll )
        if os.path.isdir( abs_coll ):
            for el in os.listdir( abs_coll ):
                abs_el = os.path.join( abs_coll, el )
                if os.path.islink( abs_el ) and el == collectionName:
                    try:
                        virtualDescribe = os.path.join( abs_coll, "DescribeCoverage.json" )
                        os.remove( virtualDescribe )
                        logger.info( 'Remove DescribeCoverage for virtual collection "%s" after uptate of "%s"' %( coll, collectionName ) )
                    except:
                        None

def _acquireLock( filename ):
    ''' acquire exclusive lock file access '''
    try:
        locked_file_descriptor = open( filename, 'r+')
    except:
        locked_file_descriptor = open( filename, 'w+')
    fcntl.lockf(locked_file_descriptor, fcntl.LOCK_EX)
    infomess = "Acquire lock for file %s" %( filename )
    logger.debug( infomess )
    return locked_file_descriptor

def _releaseLock( locked_file_descriptor ):
    ''' release exclusive lock file access '''
    locked_file_descriptor.close()

class ImageUtils():
    def __init__( self, filename ):
        """
        utility class used to extract data from raster
        """
        self.filename = filename
        self.polygon = None
        self.open()

    def open( self ):

        dataset = gdal.Open( self.filename )
        try:
            if dataset.GetSubDatasets():
                raise Exception( "Multidataset rasters are not supported. Consider to use GDAL VRT for this kind of data." )
        except:
            None

        if dataset != None:
            try:
                band         = dataset.GetRasterBand(1)
                geotransform = dataset.GetGeoTransform()

                self.size_x = dataset.RasterXSize
                self.size_y = dataset.RasterYSize
                self.size_z = dataset.RasterCount
                self.AuthorityCode = self.getAuthorityCode( dataset )
                if self.AuthorityCode == None:
                    raise Exception( "georef is not correct so the dataset can not be considered valid" )
                self.resolution    = geotransform[1]
                self.resolution2   = geotransform[5]
                self.up_left_x     = geotransform[0]
                self.up_left_y     = geotransform[3]
                self.datatype      = gdal.GetDataTypeName( band.DataType )
                self.datatype_number = band.DataType
                self.width  = self.size_x * self.resolution
                self.height = self.size_y * self.resolution2
                self.lower_right_x = self.up_left_x + self.width
                self.lower_right_y = self.up_left_y + self.height
                stats=band.GetStatistics(True,True)
                self.min=stats[0]
                self.max=stats[1]
                self.nodata=str(band.GetNoDataValue())

                self.isValid = True
            except Exception as ex:
                dataset = None

        if dataset is None:
            raise Exception( "GDAL error, unable to read raster %s" %( self.filename ) )
        dataset = None

    def getAuthorityCode( self, dataset ):
        wgssr = osr.SpatialReference()
        wgssr.ImportFromWkt( dataset.GetProjectionRef() )
        wgssr.AutoIdentifyEPSG()
        authority_code = wgssr.GetAuthorityCode(None)
        return authority_code

    def getMwcsGeom( self , of= None):
        if of is None:
            resolution_str  = "%.12f".rstrip("0") %( round( self.resolution, 12 ) )
            resolution2_str = "%.12f".rstrip("0") %( round( self.resolution2, 12 ) )
            return "G%fx%f_%sx%s_%dx%dx%d_%d" %( self.up_left_x, self.up_left_y, resolution_str, resolution2_str, self.size_x, self.size_y, self.size_z, self.datatype_number )
        else:
            resolution_str  = "%.12f".rstrip("0") %( round( self.resolution, 12 ) )
            resolution2_str = "%.12f".rstrip("0") %( round( self.resolution2, 12 ) )
            return "G%fx%f_%sx%s_%dx%dx%d_%d" %( self.up_left_x, self.up_left_y, resolution_str, resolution2_str, self.size_x, self.size_y, self.size_z, self.datatype_number )

    def getPolygon( self ):
        """
        check if polygon is defined
        """
        if self.polygon is not None:
            return self.polygon
        polygon = "POLYGON(( %f %f, %f %f, %f %f, %f %f, %f %f ))" %(
            self.convert( self.up_left_x, self.up_left_y ) +
            self.convert( self.lower_right_x,self.up_left_y ) +
            self.convert( self.lower_right_x,self.lower_right_y ) +
            self.convert( self.up_left_x, self.lower_right_y ) +
            self.convert( self.up_left_x, self.up_left_y )
        )
        return polygon

    def setPolygon( self, geojson_geom=None ):
        if geojson_geom is not None:
            try:
                geom = ogr.CreateGeometryFromJson( str( geojson_geom ) )
                self.polygon = geom.ExportToWkt()
            except Exception as e:
                print(e)

    def convert( self, x1, y1 ):
        inProj = 'EPSG:%s' %( self.AuthorityCode )
        transformer = Transformer.from_crs( inProj, 'EPSG:4326', always_xy=True )
        for pt in transformer.itransform( [ ( x1, y1 ) ] ):
             x2, y2 = pt
        return x2, y2

    def coverageId( self ):
        resolution_str = format( round( self.resolution, 6 ), 'f' ).split(".")[0] + format( round( self.resolution, 6 ), 'f' ).split(".")[1].rstrip("0")
        return "%s_%s" %( self.AuthorityCode, resolution_str )

    def getSize( self ):
        size_byte = self.size_x * self.size_y * 4
        return size_byte


import pandas as pd

# Creating DF from CSV files
# df = pd.read_csv('C:/Users/jeyar/OneDrive/Documents/Arulwork/TestFiles/gapminder.tsv', sep='\t')
# print(df)

# print(df.size)
# print(df.shape)
# print(df.columns)
# print(df.dtypes)
# print(df.info())
# subset = df[['country', 'continent', 'year']]
# print(subset)
# print(subset.to_string(columns=['country', 'year'], header=['country1', 'year1'], index=False, justify='center'))
# print(pd.__version__)
# print(subset.loc[[1, 2, 4]])
# print(pd.options.display.max_rows)

# Creating DF from JSON
# df_from_json = pd.read_json('C:/Users/jeyar/OneDrive/Documents/Arulwork/TestFiles/data.json')
# print(df_from_json)

# Creating DF from 2 Series
# data = {
#  "calories": [420, 380, 390],
#  "duration": [50, 40, 45]
# }
# myvar = pd.DataFrame(data)
# print(myvar)

# Creating DF from Dictionary of Dictonaries
data = {
    "child1":{"name":"Arul", "Age":51, "Sex":"M"},
    "child2":{"name":"Santa", "Age1":53, "Sex":"F"}
       }
df = pd.DataFrame(data)
print(df)
print(df.columns)
df1 = df.transpose()
print(df1)
print(df1.columns)


import numpy as np

def estimate_lens_distortion(intrinsics, extrinsics, model, sensor):

    uc = intrinsics[0, 2]
    vc = intrinsics[1, 2]

    D = []
    d = []

    l = 0

    for i in range(0, len(extrinsics)):
        for j in range(0, model.size/2):

            homog_model_coords = np.array([model[j][0], model[j][1], 0, 1])
            homog_coords = np.dot(extrinsics[i], homog_model_coords)

            coords = homog_coords / homog_coords[-1]
            [x, y, hom] = coords

            r = np.sqrt(x*x + y*y)

            P = np.dot(intrinsics, homog_coords)
            P = P / P[2]

            [u, v, trash] = P

            du = u - uc
            dv = v - vc

            D.append(
                np.array([
                    du * r**2, du * r**4
                ])
            )

            D.append(
                np.array([
                    dv * r**2, dv * r**4
                ])
            )

            up = sensor[i][j][0]
            vp = sensor[i][j][1]

            d.append(up - u)
            d.append(vp - v)

    k = np.linalg.lstsq(
        np.array(D),
        np.array(d)
    )

    return k


"""
The sum of the primes below 10 is 2 + 3 + 5 + 7 = 17.

Find the sum of all the primes below two million.
"""

import math
from math import sqrt

def je_prastevilo(num):
    if num < 2:
        return False # poglej funkcijo throw
    if num == 2:
        return True
    for n in range(2 , int(sqrt(num)) + 1):
        if num % n == 0:
            return False # stevilo ni prastevilo če je ta pogoj vsaj 1x res, return vedno zaključi funkcijo, takoj

    return True

a = 2
b = 2000000
vsota = 0
for a in range (2, b + 1):
    resitev = je_prastevilo(a)
    if resitev == True:
        vsota = vsota + a

print(vsota)


from django.shortcuts import render
from rest_framework import viewsets
from rest_framework.response import Response
from rest_framework.generics import get_object_or_404, ListAPIView
from rest_framework.permissions import IsAuthenticated
from rest_framework_simplejwt.authentication import JWTAuthentication
from .serializers import *

# Create your views here.
class CompanyViewSet(viewsets.ViewSet):
    authentication_classes = [JWTAuthentication]
    permission_classes = [IsAuthenticated]

    def list(self, request):
        company = Company.objects.all()
        serializer = CompanySerializer(company, many=True, context={"request":request})
        response_dict = {"error": False, "message": "All companies data", "data": serializer.data}
        return Response(response_dict)

    def create(self, request):
        try:
            serializer = CompanySerializer(data=request.data, context={"request": request})
            serializer.is_valid(raise_exception=True)
            serializer.save()
            dict_response={
                "error":False,
                "message":"Company data added successfully!"
                }
        except:
            dict_response = {
                "error": True,
                "message": "Couldnt add the data",
            }
        return Response(dict_response)

    def update(self, request, pk=None):
        try:
            queryset = Company.objects.all()
            company = get_object_or_404(queryset, pk=pk)
            serializer = CompanySerializer(company, data=request.data, context={"request":request})
            serializer.is_valid(raise_exception=True)
            serializer.save()
            dict_response = {
                "error": False,
                "message": "Company data updated successfully!",
            }
        except:
            dict_response = {
                "error": True,
                "message": "Couldnt add the data",
            }
        return Response(dict_response)

class CompanyBankViewSet(viewsets.ViewSet):
    authentication_classes = [JWTAuthentication]
    permission_classes = [IsAuthenticated]

    def create(self, request):
        try:
            serializer = CompanyBankSerializer(
                data=request.data,
                context={"request":request}
                )
            serializer.is_valid(raise_exception=True)
            serializer.save()
            dict_response = {
                "error": False,
                "message": "Bank Details created!"
            }
        except:
            dict_response = {
                "error": True,
                "message": "Unable to create Bank details!"
            }
        return Response(dict_response)

    def list(self, request):
        company_bank = CompanyBank.objects.all()
        serializer = CompanyBankSerializer(company_bank, many=True, context={"request":request})
        dict_response = {
            "error": False,
            "message": "All company bank details",
            "data": serializer.data
        }

        return Response(dict_response)

    def retrieve(self, request, pk=None):
        queryset = CompanyBank.objects.all()
        company_bank = get_object_or_404(queryset, pk=pk)
        serializer = CompanyBankSerializer(company_bank, context={"request": request})
        dict_response = {
            "error": False,
            "message": "Single company Bank details!",
            "data": serializer.data
        }

        return Response(dict_response)

    def update(self, request, pk=None):
        try:
            queryset = CompanyBank.objects.all()
            company_bank = get_object_or_404(queryset, pk=pk)
            serializer = CompanyBankSerializer(
                company_bank,
                data=request.data,
                context={"request": request}
                )
            serializer.is_valid(raise_exception=True)
            serializer.save()
            dict_response = {
                "error": False,
                "message": "Details updated successfully!"
            }
        except:
            dict_response = {
                "error": True,
                "message": "Unable to update!"
            }

        return Response(dict_response)

class CompanyNameViewSet(ListAPIView):
    authentication_classes = [JWTAuthentication]
    permission_classes = [IsAuthenticated]
    serializer_class = CompanySerializer

    def get_queryset(self):
        name=self.kwargs['name']
        return Company.objects.filter(name=name)

company_list = CompanyViewSet.as_view({"get":"list"})
company_create = CompanyViewSet.as_view({"post": "create"})
company_update = CompanyViewSet.as_view({"put": "update"})


#!/usr/bin/env python
# coding: utf-8

# In[20]:

from sklearn.datasets import fetch_openml
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import umap
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns

# In[2]:

import pandas as pd
import numpy as np

# In[3]:

import time

# In[4]:

mnist = fetch_openml('mnist_784')

# In[5]:

X = mnist.data / 255.0
y = mnist.target

# In[6]:

X.shape

# In[7]:

feat_cols = [ 'pixel'+str(i) for i in range(X.shape[1]) ]
df = pd.DataFrame(X,columns=feat_cols)
df['y'] = y
df['label'] = df['y'].apply(lambda i: str(i))
X, y = None, None

# In[8]:

pca = PCA(n_components=3)
pca_result = pca.fit_transform(df[feat_cols].values)
df['pca-one'] = pca_result[:,0]
df['pca-two'] = pca_result[:,1]
df['pca-three'] = pca_result[:,2]

# In[9]:

pca.explained_variance_ratio_

# In[10]:

rndperm = np.random.permutation(df.shape[0])

# In[11]:

plt.figure(figsize=(16,10))
sns.scatterplot(
    x="pca-one", y="pca-two",
    hue="y",
    palette=sns.color_palette("hls", 10),
    data=df.loc[rndperm,:],
    legend="full",
    alpha=0.3
).get_figure().savefig("pca_mnist.png")

# In[12]:

pca.noise_variance_

# In[13]:

pca.n_components_

# In[14]:

ax = plt.figure(figsize=(16,10)).gca(projection='3d')
ax.scatter(
    xs=df.loc[rndperm,:]["pca-one"].astype(float),
    ys=df.loc[rndperm,:]["pca-two"].astype(float),
    zs=df.loc[rndperm,:]["pca-three"].astype(float),
    c=df.loc[rndperm,:]["y"].astype(float),
    cmap='tab10'
)
ax.set_xlabel('pca-one')
ax.set_ylabel('pca-two')
ax.set_zlabel('pca-three')
plt.show()

# In[18]:

ax.get_figure().savefig("pca_3d_mnist.png")

# In[26]:

N = 10000
df_subset = df.loc[rndperm[:N],:].copy()
data_subset = df_subset[feat_cols].values

# In[27]:

time_start = time.time()
tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
tsne_results = tsne.fit_transform(data_subset)
print('t-SNE done! Time elapsed: {} seconds'.format(time.time()-time_start))

# In[38]:

df_subset['tsne-2d-one'] = tsne_results[:,0]
df_subset['tsne-2d-two'] = tsne_results[:,1]
plt.figure(figsize=(16,10))
sns_plot = sns.scatterplot(
    x="tsne-2d-one", y="tsne-2d-two",
    hue="y",
    palette=sns.color_palette("hls", 10),
    data=df_subset,
    legend="full",
    alpha=0.3
)

# In[42]:

sns_plot.get_figure().savefig("tsne_mnist.png")

# In[21]:

reducer = umap.UMAP(random_state=42)
embedding = reducer.fit_transform(mnist.data)

fig, ax = plt.subplots(figsize=(12, 10))
color = mnist.target.astype(int)
plt.scatter(
    embedding[:, 0], embedding[:, 1], c=color, cmap="Spectral", s=0.1
)
plt.setp(ax, xticks=[], yticks=[])
plt.title("MNIST data embedded into two dimensions by UMAP", fontsize=18)

plt.show()

# In[23]:

fig.savefig("umap_mnist.png")

# In[ ]:


def calculate_PRS_anc(data, ancestry):
    results_df = pd.DataFrame(columns=['phenotype', 'ancestry', 'PRS'])
    maf_beta = pd.DataFrame()
    bootstrap = data
    dict_of_PRS = dict()
    maf_df_all = pd.DataFrame()
    for anc in anc_list:
        df1 = df.loc[(df[1] == phen) & (df[8] == 0) & (df[2] == anc)]
        maf_list = list()
        for i in range(1, 23):
            try:
                maf_list += df1.loc[(df1[3] == i)][9].item()
            except ValueError:
                continue
        beta_position_map['maf'] = maf_list
        maf_beta = pd.merge(bootstrap, beta_position_map)
        maf_beta['meangen'] = maf_beta['maf'] * maf_beta['beta']
        dict_of_PRS[anc] = maf_beta['meangen'].sum()
        maf_df_all[anc] = maf_beta['maf']
    maf_df_all['beta'] = maf_beta['beta']
    maf_df_all['mean_freq'] = maf_df_all[anc_list].mean(axis=1)
    maf_df_all['score'] = maf_df_all['mean_freq']*(1-maf_df_all['mean_freq']) * (maf_df_all['beta']**2)
    mean_PRS = statistics.mean(list(dict_of_PRS.values()))
    varmean = maf_df_all['score'].sum()
    for anc in anc_list:
        meangen = dict_of_PRS[anc] - mean_PRS
        meangenvec = 2 * meangen / (math.sqrt(4*varmean))
        results_df = results_df.append({'phenotype': phen, 'ancestry': anc, 'PRS': meangenvec}, ignore_index=True)
    return results_df.loc[results_df['ancestry'] == ancestry]['PRS'].item()


"'In this model we have taken two hidden two layers and 1 i/p layer, 1 o/p layer and here we train 28 x 28 pixels images'"
import cv2 as cv           #To get my example loaded in the program
import numpy as np        #This is used for matrix
import tensorflow as tf     #This is used to type the code
import matplotlib.pyplot as plt  #For non linear functions

# mnist = tf.keras.datasets.mnist #This is used to get the training data
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data() #It splits the data into training data & test data
x_train = tf.keras.utils.normalize(x_train, axis=1)     #This is to normalize the o/p i.e to bring total output to 1
x_test = tf.keras.utils.normalize(x_test, axis=1)
model = tf.keras.models.Sequential()                  #Sequential model is loaded
model.add(tf.keras.layers.Flatten(input_shape=(28, 28)))    #i/p layer
model.add(tf.keras.layers.Dense(units = 128, activation = tf.nn.relu))  #hidden layer1
model.add(tf.keras.layers.Dense(units = 128, activation = tf.nn.relu))  #hidden layer2
model.add(tf.keras.layers.Dense(units=10, activation = tf.nn.softmax))  #o/p layer
model.compile(optimizer= 'adam', loss= 'sparse_categorical_crossentropy', metrics = ['accuracy'])   #here my loss and accuracy is calcutlated
model.fit(x_train, y_train, epochs = 10)                                                             #here we tell to train the network 3 times on the same data
loss, accuracy = model.evaluate(x_test, y_test)                                                     #here our model is trainded i.e accuracy and loss is calculated
print(accuracy*100)
print(loss*100)
model.save('digits.model')                  #here we store the data of our neural network

for x in range(1,6):
    img = cv.imread(f'{x}.png')[:,:,0]
    img =np.invert( np.array([img]))
    img = tf.keras.utils.normalize(img, axis=1)
    prediction = model.predict(img)
    print(np.argmax(prediction))
    # img = tf.keras.utils.normalize(img , axis = 1)
    # result = model.predict(img)
    # print(np.argmax(result))
    plt.imshow(img[0], cmap=plt.cm.binary)
    plt.show()


from typing import List

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class LinkList:
    def __init__(self, head: ListNode):
        self.head = head

class Heap:
    def __init__(self, lists: List[LinkList]):
        self.lists = lists
        self.size = len(lists)
        self.makeHeap()

    def makeHeap(self):
        if (self.size > 1):
            for i in range(self.size // 2 - 1, -1, -1):
                self.heapifySubTree(i)

    def heapifySubTree(self, i: int):
        left = self.left(i)
        right = left + 1
        if (left < self.size):
            if (right >= self.size or self.prior(left, right)):
                if (self.prior(left, i)):
                    self.swap(i, left)
                    self.heapifySubTree(left)
            elif (self.prior(right, i)):
                self.swap(i, right)
                self.heapifySubTree(right)

    def pop(self):
        self.size -= 1
        if (self.size > 0):
            self.lists[0] = self.lists[self.size]
            self.heapifySubTree(0)

    def top(self):
        return self.lists[0]

    def count(self):
        return self.size

    def left(self, i: int):
        return i + i + 1

    def right(self, i: int):
        return i + i + 2

    def prior(self, i: int, j: int) -> bool:
        return self.lists[i].head.val < self.lists[j].head.val

    def swap(self, i: int, j: int):
        t = self.lists[i]
        self.lists[i] = self.lists[j]
        self.lists[j] = t

class Solution:
    def mergeKLists(self, lists: List[ListNode]) -> ListNode:
        lists2 = [ LinkList(l) for l in lists if l != None ]

        if (len(lists2) == 0):
            return None

        result = []
        heap = Heap(lists2)
        while (heap.count() > 0):
            topList = heap.top()
            result.append(topList.head.val)

            topList.head = topList.head.next
            if (topList.head == None):
                heap.pop()
            else:
                heap.heapifySubTree(0)

        return result

if __name__ == "__main__":
    s = Solution()

    lists = [
        ListNode(1),
        ListNode(0)
    ]

    result = s.mergeKLists(lists)
    print(result)


import sys, random, pickle, os, json, math, re, time

import numpy as np

import torch
import torch.nn.functional as F

class BaseAgent(torch.nn.Module):

    def __init__(self, tokenizer, model):

        super().__init__()

        self.model = model
        self.tokenizer = tokenizer
        self.optimizer = torch.optim.Adam(
            model.parameters(),
            lr=0.00005,
            betas=(0.9, 0.98),
            eps=1e-9,
        )

        self.num_gpus = torch.cuda.device_count()

        if self.num_gpus > 1:
            self.model.parallelize()
        elif self.num_gpus == 1:
            self.model = self.model.cuda()

    def get_response(self, prompt, max_len = 32):

        prompt_ids = self.tokenizer.encode(
            prompt,
            return_tensors="pt",
        )

        if self.num_gpus > 0:
            prompt_ids = prompt_ids.cuda()

        prompt_len = prompt_ids.shape[1]

        output_ids = self.model.generate(
            prompt_ids,
            max_length=prompt_len+max_len,
        )

        generated_text = self.tokenizer.batch_decode(output_ids)[0]

        return generated_text

    def memorize(self, prompt, num_epochs = 3):

        """ This is a rudimentary training loop
        that will train the agent to learn one
        sequence, the prompt. With enough epochs, this should
        result in memorizing the sequence, which is why this
        class method was names memorize.
        There is nothing returned because the model attribute is modified inplace.

        Args:
            prompt (string): the text to be learned
            num_epochs (int): the number of times we cycle though the training data, only 1 sample in this case

        """

        prompt_dic = self.tokenizer(prompt,return_tensors="pt")
        prompt_ids = prompt_dic.input_ids
        prompt_mask = prompt_dic.attention_mask
        prompt_len = prompt_ids.shape[1]

        if self.num_gpus > 0:
            prompt_ids = prompt_ids.cuda()
            prompt_mask = prompt_mask.cuda()

        source_ids = prompt_ids[:,:-1]
        target_ids = prompt_ids[:,1:]
        source_mask = prompt_mask[:,:-1]
        target_mask = prompt_mask[:,1:]

        # allow params to be updated
        self.model.train()

        for e in range(num_epochs):

            # Forward Pass
            output = self.model(
                input_ids = source_ids,
                attention_mask = source_mask,
            )

            # used logits and target tokens to calculate the loss
            logits = output.logits

            scalar_loss = cross_entropy_loss(
                logits,
                target_ids,
            )

            # backward pass
            self.optimizer.zero_grad()
            scalar_loss.backward()
            torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
            self.optimizer.step()

            print("epoch", e, "loss", scalar_loss.item())

def cross_entropy_loss(logits, target_ids):

    """
    For F.cross_entropy the Input is shape (N, C), where N = batch_size x sequence_length
    and C is the number of classes, in our case C is the number of tokens in the vocabulary
    Target is shape (N).

    https://pytorch.org/docs/stable/generated/torch.nn.functional.cross_entropy.html

    we flatten the batch dimension together with the max_seq length
    so that for the loss funstion, so afterwards, there is no batch dimension,
    just a vector sized C-dimensions for each of the seq_len tokens.
    If there had been 2 sampels with a batch size of 2, with 3 tokens in each sample
    then the predictions.shape would be torch.Size([6, 50257])

    Args:
        logits (torch.tensor, float): shape [batch_size, sequence_length, vocab_size]
        target_ids (torch.tensor, int): shape [batch_size, sequence_length]

    Returns:
        scalar_loss (torch.tensor, scalar float, grad_fn=<NllLossBackward0>)): no shape
            this is a loss you can backpropagate using:

            optimizer.zero_grad()
            scalar_loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
            optimizer.step()

    """

    predictions = logits.view(-1, logits.size(-1))
    target = target_ids.view(-1)

    scalar_loss = F.cross_entropy(
        predictions,
        target,
    )

    return scalar_loss


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import math
import solfege.interval
from .interval import DiatonicInterval, ChromaticInterval,SolfegeInterval, Alteration, TooBigAlteration#, _Interval
from util import *
from .note import DiatonicNote,ChromaticNote,Note,_Note

class IntervalWithNoRole(MyException):
    """Raised when the difference between base note and an interval has no role. I.e. is 1 or 2."""
    pass

class _NoteWithBase(_Note):
    """
    Classes for notes with base. They are a note, with an additional information representing the tonic.
    """
    def __init__(self,toCopy=None,base=None, **kwargs):
        super().__init__(toCopy=toCopy,**kwargs)
        self.base=None
        if isinstance(toCopy,_NoteWithBase):
            base=toCopy.getBase()
        if base is not None:
            self.addBase(base)

    def __add__(self,other):
        sum_=super().__add__(other)
        base=self.getBase()
        if base is not None:
            sum_.addBase(base)
        return  sum_

    def __hash__(self):
        return super().__hash__()

    def __eq__(self,other):
        return super().__eq__(other) and self.getBase()==other.getBase()

    def addBase(self,base):
        assert (base is not None)
        self.base=base
        if self.__class__!=base.__class__:
            raise Exception("Adding a base of a type %s distinct from the class %s"%(base.__class__,self.__class__))

        # if self.hasNumber():
        #     role=self.__class__.role[(self.getNumber()-base.getNumber())%self.__class__.modulo]
        #     if role is None:
        #         raise IntervalWithNoRole#Exception("Adding base %s whose number is %d to note %s, (interval=%s)  the role is None, from %s[%d%%%d=%d]"%(base,base.getNumber(),self,self.interval,self.__class__.role,self.interval.getNumber(),self.__class__.modulo,self.interval.getNumber()%self.__class__.modulo))
        #     self.role= role
        #     #debug("The role of %s is %s"%(self,role))

    def getInterval(self):
        """Interval between the note and its base"""
        if "interval" not in self.dic:
            if self.base is  None or self.value is None:
                self.dic["interval"]=None
            else:
                self.dic["interval"]=self-self.base
        return self.dic["interval"]

    def getRole(self):
        """The role of this note, assuming its in a major scale"""
        if "role" not in self.dic:
            interval=self.getInterval()
            interval=interval.getNumber()%self.modulo
            role= self.role[interval]
            self.dic["role"]=role
        return self.dic["role"]

    def getBase(self):
        return self.base

class DiatonicNoteWithBase(_NoteWithBase,DiatonicNote):
    #Saved as the interval from middle C
    role=["Tonic", "supertonic","mediant", "subdominant","dominant","submediant","leading"]

class ChromaticNoteWithBase(_NoteWithBase,ChromaticNote):
    RelatedDiatonicClass=DiatonicNoteWithBase
    def getColor(self,color=True):
        if color:
            dic={"unison":"black","third": "violet","fifth": "red","interval": "green",None:None}
            return dic[self.getRole()]
        else:
            return "black"

    def getDiatonic(self):
        """Assuming no base is used"""
        if "diatonic" not in self.dic:
            if self.getNumber() is None:
                diatonic=None
            elif self.getBase() is None:
                raise Exception("Diatonic asked when the current note %s has no base"%self)
            elif self == self.getBase():
                #If we can't use the base to determine the diatonic note, we take the more likely one
                diatonic= super().getDiatonic()
                diatonic.addBase(diatonic)
            else:
                #Otherwise, we use the role to figure out which diatonic note to use
                role=self.getRole()
                diatonicNumber={"unison":0,"third": 2,"fifth":4 ,"interval": 6}[role]
                diatonicIntervalBaseOctave=DiatonicInterval(diatonic=diatonicNumber)
                octave=self.getInterval().getOctave()
                diatonicInterval=diatonicIntervalBaseOctave.addOctave(octave)
                diatonic=self.base.getDiatonic()+diatonicInterval
                diatonic.addBase(self.base.getDiatonic())
                debug("Note %s's diatonic is not base. Its interval is %s and its diatonic is %s"%(self,diatonicInterval,diatonic))
            self.dic["diatonic"]=diatonic
        return self.dic["diatonic"]

    def getNote(self):
        note=super().getNote(Class=NoteWithBase)
        base=self.getBase()
        if self is base:
            note.addBase(note)
        elif base is not None:
            note.addBase(base.getNote())
        return note

class NoteWithBase(ChromaticNoteWithBase,Note):
    IntervalClass= SolfegeInterval
    DiatonicClass=DiatonicNote
    ChromaticClass=ChromaticNote
    """A note of the scale, as an interval from middle C."""

    def getName(self,kind=None):
        diatonic = self.getDiatonic()
        try:
            alteration=self.getAlteration()
        except TooBigAlteration as tba:
            tba.addInformation("Note",self)
            raise
        diatonicName=diatonic.getName().upper()
        alterationName=alteration.getName(kind=kind)
        return "%s%s" %(diatonicName,alterationName)

    def correctAlteration(self):
        return  self.getAlteration().printable()

ChromaticNoteWithBase.RelatedSolfegeClass=NoteWithBase


from . import create_sequence

class Executor:
    def __init__(self):
        self.__sequence = []

    def run(self):
        while True:
            res = self.run_once()
            if not res:
                break

    def run_once(self):
        command = input()
        sequence = []
        if command.startswith("Create Sequence "):
            return self.__handle_create(command)

        elif command.startswith("Locate "):
            return self.__handle_locate(command)

        elif command == "Stop":
            return self.__handle_stop(command)
        return True

    def __handle_create(self, command):
        number = int(command.split()[-1])
        self.__sequence = create_sequence(number)
        print(' '.join(str(i) for i in self.__sequence))
        return True

    def __handle_locate(self, command):
        search_number = int(command.split()[-1])
        try:
            pos = self.__sequence.index(search_number)
            print(f"The number - {search_number} is at index {pos}")
        except ValueError:
            print(f"The number {search_number} is not in the sequence")
        return True

    def __handle_stop(self, command):
        return False

September 2020/07-Modules/05/Jorda's solution 2/fibonaciii/executor.py


"""
Produces Fig. 4 of Johnson & Weinberg (2019), a 1-panel plot showing the
IMF-averaged Sr yields from CCSNe as a funtion of total metallicity and
rotational velocity.
"""

import visuals # visuals.py -> matplotlib subroutines in this directory
import matplotlib.pyplot as plt
import math as m
import vice
import sys
import warnings
warnings.filterwarnings("ignore")

# The keywords accepted by VICE denoting the studies to plot yields from
_STUDIES_ = ["LC18", "CL04"]

# The rotational velocities of stars probed by a given study
_VROT_ = {
	"LC18":		[0, 150, 300],
	"CL04":		[0],
}

# The overall metallicities [M/H] probed by a given study
_MH_ = {
	"LC18":		[-3, -2, -1, 0],
	"CL04":		[-4, -2, -1, -0.37, 0.15],
}

# The colors to plot a given study in
_COLORS_ = {
	"LC18":		"blue",
	"CL04":		"crimson"
}

# The markers to use for each rotational velocity
_MARKERS_ = {
	0:		"circle",
	150:	"triangle_up",
	300:	"star"
}

# The full names of each study given the keyword
_NAMES_ = {
	"LC18":		"Limongi \& Chieffi (2018)",
	"CL04": 	"Chieffi \& Limongi (2004)"
}

def plot_all_yields(ax, element):
	"""
	Plots all yields for a given element on the axis given the studies and
	rotational velocities to determine yields from

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to plot on
	element :: str
		The element to plot the yield of
	"""
	for i in _STUDIES_:
		for j in _MH_[i]:
			for k in _VROT_[i]:
				plot_yield(ax, element, i, j, k, _MARKERS_[k], _COLORS_[i])
	plot_legend(ax)

def plot_yield(ax, element, study, MH, vrot, marker, color):
	"""
	Plots a single IMF-averaged CCSN yield of Sr on the axis

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to plot on
	element :: str
		The element to plot the yield of
	study :: str
		The study to adopt the yields from
	MH :: real number
		The value of [M/H] to calculate the yield for
	vrot :: real number
		The rotational velocity of the stars in km/s
	marker :: str
		The name of the marker to plot the point in
	color :: str
		The name of the color to plot the yield in
	"""
	y = vice.yields.ccsne.fractional(element, study = study, MoverH = MH,
		rotation = vrot)[0]
	ax.scatter(MH, y, c = visuals.colors()[color],
		marker = visuals.markers()[marker], s = 100)

def plot_functional(ax, func):
	"""
	Plots the yield as a function of metallicity on the axis

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to plot on
	func :: <function>
		The function to plot on the axis
	"""
	small, big = ax.get_xlim()
	xvals = [small + (big - small) / 1000 * i for i in range(1000)]
	ax.plot(xvals, list(map(func, xvals)), linestyle = ':',
		c = visuals.colors()["black"])

def plot_legend(ax):
	"""
	Draws the legend on the axis

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to put the legend on
	"""

	# First label the studies ...
	lines = len(_STUDIES_) * [None]
	for i in range(len(lines)):
		lines[i] = ax.plot([-1, -2], [1.e-8, 1.e-6],
			c = visuals.colors()["white"], label = _NAMES_[_STUDIES_[i]])[0]
	leg = ax.legend(loc = visuals.mpl_loc()["lower right"], ncol = 1,
		bbox_to_anchor = (0.99, 0.01), frameon = False, handlelength = 0,
		fontsize = 18)
	for i in range(len(lines)):
		lines[i].remove()
		leg.get_texts()[i].set_color(_COLORS_[_STUDIES_[i]])
	ax.add_artist(leg)

	# ... then label the rotational velocities
	points = 3 * [None]
	for i in range(len(points)):
		points[i] = ax.scatter([-1, -2], [1.e-8, 1.e-6],
			c = visuals.colors()["black"], s = 50,
			marker = visuals.markers()[_MARKERS_[[0, 150, 300][i]]],
			label = r"$v_\text{rot}$ = %g km s$^{-1}$" % ([0, 150, 300][i])
		)
	ax.legend(loc = visuals.mpl_loc()["upper left"], ncol = 1,
		bbox_to_anchor = (0.01, 0.99), frameon = False, fontsize = 18,
		handlelength = 1)
	for i in range(len(points)):
		points[i].remove()

def main():
	"""
	Produces the figure and saves it as a PDF.
	"""
	ax = visuals.subplots(1, 1)
	ax.set_xlabel(r"$\log_{10}(Z/Z_\odot)$")
	ax.set_ylabel(r"$y_\text{Sr}^\text{CC}$")
	ax.set_yscale("log")
	ax.set_xlim([-4.4, 0.6])
	ax.set_ylim([3.e-13, 3.e-6])
	plot_all_yields(ax, "sr")
	plot_functional(ax, lambda x: 3.5e-8 * 10**x)
	plot_functional(ax, lambda x: 1.e-7 * (1 - m.exp(-10**(x + 1))))
	plt.tight_layout()
	plt.savefig(sys.argv[1])
	plt.clf()

if __name__ == "__main__":
	main()


from __future__ import annotations

from typing import TYPE_CHECKING

from psycopg2.errors import IntegrityError

from data_access.dao import ContractsDAO, ProjectsDAO
from data_access.dto import ProjectsDTO
from errors import ContractAlreadyExistError, IncorrectIdError, ValidationError
from validators import validate_active_contract_id, validate_entered_id

if TYPE_CHECKING:
    from data_access.interfaces import DBGatewayProtocol

class ProjectLogic:
    """Contains methods for displaying project data."""

    def __init__(self, db_gateway: DBGatewayProtocol) -> None:
        self._db_gateway = db_gateway
        self._dao = ProjectsDAO(db_gateway=self._db_gateway)
        self._contracts_dao = ContractsDAO(db_gateway=self._db_gateway)

    def get_all_data(self) -> list[ProjectsDTO]:
        """Gets all projects information from the database."""

        data: list[ProjectsDTO] = self._dao.get_all_projects_info()
        return data

    def get_active_contracts_ids(self) -> list[tuple[int]]:
        """Gets list ids of active contracts"""
        contract_dao = ContractsDAO(db_gateway=self._db_gateway)
        active_contracts_list = contract_dao.get_active_contracts_ids()
        return active_contracts_list

    def create_record(self, entered_name: str) -> None:
        """Creates new record in database with entered data."""

        project_dto = ProjectsDTO(name=entered_name)
        try:
            self._dao.create_record(data=project_dto)
        except IntegrityError:
            raise

    def get_record_by_id(self, project_id: str) -> ProjectsDTO:
        """Gets project by entered id from database."""

        try:
            validate_entered_id(entered_id=project_id)
        except ValidationError:
            raise
        else:
            project_id_int = int(project_id)
        project_info = self._dao.get_info_by_id(project_id_int)
        if not project_info:
            raise IncorrectIdError("[ERROR]: There isn't a contract with entered ID in the database.")
        else:
            return project_info

    def add_contract_to_project(self, project_id: str, contract_id: str) -> None:
        """Adds contract to project."""

        try:
            validate_active_contract_id(
                entered_id=contract_id,
                contracts_ids_list=self.get_active_contracts_ids(),
            )
            project_info = self.get_record_by_id(project_id=project_id)
        except ValidationError:
            raise
        except IncorrectIdError:
            raise
        else:
            try:
                if project_info.contract_id:
                    raise ContractAlreadyExistError(
                        f"Project {project_id} already has an active contract! "
                        f"Please select another contract to add."
                    )
                else:
                    int_contract_id = int(contract_id)
                    int_project_id = int(project_id)
                    project_info.contract_id = int_contract_id
                    self._dao.update_record(project_info)
                    contract_info = self._contracts_dao.get_contract_info_by_id(contract_id=int_contract_id)
                    if contract_info:
                        contract_info.project_id = int_project_id
                        self._contracts_dao.update_record(data=contract_info)
                    else:
                        raise IncorrectIdError("[ERROR]: There isn't a contract with entered ID in the database.")
            except IntegrityError:
                raise

    def remove_active_contract_from_project(self, contract_id: str) -> None:
        """Removes active contract_id."""

        try:
            validate_entered_id(entered_id=contract_id)
        except ValidationError:
            raise
        int_contract_id = int(contract_id)
        data = self._dao.get_project_info_by_active_contract(contract_id=int_contract_id)
        if not data:
            raise IncorrectIdError(f"Project with entered active_contract_id {contract_id} does not exist.")
        else:
            data.contract_id = None
            self._dao.update_record(data=data)


class OP(object):
    """
    OP (grammar operator) is the root of hierarchy of fundamental grammar operators.
    Graph of instances (including descendants) of grammar ops forms grammatic structure of the language.
    Current grammar ops (see below) are derived from the first parsers implementation, so the are a bit
    different from EBNF and other widely used notations.

    Implementation of grammar ops is ready for multiple inheritance and automatic processing.
    """
    def _use_props(self, prop_names, caller_kwargs, optional=None):
        """
        generic kwargs grabber
        """
        self.prop_names = prop_names
        for prop in prop_names:
            if optional is not None and prop in optional:
                if prop not in caller_kwargs:
                    caller_kwargs[prop] = None
            else:   # required prop
                if caller_kwargs.get(prop, None) is None:
                    raise TypeError("required argument '{}' is not found or is None".format(prop))

            setattr(self, prop, caller_kwargs[prop])
            del caller_kwargs[prop]

        return caller_kwargs

    def _check_operands(self, ops):
        erroneous = filter(lambda op: not isinstance(op, OP), ops)
        if erroneous:
            raise TypeError("got non-OPs in operands: {}".format(", ".join(map(str, erroneous))))

class TerminalOP(OP):
    def __init__(self, **kwargs):
        self._use_props(('tok_isinstance', 'tok_data'), kwargs, ('tok_data', ))
        super(TerminalOP, self).__init__(**kwargs)

class ForwardDeclarationOP(OP):
    def __init__(self, **kwargs):
        self._use_props(('target', ), kwargs, ('target', ))
        super(ForwardDeclarationOP, self).__init__(**kwargs)

class AlternativesOP(OP):
    def __init__(self, **kwargs):
        self._use_props(('alternatives', ), kwargs)
        super(AlternativesOP, self).__init__(**kwargs)
        self._check_operands(self.alternatives)

class SequenceOP(OP):
    def __init__(self, **kwargs):
        self._use_props(('subops', 'optional_right_items', 'optional_left_items'), kwargs,
            ('optional_right_items', 'optional_left_items'))
        super(SequenceOP, self).__init__(**kwargs)
        self._check_operands(self.subops)

class RepetitionOP(OP):
    def __init__(self, **kwargs):
        self._use_props(('subop', 'min_matches', 'separator'), kwargs, ('separator', ))
        super(RepetitionOP, self).__init__(**kwargs)
        self._check_operands([self.subop] + ([self.separator] if self.separator is not None else []))

class EmptyOP(OP):
    def __init__(self, **kwargs):
        self._use_props((), kwargs)
        super(EmptyOP, self).__init__(**kwargs)

class OptionalOP(OP):
    def __init__(self, **kwargs):
        self._use_props(('subop', ), kwargs)
        super(OptionalOP, self).__init__(**kwargs)
        self._check_operands([self.subop])

class LookAheadOP(OP):
    def __init__(self, **kwargs):
        self._use_props(('subop', 'inverse'), kwargs)
        super(LookAheadOP, self).__init__(**kwargs)
        self._check_operands([self.subop])

OP_TYPES = (TerminalOP, ForwardDeclarationOP, AlternativesOP, SequenceOP, RepetitionOP, EmptyOP, OptionalOP,
    LookAheadOP)

def find_op_type(obj):
    if not isinstance(obj, OP):
        raise TypeError("obj is a non-OP: {}".format(obj))
    for op_t in OP_TYPES:   # fixme is there a better way?
        if isinstance(obj, op_t):
            return op_t
    assert False

def convert_grammar(root, type_map, traversed_ids=None):
    """
    Converts grammar graph to a different type system.
    type_map has to hold corresponding type for each type of OPs,
    grammar op object will be passed as first parameter to corresponding type's constructor.
    """
    if traversed_ids is None:   # root of recursion
        traversed_ids = set()
        if not isinstance(type_map, dict):
            # one type for all ops
            type_map = dict((op, type_map) for op in OP_TYPES)

    if id(root) in traversed_ids:
        return

    converted = type_map[find_op_type(root)](root, level=len(traversed_ids))
    converted.prop_names = tuple(root.prop_names)
    traversed_ids = set(traversed_ids)  # copy
    traversed_ids.add(id(root))
    for prop_name in root.prop_names:
        prop = getattr(root, prop_name)
        if isinstance(prop, OP):
            setattr(converted, prop_name, convert_grammar(prop, type_map, traversed_ids))
        elif isinstance(prop, (list, tuple)) and filter(lambda so: isinstance(so, OP), prop):
            converted_subops = map(lambda so: convert_grammar(so, type_map, traversed_ids), prop)
            setattr(converted, prop_name, converted_subops if isinstance(prop, list) else tuple(converted_subops))
        else:
            setattr(converted, prop_name, prop)

    return converted


import ConfigParser
import unittest
import os
import tempfile

from moto import mock_ec2
import boto.ec2

import graphlab
from graphlab.connect.aws import _ec2 as EC2
from graphlab.connect.aws._ec2 import _ec2_factory

class Ec2Test(unittest.TestCase):
    """
    This is the beginning of tests for the EC2 class. VERY basic stuff for now, but we'll use moto and add more tests in
    the near future.
    """

    def setUp(self):
        self.access_key = os.environ.get('AWS_ACCESS_KEY_ID', None)
        self.secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY', None)

        if 'AWS_ACCESS_KEY_ID' in os.environ:
            del os.environ['AWS_ACCESS_KEY_ID']
        if 'AWS_SECRET_ACCESS_KEY' in os.environ:
            del os.environ['AWS_SECRET_ACCESS_KEY']

    def tearDown(self):
        if self.access_key: os.environ['AWS_ACCESS_KEY_ID'] = self.access_key
        if self.secret_key: os.environ['AWS_SECRET_ACCESS_KEY'] = self.secret_key

    def test_no_auth_exception(self):
        """
        Verify that no auth shows correct error message
        """

        # Arrange
        instance_type = 'not-a-real-type'
        region = 'us-west-2'

        # Act & Assert
        try:
            ec2 = EC2._Ec2GraphLabServer(instance_type, region, None, None, None, None, '', '')
        except Exception as e:
            self.assertTrue("AWS configuration not found." in e.message, e.message)
        else:
            self.fail('Expected an exception to be raised')

    def test_bad_auth(self):
        """
        Verify that bad auth shows correct error message
        """
        # Arrange
        graphlab.aws.set_credentials('foo', 'bar')

        # Act & Assert
        try:
            ec2 = EC2._Ec2GraphLabServer('not-real-type', 'us-west-2', None, None, None, None, '', '')
        except Exception as e:
            self.assertTrue('EC2 response error. Please verify your AWS credentials' in e.message, e.message)
        else:
            self.fail('Expected an exception to be raised')

    def test_bad_region(self):
        """
        Verify that bad region shows correct error message.
        """
        # Arrange
        graphlab.aws.set_credentials('foo', 'bar')

        # Act & Assert
        try:
            ec2 = EC2._Ec2GraphLabServer('not-real-type', 'not-real-region', None, None, None, None, '', '')
        except Exception as e:
            self.assertTrue('is not a valid region.' in e.message, e.message)
        else:
            self.fail('Expected an exception to be raised')

    def test_config_reading(self):
        # Test retrieving region
        REGION = 'Never Never Land'
        config = ConfigParser.ConfigParser()
        config.add_section(EC2.CONFIG_SECTION)
        config.set(EC2.CONFIG_SECTION, 'region', REGION)
        temp_file = tempfile.NamedTemporaryFile()
        config.write(temp_file)
        temp_file.flush()
        region = EC2._get_region_from_config(temp_file.name)
        self.assertEqual(region, REGION)

        # Test config with our section but not our option
        config = ConfigParser.ConfigParser()
        config.add_section(EC2.CONFIG_SECTION)
        temp_file = tempfile.NamedTemporaryFile()
        config.write(temp_file)
        temp_file.flush()
        region = EC2._get_region_from_config(temp_file.name)
        self.assertIsNone(region)

        # Test empty file
        temp_file = tempfile.NamedTemporaryFile()
        region = EC2._get_region_from_config(temp_file.name)
        self.assertIsNone(region)

        # Test non-existent file
        region = EC2._get_region_from_config('/foo/bar/noWayThereIsAFileCalledThis')
        self.assertIsNone(region)

    @mock_ec2
    def test_ec2_factory(self):
        os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo'
        os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar'
        os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz'
        os.environ['AWS_ACCESS_KEY_ID'] = 'fake'
        os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake'
        os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake'

        ret = _ec2_factory('m3.xlarge')

        self.assertIsNotNone(ret.instance)
        self.assertIsNotNone(ret.instance.private_dns_name)
        self.assertIsNotNone(ret.instance_id)
        self.assertIsNotNone(ret.public_dns_name)
        self.assertEqual(ret.region, 'us-west-2')

        self.assertEqual(ret.instance.image_id, 'foo')
        self.assertEqual(ret.instance.instance_type, 'm3.xlarge')
        self.assertEqual(ret.instance.key_name, 'biz')

        # Upgrading to moto 0.3.7 broke this test. It worked with moto 0.3.5
        # I've verified this functionality is correct in our 1.0 release.
        #ret.instance.update()   # tags were applied before the last update
        #self.assertEquals(ret.instance.tags, {u'GraphLab': u'None'})

        self.assertEquals(ret.instance.state, u'running')
        ret.stop()
        ret.instance.update()
        self.assertEquals(ret.instance.state, u'terminated')

    @mock_ec2
    def test_launching_multiple_hosts(self):
        os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo'
        os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar'
        os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz'
        os.environ['AWS_ACCESS_KEY_ID'] = 'fake'
        os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake'
        os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake'

        ret = _ec2_factory('m3.xlarge', num_hosts = 2)

        self.assertIsInstance(ret, list)
        self.assertEquals(len(ret), 2)
        self.assertIsInstance(ret[0], EC2._Ec2Instance)
        self.assertIsInstance(ret[1], EC2._Ec2Instance)


import asyncio
from enum import Enum, unique
import numpy as np
import struct
import xml.etree.ElementTree as ET
from message import Message
import random
import string

# Source: https://pynative.com/python-generate-random-string/

def randomString(stringLength=10):
    """Generate a random string of fixed length """

    letters = string.ascii_lowercase + string.ascii_uppercase + " ,;-_"
    return ''.join(random.choice(letters) for i in range(stringLength))

class WRPServer:

    DEFAULT_PORT = 8766
    SERVER_IP_ADDRESS = "127.0.0.1"

    class FakeCamera:
        def __init__(self):
            setattr(self, 'SerialNumber', randomString())
            setattr(self, 'ModelName', randomString(50))
            setattr(self, 'Width', random.randint(1, 1920))
            setattr(self, 'Height', random.randint(1, 1080))
            setattr(self, 'CameraMaxFPS', random.random() * 30)
            setattr(
                self,
                'Version',
                f"{random.randint(0, 10)}.{random.randint(0, 10)}.{random.randint(0, 10)}")
            setattr(self, 'ManufacturerInfo', randomString(100))
            setattr(self, 'VendorName', randomString(100))

    @unique
    class State(Enum):
        IDLE = 1
        CONNECTED = 2
        CAMERA_SELECTED = 3
        CONTINUOUS_GRABBING = 4

    def __init__(self, port=DEFAULT_PORT):
        self.ip_address = WRPServer.SERVER_IP_ADDRESS
        self.port = port
        self.camera_list = WRPServer.generate_random_valid_camera_list(5)
        self.state = WRPServer.State.IDLE

    async def __handle_server_exception__(self):
        async with self.server as s:
            try:
                await s.serve_forever()
            except asyncio.CancelledError:
                pass
            except BaseException as e:
                print("Catched exception from the server:", e)

    async def run(self):
        self.server = await asyncio.start_server(self.routine, self.ip_address, self.port)
        self.server_task = asyncio.create_task(
            self.__handle_server_exception__())

    async def routine(self, reader, writer):
        self.state = WRPServer.State.CONNECTED
        while(True):
            print("Server: Waiting for the message...")
            message = await WRPServer.receive_message(reader)
            message_to_send = self.handle_message(message)
            if(message_to_send is not None):
                await WRPServer.send_message(writer, message_to_send)
            else:
                print("No response to be sent was generated")

    async def stop(self):
        self.server.close()
        await self.server.wait_closed()
        del self.server
        self.server_task.cancel()
        await self.server_task
        del self.server_task

    def handle_message(self, message):
        to_send = None
        print(f"Server: Handling message {message} in state {self.state}")
        if(message.msg_type == Message.Type.GET_CAMERA_LIST):
            if(self.state == WRPServer.State.CONNECTED):
                to_send = Message.create_message(
                    Message.Type.CAMERA_LIST,
                    xml_camera_list=WRPServer.camera_list_to_xml(
                        self.camera_list))
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)

        elif(message.msg_type == Message.Type.OPEN_CAMERA):
            if(self.state == WRPServer.State.CONNECTED):
                serial_number = getattr(
                    message, Message.CAMERA_SERIAL_NUMBER_ATTR_NAME)
                try:
                    camera = [
                        c for c in self.camera_list if c.SerialNumber == serial_number][0]
                    self.state = WRPServer.State.CAMERA_SELECTED
                    self.camera_selected = camera
                    to_send = Message.create_message(Message.Type.OK)
                except IndexError:
                    to_send = Message.create_message(
                        Message.Type.ERROR, error_code=Message.ErrorCode.CAMERA_NOT_FOUND)
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)

        elif(message.msg_type == Message.Type.CLOSE_CAMERA):
            if(self.state == WRPServer.State.CAMERA_SELECTED):
                self.state = WRPServer.State.CONNECTED
                del self.camera_selected
                to_send = Message.create_message(Message.Type.OK)
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)

        elif(message.msg_type == Message.Type.GET_FRAME):
            if(self.state == WRPServer.State.CAMERA_SELECTED):
                print("Server: frame_dict:", self.frame_dict)
                to_send = Message.create_message(
                    Message.Type.FRAME, **self.frame_dict)
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)
        else:
            print(f"Unhandled message type {message.msg_type}")
        return to_send

    @staticmethod
    async def send_message(writer, message):
        print("Server: Message to send:", message)
        writer.write(message.encode())
        await writer.drain()

    @staticmethod
    async def receive_message(reader):
        message_type_value, payload_length = struct.unpack(">BI", await reader.readexactly(Message.MESSAGE_TYPE_LENGTH + Message.PAYLOAD_SIZE_LENGTH))
        print(
            f"Server: Received message type value: {message_type_value}, payload_length: {payload_length}")
        if(payload_length > 0):
            payload = await reader.readexactly(payload_length)
        else:
            payload = bytes()
        response = Message.create_message_from_buffer(
            message_type_value=message_type_value,
            payload=payload,
            payload_length=payload_length)
        print("Server: Received message:", response)
        return response

    @staticmethod
    def generate_random_valid_camera_list(number_of_cameras):
        return [WRPServer.FakeCamera() for i in range(number_of_cameras)]

    @staticmethod
    def camera_list_to_xml(camera_list):
        root = ET.Element("Cameras")
        for cam in camera_list:
            ET.SubElement(root, "Camera", **{key: str(val)
                                             for key, val in vars(cam).items()})
        return ET.tostring(root, encoding="unicode")

    @staticmethod
    def generate_random_frame(camera):
        width = getattr(camera, "width")
        height = getattr(camera, "height")
        frame_np = (np.random.random((100, 100)) * 100).astype(np.float32)
        return {
            Message.FRAME_NUMBER_ATTR_NAME: np.random.randint(0, 10),
            Message.FRAME_TIMESTAMP_ATTR_NAME: np.random.randint(0, 2**63),
            Message.FRAME_ATTR_NAME: frame_np
        }