#!/usr/bin/env python3
# Model postprocessing for python-based HDWX
# Created 9 September 2021 by Sam Gardner <stgardner4@tamu.edu>

import sys
from os import path
from pathlib import Path
import xarray as xr
import metpy
from metpy import calc as mpcalc
from cartopy import crs as ccrs
from cartopy import feature as cfeat
from matplotlib import pyplot as plt
from matplotlib.gridspec import GridSpec
import numpy as np
from datetime import datetime as dt, timedelta
from matplotlib import colors as pltcolors
import warnings

# modelPlot.py <model> <initialization> <fhour> <field to plot>
modelName = sys.argv[1]
initDateTime = dt.strptime(sys.argv[2], "%Y%m%d%H%M")
fhour = int(sys.argv[3])
fieldToPlot = sys.argv[4]
axExtent = [-130, -60, 20, 50]
basePath = path.dirname(path.abspath(__file__))

hasHelpers = False
if path.exists(path.join(basePath, "HDWX_helpers.py")):
    import HDWX_helpers
    hasHelpers = True

if modelName == "gfs":
    productTypeBase = 300
elif modelName == "nam":
    productTypeBase = 500
elif modelName == "namnest":
    productTypeBase = 600
elif modelName == "hrrr":
    productTypeBase = 800
elif modelName == "ecmwf-hres":
    productTypeBase = 1000
else:
    raise Exception("<model> must be 'gfs', 'nam', 'namnest', 'hrrr', or 'ecmwf-hres'")

def set_size(w,h, ax=None):
    if not ax: ax=plt.gca()
    l = ax.figure.subplotpars.left
    r = ax.figure.subplotpars.right
    t = ax.figure.subplotpars.top
    b = ax.figure.subplotpars.bottom
    figw = float(w)/(r-l)
    figh = float(h)/(t-b)
    ax.figure.set_size_inches(figw, figh)

def staticSFCTempWindMSLPPlot():
    fig = plt.figure()
    px = 1/plt.rcParams["figure.dpi"]
    fig.set_size_inches(1920*px, 1080*px)
    ax = plt.axes(projection=ccrs.LambertConformal())
    ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    contourmap = sfcTempPlot(False, ax=ax)
    sfcWindPlot(False, ax=ax)
    mslpPlot(False, ax=ax)
    ax.add_feature(cfeat.STATES.with_scale("50m"), linewidth=0.5)
    ax.add_feature(cfeat.COASTLINE.with_scale("50m"), linewidth=0.5)
    validTime = initDateTime + timedelta(hours=fhour)
    if "ecmwf" in modelName:
        noticeStr = "Copyright © "+initDateTime.strftime("%Y")+" European Centre for Medium-Range Weather Forecasts (ECMWF)\nhttps://www.ecmwf.int/"
    else:
        noticeStr = None
    if hasHelpers:
        HDWX_helpers.dressImage(fig, ax, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\n2m Temp, 10m Winds, MSLP", validTime, fhour=fhour, notice=noticeStr, plotHandle=contourmap, cbticks=np.sort(np.append(np.arange(-40, 120, 10), 32)), tickhighlight=32, cbextend="both", colorbarLabel="Temperature (°F)")
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    staticSavePath = path.join(basePath, "output/products/"+modelName+"/sfcTWndMSLP/"+runPathExt)
    Path(staticSavePath).mkdir(parents=True, exist_ok=True)
    gisInfo = ["0,0", "0,0"]
    productId = productTypeBase + 3
    if hasHelpers:
        HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
    else:
        fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
    plt.close(fig)

def sfcTempPlot(standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "t2m.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output/gisproducts/"+modelName+"/sfcT/", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    tempData = modelDataArray["t2m"]
    tempData = tempData.metpy.quantify()
    tempData = tempData.metpy.convert_units("degF")
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([tempData.longitude.data]), (tempData.data.shape[0], 1))
        latsToPlot = np.tile(tempData.latitude.data, (tempData.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = tempData.longitude
        latsToPlot = tempData.latitude
    frozenColorMap = plt.cm.cool_r(np.linspace(0, 1, 25))
    meltedColorMap = plt.cm.turbo(np.linspace(0.35, 1, 25))
    all_colors = np.vstack((frozenColorMap, meltedColorMap))
    temperatureColorMap = pltcolors.LinearSegmentedColormap.from_list("temperatureColorMap", all_colors)
    temperatureNorm = pltcolors.TwoSlopeNorm(vcenter=32, vmin=-40, vmax=130)
    levelsToContour = np.sort(np.append(np.arange(-40, 120, 10), 32))
    contourmap = ax.contourf(lonsToPlot, latsToPlot, tempData, levels=levelsToContour,  cmap=temperatureColorMap, norm=temperatureNorm, transform=ccrs.PlateCarree(), transform_first=True, extend="both")
    ax.contour(lonsToPlot, latsToPlot, tempData, levels=[32], colors="red", transform=ccrs.PlateCarree(), transform_first=True, linewidths=1)
    contourLabels = ax.clabel(contourmap, levels=np.arange(-40, 120, 20), inline=True, fontsize=10, colors="black")
    [label.set_rotation(0) for label in contourLabels]
    [label.set_text(label.get_text()+"F") for label in contourLabels]
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productTypeBase, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return contourmap

def sfcWindPlot(standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "sfcwind.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output", "gisproducts", modelName, "sfcWnd", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    uwind = modelDataArray["u10"]
    uwind = uwind.metpy.quantify()
    uwind = uwind.metpy.convert_units("kt")
    vwind = modelDataArray["v10"]
    vwind = vwind.metpy.quantify()
    vwind = vwind.metpy.convert_units("kt")
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        spatialLimit = slice(None, None, 5)
        dataLimit = (slice(None, None, 5), slice(None, None, 5))
    elif modelName == "nam":
        spatialLimit = (slice(None, None, 10), slice(None, None, 10))
        dataLimit = (slice(None, None, 10), slice(None, None, 10))
    elif modelName == "namnest" or modelName == "hrrr":
        spatialLimit = (slice(None, None, 40), slice(None, None, 40))
        dataLimit = (slice(None, None, 40), slice(None, None, 40))
    windbarbs = ax.barbs(uwind.longitude.data[spatialLimit], uwind.latitude.data[spatialLimit], uwind.data[dataLimit], vwind.data[dataLimit], pivot='middle', color='black', transform=ccrs.PlateCarree(), length=5, linewidth=0.5, zorder=2)
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        productId = productTypeBase + 1
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return windbarbs

def mslpPlot(standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "sp.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output/gisproducts/"+modelName+"/sfcMSLP/", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    if modelName == "ecmwf-hres":
        mslpData = modelDataArray["msl"]
        mslpData = mslpData.metpy.quantify()
        mslpData = mslpData.metpy.convert_units("hPa")
    else:
        barometricPressData = modelDataArray["sp"]
        barometricPressData = barometricPressData.metpy.quantify()
        orogData = modelDataArray["orog"]
        orogData = orogData.metpy.quantify()
        with warnings.catch_warnings():
            warnings.filterwarnings("ignore")
            tempData = xr.open_dataset(path.join(inputPath, "t2m.grib2"), engine="cfgrib", backend_kwargs={"indexpath" : ""})
        tempData = tempData["t2m"]
        tempData = tempData.metpy.quantify()
        # I tried using mpcalc altimeter->mslp function here, but it ended up doing nothing and I don't feel like figuring out why
        # Therefore I implemented the same equation manually...
        from metpy import constants
        mslpData = barometricPressData * np.exp(orogData*constants.earth_gravity/(constants.dry_air_gas_constant*tempData))
        mslpData = mslpData.metpy.quantify()
        mslpData = mslpData.metpy.convert_units("hPa")
    # Unidata says smoothing MSLP "a little" is... well they didn't comment on why, they just did it, and it makes the rocky mtns less noisy...
    # https://unidata.github.io/python-gallery/examples/MSLP_temp_winds.html
    from scipy import ndimage
    if modelName == "namnest" or modelName == "hrrr":
        mslpData.data = ndimage.gaussian_filter(mslpData.data, 5)
    else:
        mslpData.data = ndimage.gaussian_filter(mslpData.data, 3)
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([mslpData.longitude.data]), (mslpData.data.shape[0], 1))
        latsToPlot = np.tile(mslpData.latitude.data, (mslpData.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = mslpData.longitude
        latsToPlot = mslpData.latitude
    levelsToContour = np.arange((np.nanmin(mslpData.data) // 2) * 2, np.nanmax(mslpData.data)+2, 2)
    contourmap = ax.contour(lonsToPlot, latsToPlot, mslpData, levels=levelsToContour, colors="black", transform=ccrs.PlateCarree(), transform_first=True, linewidths=0.5)
    contourLabels = ax.clabel(contourmap, levels=levelsToContour, inline=True, fontsize=10)
    [label.set_rotation(0) for label in contourLabels]
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        productId = productTypeBase + 2
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return contourmap

def windsAtHeightPlot(pressureLevel, standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "winds.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    modelDataArray = modelDataArray.sel(isobaricInhPa=pressureLevel)
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output", "gisproducts", modelName, str(pressureLevel)+"wind", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    uwind = modelDataArray["u"]
    uwind = uwind.metpy.quantify()
    uwind = uwind.metpy.convert_units("kt")
    vwind = modelDataArray["v"]
    vwind = vwind.metpy.quantify()
    vwind = vwind.metpy.convert_units("kt")
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        spatialLimit = slice(None, None, 10)
        dataLimit = (slice(None, None, 10), slice(None, None, 10))
    elif modelName == "nam":
        spatialLimit = (slice(None, None, 20), slice(None, None, 20))
        dataLimit = (slice(None, None, 20), slice(None, None, 20))
    elif modelName == "namnest" or modelName == "hrrr":
        spatialLimit = (slice(None, None, 80), slice(None, None, 80))
        dataLimit = (slice(None, None, 80), slice(None, None, 80))
    windbarbs = ax.barbs(uwind.longitude.data[spatialLimit], uwind.latitude.data[spatialLimit], uwind.data[dataLimit], vwind.data[dataLimit], pivot='middle', color='black', transform=ccrs.PlateCarree(), length=5, linewidth=0.5, zorder=2)
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        if pressureLevel == 250:
            addon = 21
        elif pressureLevel == 500:
            addon = 16
        elif pressureLevel == 850:
            addon = 25
        productId = productTypeBase + addon
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return windbarbs

def simReflectivityPlot(fileToRead, standaloneFig, ax=None):
    pathToRead = path.join(inputPath, fileToRead)
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    if "refc" in list(modelDataArray.variables):
        prodString = "simrefc"
        simDBZ = modelDataArray.refc
    elif "refd" in list(modelDataArray.variables):
        prodString = "simrefd"
        simDBZ = modelDataArray.refd
    else:
        return
    import pyart
    cmap = "pyart_ChaseSpectral"
    vmin=-10
    vmax=80
    dataMask = np.where(np.logical_and(simDBZ.data>=10, simDBZ.data<=80), 0, 1)
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    rdr = ax.pcolormesh(simDBZ.longitude, simDBZ.latitude, np.ma.masked_array(simDBZ, mask=dataMask), cmap=cmap, vmin=vmin, vmax=vmax, transform=ccrs.PlateCarree(), zorder=1)

    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
        gisSavePath = path.join(basePath, "output", "gisproducts", modelName, prodString, runPathExt)
        Path(gisSavePath).mkdir(parents=True, exist_ok=True)
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        if prodString == "simrefc":
            addon = 8
        elif prodString == "simrefd":
            addon = 80
        productId = productTypeBase + addon
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return rdr

def updraftHelicityPlot(standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "udh.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    udhel = modelDataArray.unknown
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
            lonsToPlot = np.tile(np.array([udhel.longitude.data]), (udhel.data.shape[0], 1))
            latsToPlot = np.tile(udhel.latitude.data, (udhel.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = udhel.longitude
        latsToPlot = udhel.latitude
    if np.nanmax(udhel.data) > 50:
        ax.contourf(lonsToPlot, latsToPlot, udhel, levels=[50, 999999], cmap="Greys", vmin=0, vmax=100, transform=ccrs.PlateCarree(), zorder=2, transform_first=True, alpha=0.5)
        ax.contour(lonsToPlot, latsToPlot, udhel, levels=[50], colors="black", transform=ccrs.PlateCarree(), zorder=2, transform_first=True, linewidths=0.5)
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
        gisSavePath = path.join(basePath, "output", "gisproducts", modelName, "udhelicity", runPathExt)
        Path(gisSavePath).mkdir(parents=True, exist_ok=True)
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        productId = productTypeBase + 9
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)

def staticSimDBZPlot(compOrAGL):
    fig = plt.figure()
    px = 1/plt.rcParams["figure.dpi"]
    fig.set_size_inches(1920*px, 1080*px)
    ax = plt.axes(projection=ccrs.LambertConformal())
    ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    sfcWindPlot(False, ax=ax)
    if modelName in ["namnest", "hrrr"] and fhour != 0:
        updraftHelicityPlot(False, ax=ax)
    if compOrAGL == "refccomposite":
        addon = 10
        if modelName in ["namnest", "hrrr"]:
            titleStr = "Composite Reflectivity, Updraft Helicity > 50 $m^2 s^{-2}$, 10m Winds"
        else:
            titleStr = "Composite Reflectivity, 10m Winds"
        rdr = simReflectivityPlot("refc.grib2", False, ax=ax)
    elif compOrAGL == "refdcomposite":
        addon = 81
        if modelName in ["namnest", "hrrr"]:
            titleStr = "1km AGL Reflectivity, Updraft Helicity > 50 $m^2 s^{-2}$, 10m Winds"
        else:
            titleStr = "1km AGL Reflectivity, 10m Winds"
        rdr = simReflectivityPlot("refd.grib2", False, ax=ax)

    ax.add_feature(cfeat.STATES.with_scale("50m"), linewidth=0.5)
    ax.add_feature(cfeat.COASTLINE.with_scale("50m"), linewidth=0.5)
    validTime = initDateTime + timedelta(hours=fhour)
    if hasHelpers:
        HDWX_helpers.dressImage(fig, ax, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\n"+titleStr, validTime, fhour=fhour, notice=None, plotHandle=rdr, colorbarLabel="Simulated Reflectivity (dBZ)")
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    staticSavePath = path.join(basePath, "output", "products", modelName, compOrAGL, runPathExt)
    Path(staticSavePath).mkdir(parents=True, exist_ok=True)
    gisInfo = ["0,0", "0,0"]
    productId = productTypeBase + addon
    if hasHelpers:
        HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
    else:
        fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
    plt.close(fig)

def heightsPlot(pressureLevel, standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "heights.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    if pressureLevel not in modelDataArray.isobaricInhPa.values:
        return
    modelDataArray = modelDataArray.sel(isobaricInhPa=pressureLevel)
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output", "gisproducts", modelName, str(pressureLevel)+"hgt", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    heightData = modelDataArray.gh
    heightData = heightData.metpy.quantify()
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([heightData.longitude.data]), (heightData.data.shape[0], 1))
        latsToPlot = np.tile(heightData.latitude.data, (heightData.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = heightData.longitude
        latsToPlot = heightData.latitude
    if pressureLevel < 400:
        contourLevels = np.arange((np.nanmin(heightData.data.magnitude // 100)*100 - 100), np.nanmax(heightData.data.magnitude)+121, 120)
    else:
        contourLevels = np.arange((np.nanmin(heightData.data.magnitude // 100)*100 - 100), np.nanmax(heightData.data.magnitude)+61, 60)
    contourmap = ax.contour(lonsToPlot, latsToPlot, heightData.data, levels=contourLevels, colors="black", transform=ccrs.PlateCarree(), transform_first=True, linewidths=0.5, zorder=2)
    contourLabels = ax.clabel(contourmap, levels=contourLevels, inline=True, fontsize=10)
    [label.set_rotation(0) for label in contourLabels]
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        if pressureLevel == 250:
            addon = 22
        elif pressureLevel == 500:
            addon = 18
        elif pressureLevel == 850:
            addon = 26
        productId = productTypeBase + addon
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return contourmap

def tempsPlot(pressureLevel, standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "temps.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    if pressureLevel not in modelDataArray.isobaricInhPa.values:
        return
    if "ecmwf" in modelName:
        modelDataArray = modelDataArray.sel(isobaricInhPa=pressureLevel)
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output", "gisproducts", modelName, str(pressureLevel)+"temps", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    tempsData = modelDataArray.t
    tempsData = tempsData.metpy.quantify()
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([tempsData.longitude.data]), (tempsData.data.shape[0], 1))
        latsToPlot = np.tile(tempsData.latitude.data, (tempsData.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = tempsData.longitude
        latsToPlot = tempsData.latitude
    frozenColorMap = plt.cm.cool_r(np.linspace(0, 1, 25))
    meltedColorMap = plt.cm.turbo(np.linspace(0.35, 1, 25))
    all_colors = np.vstack((frozenColorMap, meltedColorMap))
    temperatureColorMap = pltcolors.LinearSegmentedColormap.from_list("temperatureColorMap", all_colors)
    temperatureNorm = pltcolors.TwoSlopeNorm(vcenter=0, vmin=-40, vmax=40)
    levelsToContour = np.arange(-40, 41, 5)
    tempsData = tempsData.data.to("degC")
    contourmap = ax.contourf(lonsToPlot, latsToPlot, tempsData, levels=levelsToContour,  cmap=temperatureColorMap, norm=temperatureNorm, transform=ccrs.PlateCarree(), transform_first=True, extend="both")
    ax.contour(lonsToPlot, latsToPlot, tempsData, levels=[0], colors="red", transform=ccrs.PlateCarree(), transform_first=True, linewidths=1)
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        if pressureLevel == 850:
            addon = 27
        productId = productTypeBase + addon
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return contourmap

def rhPlot(pressureLevel, standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "rh.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    if pressureLevel not in modelDataArray.isobaricInhPa.values:
        return
    if "ecmwf" in modelName:
        modelDataArray = modelDataArray.sel(isobaricInhPa=pressureLevel)
        modelDataArray = modelDataArray.sel(latitude=slice(54.5, 14.2), longitude=slice(-144.5, -44.5))
    modelDataArray = modelDataArray.metpy.quantify()
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output", "gisproducts", modelName, str(pressureLevel)+"rh", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    if "r" in modelDataArray.variables:
        rhData = modelDataArray.r
    elif "dpt" in modelDataArray.variables and "t" in modelDataArray.variables:
        rhData = metpy.calc.relative_humidity_from_dewpoint(modelDataArray.t, modelDataArray.dpt) * 100
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([rhData.longitude.data]), (rhData.data.shape[0], 1))
        latsToPlot = np.tile(rhData.latitude.data, (rhData.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = rhData.longitude
        latsToPlot = rhData.latitude
    levelsToContour = np.arange(60, 110, 10)
    rhArr = np.array([[1, 185/255, 0, 1], [0, 1, 0, 1], [0, 200/255, 0, 1], [0, 139/255, 0, 1]])
    rhcm = pltcolors.LinearSegmentedColormap.from_list("hdwx-humidity", rhArr)
    contourmap = ax.contourf(lonsToPlot, latsToPlot, rhData.data.clip(0,100), levels=levelsToContour,  cmap=rhcm, vmin=60, vmax=100, transform=ccrs.PlateCarree(), transform_first=True, zorder=1)
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        if pressureLevel == 700:
            addon = 29
        productId = productTypeBase + addon
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return contourmap

def vort500Plot(standaloneFig, ax=None):
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.LambertConformal())
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    windsAtHeightPlot(500, False, ax=ax)
    heightsPlot(500, False, ax=ax)
    pathToRead = path.join(inputPath, "winds.grib2")
    modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""}).sel(isobaricInhPa=500)
    if modelDataArray.u.attrs["GRIB_gridType"] == "regular_ll":
        modelDataArray = modelDataArray.metpy.assign_crs({"grid_mapping_name":"latitude_longitude"})
    elif modelDataArray.u.attrs["GRIB_gridType"] == "lambert":
        modelDataArray = modelDataArray.metpy.assign_crs({
            "semi_major_axis": 6371200.0,
            "semi_minor_axis": 6371200.0,
            "grid_mapping_name": "lambert_conformal_conic",
            "standard_parallel": [
                modelDataArray.u.attrs["GRIB_Latin1InDegrees"],
            modelDataArray.u.attrs["GRIB_Latin2InDegrees"]
            ],
            "latitude_of_projection_origin": modelDataArray.u.attrs["GRIB_LaDInDegrees"],
            "longitude_of_central_meridian": modelDataArray.u.attrs["GRIB_LoVInDegrees"],
    }).metpy.assign_y_x()
    if modelName == "namnest":
        modelDataArray = modelDataArray.coarsen(y=5, x=5, boundary="trim").mean()
    uwind = modelDataArray.u
    uwind = uwind.metpy.quantify()
    uwind = uwind.metpy.convert_units("m/s")
    vwind = modelDataArray.v
    vwind = vwind.metpy.quantify()
    vwind = vwind.metpy.convert_units("m/s")
    vortData = mpcalc.absolute_vorticity(uwind, vwind)
    posVortMap = plt.cm.plasma_r(np.linspace(0, 1, 180))
    neutralVortMap = np.array([[1, 1, 1, 1]]*90)
    negVortMap = plt.cm.cool_r(np.linspace(0.4, 1, 90))
    vortArr = np.vstack((negVortMap, neutralVortMap, posVortMap))
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([vortData.longitude.data]), (vortData.data.shape[0], 1))
        latsToPlot = np.tile(vortData.latitude.data, (vortData.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = vortData.longitude
        latsToPlot = vortData.latitude
    vortcm = pltcolors.LinearSegmentedColormap.from_list("hdwx-vorticity", vortArr)
    vortmap = ax.contourf(lonsToPlot, latsToPlot, vortData.data, levels=np.arange(-.00009, .00028, .00003), cmap=vortcm, transform=ccrs.PlateCarree(), transform_first=True, extend="both", zorder=1)
    ax.add_feature(cfeat.STATES.with_scale("50m"), linewidth=0.5)
    ax.add_feature(cfeat.COASTLINE.with_scale("50m"), linewidth=0.5)
    if standaloneFig:
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.dressImage(fig, ax, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\n 500 hPa Heights, Winds, Absolute Vorticity", validTime, fhour=fhour, notice=None, plotHandle=vortmap, colorbarLabel=r"$\frac{1}{s}$")
        runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
        staticSavePath = path.join(basePath, "output", "products", modelName, "500staticvort", runPathExt)
        Path(staticSavePath).mkdir(parents=True, exist_ok=True)
        gisInfo = ["0,0", "0,0"]
        productId = productTypeBase + 20
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        plt.close(fig)
    return ax, vortmap

def jetIsotachsPlot(standaloneFig, ax=None):
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.LambertConformal())
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    heightsPlot(250, False, ax=ax)
    windsAtHeightPlot(250, False, ax=ax)
    pathToRead = path.join(inputPath, "winds.grib2")
    modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""}).sel(isobaricInhPa=250)
    uwind = modelDataArray.u
    uwind = uwind.metpy.quantify()
    uwind = uwind.metpy.convert_units("kt")
    vwind = modelDataArray.v
    vwind = vwind.metpy.quantify()
    vwind = vwind.metpy.convert_units("kt")
    speed = mpcalc.wind_speed(uwind, vwind)
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([speed.longitude.data]), (speed.shape[0], 1))
        latsToPlot = np.tile(speed.latitude.data, (speed.shape[1], 1)).transpose()
    else:
        lonsToPlot = speed.longitude
        latsToPlot = speed.latitude
    jetmap = ax.contourf(lonsToPlot, latsToPlot, speed, cmap="plasma_r", levels=np.arange(70, 171, 20), transform=ccrs.PlateCarree(), transform_first=True, extend="max", zorder=1)
    ax.add_feature(cfeat.STATES.with_scale("50m"), linewidth=0.5)
    ax.add_feature(cfeat.COASTLINE.with_scale("50m"), linewidth=0.5)
    if standaloneFig:
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.dressImage(fig, ax, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\n 250 hPa Heights, Winds, Isotachs", validTime, fhour=fhour, notice=None, plotHandle=jetmap, colorbarLabel="kt")
        runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
        staticSavePath = path.join(basePath, "output", "products", modelName, "250staticjet", runPathExt)
        Path(staticSavePath).mkdir(parents=True, exist_ok=True)
        gisInfo = ["0,0", "0,0"]
        productId = productTypeBase + 24
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        plt.close(fig)
    return ax, jetmap

def temps850Plot(standaloneFig, ax=None):
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.LambertConformal())
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    heightsPlot(850, False, ax=ax)
    windsAtHeightPlot(850, False, ax=ax)
    tempsHandle = tempsPlot(850, False, ax=ax)
    ax.add_feature(cfeat.STATES.with_scale("50m"), linewidth=0.5)
    ax.add_feature(cfeat.COASTLINE.with_scale("50m"), linewidth=0.5)
    if standaloneFig:
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.dressImage(fig, ax, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\n 850 hPa Heights, Winds, Temperatures", validTime, fhour=fhour, notice=None, plotHandle=tempsHandle, colorbarLabel="°C")
        runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
        staticSavePath = path.join(basePath, "output", "products", modelName, "850statictemps", runPathExt)
        Path(staticSavePath).mkdir(parents=True, exist_ok=True)
        gisInfo = ["0,0", "0,0"]
        productId = productTypeBase + 28
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        plt.close(fig)
    return ax, tempsHandle

def rh700Plot(standaloneFig, ax=None):
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.LambertConformal())
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    mslpPlot(False, ax=ax)
    rhHandle = rhPlot(700, False, ax=ax)
    pathToRead = path.join(inputPath, "heights.grib2")
    heightsData = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    oneThousandHeights = heightsData.sel(isobaricInhPa=1000).metpy.quantify()
    fiveHundredHeights = heightsData.sel(isobaricInhPa=500).metpy.quantify()
    thicknessData = fiveHundredHeights.gh - oneThousandHeights.gh
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([thicknessData.longitude.data]), (thicknessData.shape[0], 1))
        latsToPlot = np.tile(thicknessData.latitude.data, (thicknessData.shape[1], 1)).transpose()
    else:
        lonsToPlot = thicknessData.longitude
        latsToPlot = thicknessData.latitude
    coldLevels = np.arange(5340, np.nanmin(thicknessData.data.magnitude)-.01, -60)[::-1]
    hotLevels = np.arange(5460, np.nanmax(thicknessData.data.magnitude)+.01, 60)
    coldContours = ax.contour(lonsToPlot, latsToPlot, thicknessData, colors="blue", levels=coldLevels, linewidths=0.5, linestyles="dashdot", transform=ccrs.PlateCarree(), transform_first=True, zorder=2)
    criticalContour = ax.contour(lonsToPlot, latsToPlot, thicknessData, colors="red", levels=[5400], linewidths=2, transform=ccrs.PlateCarree(), transform_first=True, zorder=2)
    hotContours = ax.contour(lonsToPlot, latsToPlot, thicknessData, colors="red", levels=hotLevels, linewidths=0.5, linestyles="dashdot", transform=ccrs.PlateCarree(), transform_first=True, zorder=2)
    coldLabels = ax.clabel(coldContours, levels=coldLevels, inline=True, fontsize=10)
    [label.set_rotation(0) for label in coldLabels]
    hotLabels = ax.clabel(hotContours, levels=hotLevels, inline=True, fontsize=10)
    [label.set_rotation(0) for label in hotLabels]
    ax.add_feature(cfeat.STATES.with_scale("50m"), linewidth=0.5)
    ax.add_feature(cfeat.COASTLINE.with_scale("50m"), linewidth=0.5)
    if standaloneFig:
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.dressImage(fig, ax, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\n 700 hPa RH, MSLP, 1000->500 hPa Thicknesses", validTime, fhour=fhour, notice=None, plotHandle=rhHandle, colorbarLabel="%")
        runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
        staticSavePath = path.join(basePath, "output", "products", modelName, "700staticrh", runPathExt)
        Path(staticSavePath).mkdir(parents=True, exist_ok=True)
        gisInfo = ["0,0", "0,0"]
        productId = productTypeBase + 31
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
            HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        plt.close(fig)
    return ax, rhHandle

def fourPanelPlot():
    fig = plt.figure()
    px = 1/plt.rcParams["figure.dpi"]
    fig.set_size_inches(1920*px, 1080*px)
    gs = GridSpec(3, 4, figure=fig, width_ratios=[0.01, 1, 1, 0.01], height_ratios=[1, 1, 0.1])
    gs.update(left=75/1920,right=1-(75/1920), top=.9925, bottom=0.01, wspace=0, hspace=0.02)
    cax1 = fig.add_subplot(gs[0, 0])
    ax1 = fig.add_subplot(gs[0, 1], projection=ccrs.LambertConformal())
    ax2 = fig.add_subplot(gs[0, 2], projection=ccrs.LambertConformal())
    cax2 = fig.add_subplot(gs[0, 3])
    cax3 = fig.add_subplot(gs[1, 0])
    ax3 = fig.add_subplot(gs[1, 1], projection=ccrs.LambertConformal())
    ax4 = fig.add_subplot(gs[1, 2], projection=ccrs.LambertConformal())
    cax4 = fig.add_subplot(gs[1, 3])
    lax = fig.add_axes([0,0,1/5,.06])
    lax.set_aspect(2821/11071, anchor="SE")
    tax = fig.add_axes([1/3,0,1/4,.11])
    [ax.set_extent(axExtent, crs=ccrs.PlateCarree()) for ax in [ax1, ax2, ax3, ax4]]
    ax1, vortHandle = vort500Plot(False, ax=ax1)
    ax1.set_title("500 hPa Heights, Winds, Absolute Vorticity")
    ax2, jetHandle = jetIsotachsPlot(False, ax=ax2)
    ax2.set_title("250 hPa Heights, Winds, Isotachs")
    ax3, tempHandle = temps850Plot(False, ax=ax3)
    ax3.set_title("850 hPa Heights, Winds, Temperatures")
    ax4, rhHandle = rh700Plot(False, ax=ax4)
    ax4.set_title("700 hPa RH, MSLP, 1000->500 hPa Thicknesses")

    cb1 = fig.colorbar(vortHandle, cax=cax1, orientation="vertical", label="1/s", format="%.0e")
    newticks = []
    for tick in cb1.get_ticks():
        if np.abs(tick) > 1e-10:
            newticks.append(tick)
        else:
            newticks.append(0)
    cb1.set_ticks(newticks)
    cb1.ax.yaxis.set_ticks_position("left")
    cb1.ax.yaxis.set_label_position("left")
    fig.colorbar(jetHandle, cax=cax2, orientation="vertical", label="kt")
    cb3 = fig.colorbar(tempHandle, cax=cax3, orientation="vertical", label="°C")
    cb3.ax.yaxis.set_ticks_position("left")
    cb3.ax.yaxis.set_label_position("left")
    fig.colorbar(rhHandle, cax=cax4, orientation="vertical", label="%")

    validTime = initDateTime + timedelta(hours=fhour)
    if hasHelpers:
        HDWX_helpers.dressImage(fig, None, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\nForecast 4-Panel", validTime, fhour=fhour, notice=None, plotHandle=None, tax=tax, lax=lax)
    tax.set_position([(0.5-tax.get_position().width/2), tax.get_position().y0, tax.get_position().width, tax.get_position().height])
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    staticSavePath = path.join(basePath, "output", "products", modelName, "4pnl", runPathExt)
    Path(staticSavePath).mkdir(parents=True, exist_ok=True)
    gisInfo = ["0,0", "0,0"]
    productId = productTypeBase + 32
    if hasHelpers:
        HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
    else:
        fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
    plt.close(fig)

def staticDewWindMSLPPlot():
    fig = plt.figure()
    px = 1/plt.rcParams["figure.dpi"]
    fig.set_size_inches(1920*px, 1080*px)
    ax = plt.axes(projection=ccrs.LambertConformal())
    ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    contourmap = sfcDewPlot(False, ax=ax)
    sfcWindPlot(False, ax=ax)
    mslpPlot(False, ax=ax)
    ax.add_feature(cfeat.STATES.with_scale("50m"), linewidth=0.5)
    ax.add_feature(cfeat.COASTLINE.with_scale("50m"), linewidth=0.5)
    validTime = initDateTime + timedelta(hours=fhour)
    if "ecmwf" in modelName:
        noticeStr = "Copyright © "+initDateTime.strftime("%Y")+" European Centre for Medium-Range Weather Forecasts (ECMWF)\nhttps://www.ecmwf.int/"
    else:
        noticeStr = None
    if hasHelpers:
        HDWX_helpers.dressImage(fig, ax, initDateTime.strftime("%H")+"Z "+modelName.upper()+"\n2m Dew Point Temp, 10m Winds, MSLP", validTime, fhour=fhour, notice=noticeStr, plotHandle=contourmap, cbextend="both", colorbarLabel="Dew Point (°F)")
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    staticSavePath = path.join(basePath, "output/products/"+modelName+"/sfcTdWndMSLP/"+runPathExt)
    Path(staticSavePath).mkdir(parents=True, exist_ok=True)
    gisInfo = ["0,0", "0,0"]
    productId = productTypeBase + 5
    if hasHelpers:
        HDWX_helpers.saveImage(fig, path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
        HDWX_helpers.writeJson(basePath, productId, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
    else:
        fig.savefig(path.join(staticSavePath, "f"+str(fhour)+".png"), bbox_inches="tight")
    plt.close(fig)

def sfcDewPlot(standaloneFig, ax=None):
    pathToRead = path.join(inputPath, "td2m.grib2")
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore")
        modelDataArray = xr.open_dataset(pathToRead, engine="cfgrib", backend_kwargs={"indexpath" : ""})
    runPathExt = initDateTime.strftime("%Y/%m/%d/%H%M")
    gisSavePath = path.join(basePath, "output/gisproducts/"+modelName+"/sfcTd/", runPathExt)
    Path(gisSavePath).mkdir(parents=True, exist_ok=True)
    dewData = modelDataArray["d2m"]
    dewData = dewData.metpy.quantify()
    dewData = dewData.metpy.convert_units("degF")
    if standaloneFig:
        fig = plt.figure()
        px = 1/plt.rcParams["figure.dpi"]
        fig.set_size_inches(1920*px, 1080*px)
        ax = plt.axes(projection=ccrs.epsg(3857))
        ax.set_extent(axExtent, crs=ccrs.PlateCarree())
    if modelName == "gfs" or modelName == "ecmwf-hres":
        lonsToPlot = np.tile(np.array([dewData.longitude.data]), (dewData.data.shape[0], 1))
        latsToPlot = np.tile(dewData.latitude.data, (dewData.data.shape[1], 1)).transpose()
    else:
        lonsToPlot = dewData.longitude
        latsToPlot = dewData.latitude
    levelsToContour = np.arange(10, 91, 5)
    contourmap = ax.contourf(lonsToPlot, latsToPlot, dewData, levels=levelsToContour,  cmap="BrBG", transform=ccrs.PlateCarree(), transform_first=True, extend="both")
    contourLabels = ax.clabel(contourmap, levels=levelsToContour, inline=True, fontsize=10, colors="black")
    [label.set_rotation(0) for label in contourLabels]
    [label.set_text(label.get_text()+"F") for label in contourLabels]
    if standaloneFig:
        set_size(1920*px, 1080*px, ax=ax)
        extent = ax.get_tightbbox(fig.canvas.get_renderer()).transformed(fig.dpi_scale_trans.inverted())
        gisInfo = [str(axExtent[2])+","+str(axExtent[0]), str(axExtent[3])+","+str(axExtent[1])]
        validTime = initDateTime + timedelta(hours=fhour)
        if hasHelpers:
            HDWX_helpers.saveImage(fig, path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
            HDWX_helpers.writeJson(basePath, productTypeBase+4, initDateTime, "f"+str(fhour)+".png", validTime, gisInfo, 60)
        else:
            fig.savefig(path.join(gisSavePath, "f"+str(fhour)+".png"), transparent=True, bbox_inches=extent)
        plt.close(fig)
    return contourmap

if __name__ == "__main__":
    print(str("Plotting init "+str(initDateTime.hour)+"Z f"+str(fhour)+" "+modelName+" "+fieldToPlot))
    inputPath = path.join(basePath, "modelData/"+modelName+"/"+dt.strftime(initDateTime, "%Y%m%d")+"/"+dt.strftime(initDateTime, "%H")+"/"+str(fhour))
    sfcTempPath = path.join(inputPath, "t2m.grib2")
    sfcWindsPath = path.join(inputPath, "sfcwind.grib2")
    sfcPressPath = path.join(inputPath, "sp.grib2")
    windsAtHeightPath = path.join(inputPath, "winds.grib2")
    compositeReflectivityPath = path.join(inputPath, "refc.grib2")
    updraftHelicityPath = path.join(inputPath, "udh.grib2")
    aglReflectivityPath = path.join(inputPath, "refd.grib2")
    heightsPath = path.join(inputPath, "heights.grib2")
    tempsPath = path.join(inputPath, "temps.grib2")
    dewpointPath = path.join(inputPath, "dwpt.grib2")
    rhPath = path.join(inputPath, "rh.grib2")
    surfaceDewPath = path.join(inputPath, "td2m.grib2")
    if path.exists(inputPath):
        if fieldToPlot == "t2m" and path.exists(sfcTempPath) and "--no-gis" not in sys.argv:
            sfcTempPlot(True)
        if fieldToPlot == "sfcwind" and path.exists(sfcWindsPath) and "--no-gis" not in sys.argv:
            sfcWindPlot(True)
        if fieldToPlot == "sp" and path.exists(sfcPressPath) and path.exists(sfcTempPath) and "--no-gis" not in sys.argv:
            mslpPlot(True)
        if fieldToPlot == "sfccomposite" and path.exists(sfcTempPath) and path.exists(sfcWindsPath) and path.exists(sfcPressPath):
            staticSFCTempWindMSLPPlot()
        if fieldToPlot == "winds" and path.exists(windsAtHeightPath) and "--no-gis" not in sys.argv:
            [windsAtHeightPlot(pressSfc, True) for pressSfc in [250, 500, 850]]
        if fieldToPlot == "refc" and path.exists(compositeReflectivityPath) and "--no-gis" not in sys.argv:
            simReflectivityPlot("refc.grib2", True)
        if fieldToPlot == "udh" and path.exists(updraftHelicityPath) and "--no-gis" not in sys.argv:
            updraftHelicityPlot(True)
        if fieldToPlot == "refccomposite" and path.exists(compositeReflectivityPath) and path.exists(sfcWindsPath):
            if modelName in ["namnest", "hrrr"] and fhour != 0:
                if path.exists(updraftHelicityPath):
                    staticSimDBZPlot("refccomposite")
            else:
                staticSimDBZPlot("refccomposite")
        if fieldToPlot == "refd" and path.exists(aglReflectivityPath) and "--no-gis" not in sys.argv:
            simReflectivityPlot("refd.grib2", True)
        if fieldToPlot == "refdcomposite" and path.exists(aglReflectivityPath) and path.exists(sfcWindsPath):
            if modelName in ["namnest", "hrrr"] and fhour != 0:
                if path.exists(updraftHelicityPath) or fhour == 0:
                    staticSimDBZPlot("refdcomposite")
            else:
                staticSimDBZPlot("refdcomposite")
        if fieldToPlot == "heights" and path.exists(heightsPath) and "--no-gis" not in sys.argv:
            [heightsPlot(pressSfc, True) for pressSfc in [250, 500, 850]]
        if fieldToPlot == "temps" and path.exists(tempsPath) and "--no-gis" not in sys.argv:
            [tempsPlot(pressSfc, True) for pressSfc in [850]]
        if fieldToPlot == "rh" and path.exists(rhPath) and "--no-gis" not in sys.argv:
            [rhPlot(pressSfc, True) for pressSfc in [700]]
        if fieldToPlot == "500vort" and path.exists(heightsPath) and path.exists(windsAtHeightPath):
            vort500Plot(True)
        if fieldToPlot == "jetisotachs" and path.exists(heightsPath) and path.exists(windsAtHeightPath):
            jetIsotachsPlot(True)
        if fieldToPlot == "850temps" and path.exists(heightsPath) and path.exists(windsAtHeightPath) and path.exists(tempsPath):
            temps850Plot(True)
        if fieldToPlot == "700rh" and path.exists(sfcPressPath) and path.exists(heightsPath) and path.exists(rhPath):
            rh700Plot(True)
        if fieldToPlot == "4pnl" and path.exists(heightsPath) and path.exists(windsAtHeightPath) and path.exists(tempsPath) and path.exists(sfcPressPath) and path.exists(rhPath):
            fourPanelPlot()
        if fieldToPlot == "td2m" and path.exists(surfaceDewPath) and "--no-gis" not in sys.argv:
            sfcDewPlot(True)
        if fieldToPlot == "dewcomposite" and path.exists(surfaceDewPath) and path.exists(sfcTempPath) and path.exists(sfcWindsPath) and path.exists(sfcPressPath):
            staticDewWindMSLPPlot()


##
from scipy.io.arff import loadarff
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

class MyPreprocessing:
    def __init__(self, one_hot=False):
        pd.set_option('display.max_rows', 500)
        pd.set_option('display.max_columns', 500)
        pd.set_option('display.width', 1000)
        self.one_hot = one_hot

    def fit(self, df):
        # get label
        labels = df.iloc[:, -1]
        self.labels_ = labels
        labels_fac = pd.Series(pd.factorize(labels)[0])
        self.labels_fac = labels_fac
        df = df.drop(df.columns[len(df.columns) - 1], axis=1)
        #nan_cols = df.loc[:, df.isna().any()].columns

        # normalize numerical data
        df_num = df.select_dtypes(exclude=['object', 'bool'])
        df_obj = df.select_dtypes(include=['object', 'bool'])
        df_obj = df_obj.astype('object')
        #if df_num.size > 0:
        df_num = df_num.fillna(df_num.mean())
        df_num = pd.DataFrame(df_num, columns=df_num.columns)
        #else:
        #    df_normalized = pd.DataFrame()

        #if df_obj.size > 0:
        df_obj = df_obj.fillna('missing42')
        booleanDictionary = {True: 'TRUE', False: 'FALSE'}
        df_obj = df_obj.replace(booleanDictionary)
        if self.one_hot:
            col_2vals = np.array([df_obj[col].unique().size==2 for col in df_obj.columns])
            df_obj_2val = df_obj.iloc[:, col_2vals]
            df_obj_vals = df_obj.iloc[:, ~col_2vals]
            if df_obj_2val.empty:
                df_encoded_2val = pd.DataFrame()
            else:
                df_encoded_2val = pd.get_dummies(df_obj_2val,
                                    drop_first=True)
            if df_obj_vals.empty:
                df_encoded_vals = pd.DataFrame()
            else:
                df_encoded_vals = pd.get_dummies(df_obj_vals)

            df_obj = pd.concat(
                [df_encoded_2val, df_encoded_vals],
                axis=1,
                sort=False)

        self.new_df = pd.concat(
            [df_num, df_obj], #, labels_fac],
            axis=1,
            sort=False)


from collections import defaultdict
from functools import lru_cache
from typing import Dict, DefaultDict, Tuple, Generator

from covid19_il.logger.logger import Logger
from covid19_il.data_handler.data_handlers.data_handler import DataHandler

class AgeGender(DataHandler):
    """ Covid19_IL Age Gender Data Handler.

    Attributes:
        None.

    Methods:
        statistics_by_gender(self): Yields statistic data via first week day with results which grouped
            by gender..
        _get_statistics_by_columns_names(self, columns_names: Tuple): private method - Yields statistic data as
            generator via first week day with results which grouped by gender.
        statistics_by_given_first_week_day(self, week_day: str): Yields statistic data by given first week day grouped
            by gender.
        statistics_by_age_group(self): Yields statistic of calculated fields by ordered by age group.

    """

    calculated_fields = ('weekly_tests_num', 'weekly_newly_tested', 'weekly_cases', 'weekly_deceased')

    def __init__(self, logger: Logger.logger, json_data: Dict) -> None:
        """ Initialize Base Class & Instance Attributes """
        super().__init__(logger, json_data)

    @lru_cache
    def statistics_by_gender(self) -> \
            Generator[DefaultDict[str, DefaultDict[str, Dict[str, Dict[str, int or str]]]], None, None] or \
            Generator[str, None, None]:
        """ Yields statistic data via first week day with results which grouped by gender.

        Args:
            None.

        Yields:
            Tuple[str, DefaultDict[str, Dict[str, Dict[str, int or str]]] or str: desired data or "No Data" as
                bad result.

        """

        return self._get_statistics_by_columns_names(AgeGender.calculated_fields)

    def _get_statistics_by_columns_names(self, columns_names: Tuple) ->\
            Generator[DefaultDict[str, DefaultDict[str, Dict[str, Dict[str, int or str]]]], None, None] or \
            Generator[str, None, None]:
        """ Yields statistic data via first week day with results which grouped by gender.

        Note:
            private method which get called by other method for data statistic calculation.

        Args:
            columns_names(Tuple): required columns name of data frames for data manipulations.

        Yields:
            Tuple[str, DefaultDict[str, Dict[str, Dict[str, int or str]]]] or str: desired data or "No Data" as
                bad result.

        """

        data_dict = None
        try:
            df = self._get_clean_copy_df_data()
            ser = df.groupby([*df.columns])['gender']
            data = ser.unique()
            data_dict = defaultdict(lambda: defaultdict(lambda: dict()))
            for key in data.keys():
                # key[0]: first week day, key[2]: age_group, key[3]: gender, key[4:]: the actual values
                data_dict[key[3]][key[0]][key[2]] = {key: value for key, value in zip(columns_names, key[4:])}
                # data_dict[key[0]][key[3]][key[2]] = {key: value for key, value in zip(columns_names[1:], key[4:])}
        except KeyError as ke:
            self._logger.exception(ke, "No DataFrame's key exists according to the api client's query results")
        finally:
            if bool(data_dict):
                yield from data_dict.items()
            else:
                yield "No Data"

    @lru_cache
    def statistics_by_given_first_week_day(self, week_day: str) ->\
            Generator[DefaultDict[str, Dict[str, Dict[str, str]]], None, None] or Generator[str, None, None]:
        """ Yields statistic data by given first week day grouped by gender.

        Args:
            week_day(str): given week day for data manipulation.

        Yields:
            Tuple[str, Dict[str, Dict[str, str]] or str: desired data or "No Data" for bad result.

        """

        data_dict = None
        try:
            df = self._get_clean_copy_df_data()
            df = df[df['first_week_day'] == week_day]
            ser = df.groupby([*df.columns])['gender']
            data = ser.unique()
            data_dict = defaultdict(lambda: dict())
            for key in data.keys():
                # key[0]: first week day, key[2]: age_group, key[3]: gender, key[4:]: the actual values
                data_dict[key[3]][key[2]] = {key: value for key, value in zip(AgeGender.calculated_fields, key[4:])}
        except KeyError as ke:
            self._logger.exception(ke, "No DataFrame's key exists according to the api client's query results")
        finally:
            if bool(data_dict):
                yield from data_dict.items()
            else:
                yield "No Data"

    @lru_cache
    def statistics_by_age_group(self) ->\
            Generator[DefaultDict[str, DefaultDict[str, Dict[str, int or float]]], None, None] or\
            Generator[str, None, None]:
        """ Yields statistic of calculated fields by ordered by age group.

        Args:
            None.

        Yields:
            Tuple[str, [DefaultDict[str, Dict[str, int or float]] or str: desired data or "No Data" for bad result.
        """

        data_dict = None
        try:
            df = self._get_clean_copy_df_data()
            for column_name in AgeGender.calculated_fields:
                df[column_name] = [self._convert_string_to_int(item) for item in df[column_name]]
            age_groups = df['age_group'].unique()
            data_dict = defaultdict(lambda: dict())

            for group in age_groups:
                temp_df = df[df['age_group'] == group]
                for field in AgeGender.calculated_fields:
                    df_by_field = temp_df[field]
                    data_dict[group][field] = {"min": df_by_field.min(),
                                               "max": df_by_field.max(),
                                               "mean": df_by_field.mean(),
                                               "total": df_by_field.sum()}
        except KeyError as ke:
            self._logger.exception(ke, "No DataFrame's key exists according to the api client's query results")
        finally:
            if bool(data_dict):
                yield from data_dict.items()
            else:
                yield "No Data"


from sys import argv
import ast

import math

script, modelfile, testfile = argv

model_handle = open(modelfile,'r',errors='ignore')
learning = model_handle.read();

learnComponents = learning.split("\n");

PriorProbDict = dict()
PriorProbDict = ast.literal_eval(learnComponents[1])

classWordCount = dict()
classWordCount = ast.literal_eval(learnComponents[3])

uniqueWords = learnComponents[5]

wordProbOfClass = dict()
wordProbOfClass = ast.literal_eval(learnComponents[7])

listOfDicClass = []
'''
for k,v in wordProbOfClass.items():
	listOfDicClass.append(wordProbOfClass[k])
'''
name = modelfile.split(".")
ofile = name[0] + ".out"
outputhandle = open(ofile,'w',errors='ignore')
#outputhandle.write(str(listOfDicClass))
#print listOfDicClass

classList = []
for key in PriorProbDict:
	classList.append(key)

test_handle = open(testfile,'r',errors='ignore')
testcontent = test_handle.read()
docmts = testcontent.split("\n")
docmts.remove('')

finalpb = {}
#listOfDicClass

for d in docmts:
	doc = d.split(" ")
	for iclass in classList:
		dic = wordProbOfClass[iclass + "CountDict"]

		pb = float(PriorProbDict[iclass])
		#print pb
		for word in doc:
			if word in  dic:
				pb += dic[word]
			else:
				pb += math.log(1.0/(int(classWordCount[iclass + "CountDictwords"]) + int(uniqueWords)))
		finalpb[iclass] = pb

	maxval = float("-inf")
	classified = ""
	for k,v in finalpb.items():

		if(v > maxval):
			maxval = v
			classified = k
	outputhandle.write(classified)
	outputhandle.write("\n")
	finalpb = {}

outputhandle.close()


# -*- coding: utf-8 -*-
# __author__ = 'qiuping'
from biocluster.agent import Agent
from biocluster.tool import Tool
from biocluster.core.exceptions import OptionError
from collections import defaultdict
import pandas as pd
import os
from bson.objectid import ObjectId

class KeggRichstatAgent(Agent):
    """
    Kegg富集分析
    version v1.0.1
    author: qiuping
    last_modify: 2016.11.23
    """
    def __init__(self, parent):
        super(KeggRichstatAgent, self).__init__(parent)
        options = [
            {"name": "geneset_kegg_enrich_info", "type": "string"},
            # 筛选过后的基因集kegg详情
            {"name": "geneset_kegg_enrich_stat_id", "type": "string"},
            # 客户输入的gene名字文件
            #{"name": "geneset_name", "type": "string"}  # 基因集名称

        ]
        self.add_option(options)
        self.step.add_steps("kegg_enrich_stat")
        self.on('start', self.stepstart)
        self.on('end', self.stepfinish)

    def stepstart(self):
        self.step.kegg_enrich_stat.start()
        self.step.update()

    def stepfinish(self):
        self.step.kegg_enrich_stat.finish()
        self.step.update()

    def check_options(self):
        """
        重写参数检测函数
        :return:
        """
        return True

    def set_resource(self):
        """
        设置所需资源，需在之类中重写此方法 self._cpu ,self._memory
        :return:
        """
        self._cpu = 10
        self._memory = '4G'

    def end(self):
        result_dir = self.add_upload_dir(self.output_dir)
        result_dir.add_relpath_rules([
            [".", "", "结果输出目录"]
        ])
        result_dir.add_regexp_rules([
            [r"kegg_enrichment.xls$", "xls", "kegg富集分析结果"]
        ])
        super(KeggRichstatAgent, self).end()

class KeggRichstatTool(Tool):
    def __init__(self, config):
        super(KeggRichstatTool, self).__init__(config)

    def run(self):
        """
        运行
        :return:
        """
        super(KeggRichstatTool, self).run()
        list_custom = ['Metabolism', 'Genetic Information Processing', 'Environmental Information Processing',
                       'Cellular Processes', 'Organismal Systems', 'Human Diseases',
                       'Drug Development']
        new_data = pd.read_table(self.option("geneset_kegg_enrich_info"), sep='\t', header=0)
        new_data.groupby("second_category")
        group_obj = new_data.groupby("second_category")
        groups = group_obj.groups.keys()
        genesets = ["seq_list"]
        result = defaultdict(dict)
        for each in groups:
            first = new_data.loc[new_data["second_category"] == each]['first_category']
            first = first.to_dict().values()[0]
            for geneset in genesets:
                group_detail = group_obj.get_group(each)
                genes = list()
                for g in group_detail[geneset]:
                    if not pd.isnull(g):
                        tmp = g.split('|')
                        genes += [x for x in tmp]
                        genes = list(set(genes))
                    else:
                        genes = []
                result[geneset][each] = [len(genes), first]
        a = pd.DataFrame(result)
        a.reset_index(inplace=True)
        a.rename(columns={a.columns[0]: "second_category"}, inplace=True)
        a.to_csv(self.work_dir + "/" + "k", sep='\t', index=False)
        with open(self.work_dir + "/" + "k") as f1, open(self.work_dir + "/" + "kegg_statistic", "w") as fw:
            header = f1.readline()
            fw.write("first_category" + "\t" + header.strip().split("\t")[0] + "\t" + "num" + "\n")
            for line in f1:
                line_split = line.strip().split("\t")
                sec = line_split[0]
                num1 = line_split[1].strip("[]").split(",")[0]
                first_cate = line_split[1].strip("[]").split(",")[1].strip().strip("'")
                fw.write(first_cate + "\t" + sec + "\t" + num1 + "\n")
        df_a = pd.read_table(self.work_dir + "/" + "kegg_statistic", header=0, sep="\t")
        appended_data_new_1 = []
        for i in list_custom:
            if i in list(df_a.first_category):
                data = df_a.loc[df_a['first_category'] == i]
                appended_data_new_1.append(data)

        appended_data_new_1 = pd.concat(appended_data_new_1)
        data_new = appended_data_new_1.to_dict('records')
        appended_data_new_1.to_csv(self.work_dir + "/" + "kegg_statistic", sep='\t', index=False)
        self.end()


pila = []
es_pila = 1
from sys import stdin
for line in stdin:
    op, num = map(int, line.split())
    if(op == 1 ):
        pila.append(num)
    else:
        if(pila.pop() != num): es_pila = 0

if(es_pila): print("Pila")
else: print("Cola")


#词性标注
'''jieba.posseg.POSTokenizer(tokenizer=None)新建自定义分词，
tokenizer参数可指定内部使用的jieba.Tokenizer分词器。jieba
.posseg.dt为默认词性标注分词器。
标注句子分词后每个词的词性，采用ictclas兼容的标记法。'''

#官方历程
import jieba.posseg as pseg

words=pseg.cut("我爱学习蟒蛇")
#words类别为：generator

for word,flag in words:
    print('%s %s'%(word,flag))

Python 数据分析/结巴分词和词云/词性标注.py


def InsertionSort(ar):
    length = len(ar)
    for i in range(1, length):
        '''
        "first" is assigned the second element as the first element is already taken as sorted.
        '''
        first = ar[i]
        j = i-1
        '''
        "j" moves from last element of sorted values to first element of sorted values.

        Example:
        ar = [3, 2, 1, 4, 5, 9, 8, 7, 6]

        first = 2
        2 < 3 --> ar = [3, 3, 1, 4, 5, 9, 8, 7, 6] --> ar = [2, 3, 1, 4, 5, 9, 8, 7, 6]

        first = 1
        1 < 3 --> ar = [2, 3, 3, 4, 5, 9, 8, 7, 6]
        1 < 2 --> ar = [2, 2, 3, 4, 5, 9, 8, 7, 6] --> ar = [1, 2, 3, 4, 5, 9, 8, 7, 6]

        so on...

        '''
        while j >= 0 and first < ar[j]:
            ar[j+1] = ar[j]
            j -= 1
        ar[j+1] = first

ar = [3, 2, 1, 4, 5, 9, 8, 7, 6]
InsertionSort(ar)
print(ar)


import pyglet
from pyglet import shapes
import random
from cell import Cell

width = 2500
height = 1300
window = pyglet.window.Window(width, height)
scale = 10
cols = width // scale
rows = height // scale
batch = pyglet.graphics.Batch()
width = width - scale
height = height - scale
DEAD = [0, 0, 0]
ALIVE = [255, 255, 255]
PAUSED = False
LIVELIHOOD = 0.45
cells = []
alreadyvisited = []
lastshownplace = None
mousepos = 0, 0
figureToPlace = None
figures = [
    [(0, -1), (1, -1), (1, 0), (0, 0)],
    [(-4, 0), (-4, -1), (-4, 1), (-3, -2), (-3, 2), (-2, -3), (-2, 3), (-1, -3), (-1, 3), (0,0), (1, -2), (1, 2), (2, -1), (2, 0), (2, 1), (3, 0)],
    [(-7, -8), (-7, 7), (-7, -7), (-7, 6), (-7, -6), (-7, 5),
    (-6, -5), (-6, 4), (-6, -8), (-6, 7),
    (-5, -8), (-5, 7),
    (-4, -4), (-4, 3), (-4, -8), (-4, 7),
    (-3, -4), (-3, 3), (-3, -8), (-3, 7),
    (-2, -3), (-2, 2), (-2, 6), (-2, -7),
    (-1, -2), (-1, 1),
    (0, 0), (0, -1),
    (1, 1), (1, -2), (1, -4), (1, 3),
    (2, -4), (2, 3),
    (3, -3), (3, 2), (3, -2), (3, 1),
    (4, -5), (4, 4), (4, -4), (4, 3),
    (5, -6), (5, 5), (5, -5), (5, 4), (5, -4), (5, 3), (5, -2), (5, 1),
    (6, -7), (6, 6), (6, -6), (6, 5), (6, -3), (6, 2),
    (7, -7), (7, 6), (7, -5), (7, 4),
    (8, -6), (8, 5), (8, -5), (8, 4), (8, -4), (8, 3),
    (9, -6), (9, 5), (9, -5), (9, 4), (9, -4), (9, 3),
    (10, -6), (10, 5), (10, -5), (10, 4)
    ]

]

for j in range(0, rows):
    for i in range(0, cols):
        state = DEAD
        if random.random() > LIVELIHOOD:
            state = ALIVE
        gameobject = shapes.Rectangle(0+i*scale, height-j*scale,
                                        scale-1, scale-1, color=state,
                                        batch=batch)
        cells.append(Cell(gameobject, i, j, state))

for cell in cells:
    cell.getNeighbours(cells, cols)

def calculateNextBoard():
    global cells
    for cell in cells:
        cell.countAliveNeighbours()
    for cell in cells:
        cell.setState()
        cell.setColor()

def cellCountAliveNeighbours(cells):
    for cell in cells:
        cell.countAliveNeighbours()

def cellSetState(cells):
    for cell in cells:
        cell.setState()

def cleanEverything():
    global cells
    for cell in cells:
        cell.die()

@window.event
def on_draw():
    window.clear()
    batch.draw()

@window.event
def on_key_press(symbol, modifiers):
    global PAUSED, figures
    if chr(symbol) == 'p':
        PAUSED = not PAUSED
    if chr(symbol) == 'c':
        PAUSED = True
        cleanEverything()
    if chr(symbol) == '1':
        PAUSED = True
        # figureToPlace = figures[1]
        showFigure(figures[0])
    if chr(symbol) == '2':
        PAUSED = True
        # figureToPlace = figures[1]
        showFigure(figures[1])
    if chr(symbol) == '3':
        PAUSED = True
        # figureToPlace = figures[1]
        showFigure(figures[2])

def showFigure(figure):
    global cells, mousepos, height, cols, lastshownplace
    i, j = mousepos
    x = (i // scale)
    y = (height - j + scale//2)//scale
    lookup = cols * y + x
    origin = cells[lookup]
    offsets = figure
    if lastshownplace != (x, y) and lastshownplace is not None:
        for offset in offsets:
            xoffset = offset[0]
            yoffset = offset[1]
            lastshownplacex = lastshownplace[0]
            lastshownplacey = lastshownplace[1]
            neighbourlookup = (cols * (lastshownplacey + yoffset) + (lastshownplacex + xoffset)) % len(cells) - 1
            neighbour = cells[neighbourlookup]
            neighbour.state = DEAD
            neighbour.gameobject.color = DEAD
    for offset in offsets:
        xoffset = offset[0]
        yoffset = offset[1]
        neighbourlookup = (cols * (y + yoffset) + (x + xoffset)) % len(cells) - 1
        neighbour = cells[neighbourlookup]
        neighbour.state = ALIVE
        neighbour.gameobject.color = ALIVE
    lastshownplace = (x, y)
    pass

@window.event
def on_mouse_press(x, y, button, modifiers):
    print('mousepress')
    global height
    x = x//scale
    y = (height - y + scale//2)//scale
    global cells, cols, ALIVE, DEAD
    lookup = cols * y + x
    print(cells[lookup])
    state = cells[lookup].state
    if state != DEAD:
        state = DEAD
    elif state == DEAD:
        state = ALIVE
    print(state)
    cells[lookup].state = state
    cells[lookup].gameobject.state = state
    print(cells[lookup].state, cells[lookup].gameobject.state)

@window.event
def on_mouse_drag(x, y, dx, dy, buttons, modifiers):
    print('mousedrag')
    global height, cells, cols, ALIVE, DEAD, alreadyvisited
    x = x//scale
    y = (height - y + scale//2)//scale
    lookup = cols * y + x
    if lookup not in alreadyvisited:
        state = cells[lookup].state
        if state != DEAD:
            state = DEAD
        elif state == DEAD:
            state = ALIVE
        cells[lookup].state = state
        cells[lookup].gameobject.state = state
    alreadyvisited.append(lookup)

@window.event
def on_mouse_release(x, y, button, modifiers):
    global alreadyvisited
    alreadyvisited = []

@window.event
def on_mouse_motion(x, y, dx, dy):
    global mousepos
    mousepos = x, y

def update(dt):
    window.clear()
    batch.draw()
    global PAUSED
    if not PAUSED:
        calculateNextBoard()

pyglet.clock.schedule_interval(update, 1/20)
pyglet.app.run()


#!/anaconda3/envs/FEALPy/bin python3.8
# -*- coding: utf-8 -*-
# ---
# @Software: PyCharm
# @File: HHOSolver.py
# @Author: Yongchao Zhang, Northwest University, Xi'an
# @E-mail: yoczhang@nwu.edu.cn
# @Site:
# @Time: Sep 29, 2020
# ---

__doc__ = """
The solver hybrid high-order (HHO) method.
"""

import numpy as np
from numpy.linalg import solve, inv
from scipy.sparse.linalg import spsolve
from scipy.sparse import coo_matrix, csc_matrix, csr_matrix, spdiags, eye, bmat
from timeit import default_timer as timer
from multiprocessing.dummy import Pool as ThreadPool

class HHOSolver:
    def __init__(self, M, R, space, pde):
        self.M = M
        self.R = R
        self.space = space
        self.pde = pde
        self.p = self.space.p
        self.mesh = self.space.mesh
        self.NC = self.mesh.number_of_cells()
        self.NE = self.mesh.number_of_edges()

    def StokesSolver(self):
        vSpace = self.space.vSpace
        nu = self.pde.nu
        M = self.M
        R = self.R
        p = self.p
        mesh = self.mesh
        NC = self.NC
        NE = self.NE

        # # 'l' denotes 'local';
        # # 'g' denotes 'global'.
        TlNdof = (p + 1) * (p + 2) // 2
        uTlNdof = TlNdof
        uFlNdof = p + 1
        pTlNdof = TlNdof - 1
        pFlNdof = 1

        uTgNdof = NC * uTlNdof
        uFgNdof = NE * uFlNdof
        ugNdof = uTgNdof + uFgNdof
        pTgNdof = NC * pTlNdof
        pFgNdof = NC * pFlNdof
        pgNdof = pTgNdof + pFgNdof

        # --- matrix setting --- #
        MA1_00 = M[:uTgNdof, :uTgNdof]
        MA1_0b = M[:uTgNdof, uTgNdof:ugNdof]
        MA1_b0 = M[uTgNdof:ugNdof, :uTgNdof]
        MA1_bb = M[uTgNdof:ugNdof, uTgNdof:ugNdof]

        MA2_00 = MA1_00.copy()
        MA2_0b = MA1_0b.copy()
        MA2_b0 = MA1_b0.copy()
        MA2_bb = MA1_bb.copy()

        MB1_00 = M[2*ugNdof:(2*ugNdof + pgNdof), :uTgNdof]
        MB1_0b = M[2*ugNdof:(2*ugNdof + pgNdof), uTgNdof:ugNdof]
        MB2_00 = M[2*ugNdof:(2*ugNdof + pgNdof), ugNdof:(ugNdof + uTgNdof)]
        MB2_0b = M[2*ugNdof:(2*ugNdof + pgNdof), (ugNdof + uTgNdof):2*ugNdof]

        MB1_00t = M[:uTgNdof, 2*ugNdof:(2*ugNdof + pgNdof)]
        MB1_b0 = M[uTgNdof:ugNdof, 2*ugNdof:(2*ugNdof + pgNdof)]
        MB2_00t = M[ugNdof:(ugNdof + uTgNdof), 2*ugNdof:(2*ugNdof + pgNdof)]
        MB2_b0 = M[(ugNdof + uTgNdof):2*ugNdof, 2*ugNdof:(2*ugNdof + pgNdof)]

        # # the right hand side
        r0_0 = R[:uTgNdof, :]
        r0_1 = R[ugNdof:(ugNdof+uTgNdof), :]
        R0 = np.concatenate([r0_0, r0_1, np.zeros((pTgNdof, 1), dtype=np.float)])  # (2*uTgNdof+pTgNdof, 1)
        Rb = np.zeros((2*uFgNdof+pFgNdof+1, 1), dtype=np.float)

        # # the Lagrange multiplier vector (for the pressure condition: \int p = 0)
        # # In HHO-Stokes equation, the velocity and pressure have the same order, here we just use the velocity space.
        pphi = vSpace.basis
        intp = vSpace.integralalg.integral(pphi, celltype=True)  # (NC,pldof)

        LF = intp[:, 0][np.newaxis, :]  # (1,NC)
        LT = intp[:, 1:].reshape(1, -1)  # (1,...)

        # # to reconstruct the global matrix
        pgIdx = np.arange(0, pgNdof, TlNdof)  # (NC,)
        MB1_bb = MB1_0b[pgIdx, :]  # (NC,uFgNdof)
        MB2_bb = MB2_0b[pgIdx, :]  # (NC,uFgNdof)
        MB1_bbt = MB1_b0[:, pgIdx]  # (uFgNdof,NC)
        MB2_bbt = MB2_b0[:, pgIdx]  # (uFgNdof,NC)

        MB1_00 = np.delete(MB1_00.todense(), pgIdx, axis=0)
        MB1_0b = np.delete(MB1_0b.todense(), pgIdx, axis=0)
        MB2_00 = np.delete(MB2_00.todense(), pgIdx, axis=0)
        MB2_0b = np.delete(MB2_0b.todense(), pgIdx, axis=0)

        MB1_00t = np.delete(MB1_00t.todense(), pgIdx, axis=1)
        MB1_b0 = np.delete(MB1_b0.todense(), pgIdx, axis=1)
        MB2_00t = np.delete(MB2_00t.todense(), pgIdx, axis=1)
        MB2_b0 = np.delete(MB2_b0.todense(), pgIdx, axis=1)

        # --- get global matrix --- #
        VP_F0 = np.zeros((pFgNdof, uTgNdof))
        LF0 = np.zeros((1, uFgNdof))
        LT0 = np.zeros((1, uTgNdof))
        A = bmat([[MA1_00, None, MB1_00t], [None, MA2_00, MB2_00t], [MB1_00, MB2_00, None]], format='csr')
        B = bmat([[MA1_0b, None, VP_F0.T, LT0.T], [None, MA2_0b, VP_F0.T, LT0.T], [MB1_0b, MB2_0b, None, LT.T]], format='csr')
        C = bmat([[MA1_b0, None, MB1_b0], [None, MA2_b0, MB2_b0], [VP_F0, VP_F0, None], [LT0, LT0, LT]], format='csr')
        D = bmat([[MA1_bb, None, MB1_bbt, LF0.T], [None, MA2_bb, MB2_bbt, LF0.T], [MB1_bb, MB2_bb, None, LF.T], [LF0, LF0, LF, None]], format='csr')

        # # to get the inv A
        cell2dof, doflocation = vSpace.dof.cell_to_dof()
        StiffM = vSpace.reconstruction_stiff_matrix()  # list, (NC,), each-term.shape (Cldof,Cldof)
        StabM = vSpace.reconstruction_stabilizer_matrix()  # list, (NC,), each-term.shape (Cldof,Cldof)
        divM0, divM1 = self.space.cell_divergence_matrix()  # e.g., divM0, list, (NC,); divM0: (TlNdof,\sum_C{Cldof})
        divM0_split = np.hsplit(divM0, doflocation[1:-1])
        divM1_split = np.hsplit(divM1, doflocation[1:-1])

        AlNdof = 2*uTlNdof + pTlNdof
        invA = np.zeros((NC * AlNdof, NC * AlNdof), dtype=np.float)

        def func_invA(x):
            r = None
            stiffM_TT = nu * x[0][:uTlNdof, :uTlNdof]
            stabM_TT = nu * x[1][:uTlNdof, :uTlNdof]
            divM0_TT = x[2][1:, :uTlNdof]
            divM1_TT = x[3][1:, :uTlNdof]
            CIdx = x[4]

            Al = np.zeros((AlNdof, AlNdof), dtype=np.float)

            Al[:uTlNdof, :uTlNdof] = stiffM_TT + stabM_TT
            Al[uTlNdof:2*uTlNdof, uTlNdof:2*uTlNdof] = stiffM_TT + stabM_TT
            Al[2*uTlNdof:, :uTlNdof] = divM0_TT
            Al[2*uTlNdof:, uTlNdof:2*uTlNdof] = divM1_TT
            Al[:uTlNdof, 2*uTlNdof:] = divM0_TT.T
            Al[uTlNdof:2*uTlNdof, 2*uTlNdof:] = divM1_TT.T

            u0Idx = CIdx * uTlNdof + np.arange(uTlNdof)
            u1Idx = uTgNdof + u0Idx
            pIdx = 2 * uTgNdof + CIdx * pTlNdof + np.arange(pTlNdof)
            Idx = np.concatenate([u0Idx, u1Idx, pIdx])

            # # one way assembling, this way is more quickly
            i, j = np.ix_(Idx, Idx)
            invA[i, j] = inv(Al)

            # # other way assembling
            # rowIdx = np.einsum('i, k->ik', Idx, np.ones((AlNdof, ), dtype=np.int))
            # colIdx = rowIdx.T
            # r = csr_matrix((inv(Al).flat, (rowIdx.flat, colIdx.flat)), shape=(NC * AlNdof, NC * AlNdof), dtype=np.float)
            return r

        start = timer()
        # invAt = sum(list(map(func_invA, zip(StiffM, StabM, divM0_split, divM1_split, range(NC)))))
        invAt = list(map(func_invA, zip(StiffM, StabM, divM0_split, divM1_split, range(NC))))
        # pool = ThreadPool()
        # invAt = sum(pool.map(func_invA, zip(StiffM, StabM, divM0_split, divM1_split, range(NC))))
        # pool.close()
        # pool.join()
        end = timer()
        print("  |___ TIME: get inv matrix (cell-by-cell):", end - start)

        # start = timer()
        # invAtt = inv(A)
        # invAtt = np.linalg.inv(A.todense())
        # end = timer()
        # print("get inv matrix (directly):", end - start)
        # print("max( abs(invA - inv(A)) ) = ", np.max(abs(invAtt-invA)))

        # --- solve the Static Condensation system --- #
        # print("solve system:")
        invA = csr_matrix(invA)  # To makesure invA is sparse matrix is important!!!
        stacM = D - C@invA@B
        stacR = Rb - C@invA@R0

        # # deal the Dirichlet boundary edges
        uD = self.pde.dirichlet  # uD(bcs): (NQ,NC,ldof,2)
        idxDirEdge = self.pde.idxDirEdge
        MD, RD = self.space.applyDirichletBC(stacM, stacR, uD, idxDirEdge=idxDirEdge, StaticCondensation=True)

        # # solve the system
        print("  |___ Begin solve 'sparse' algebraic equations:")
        start = timer()
        Xb = spsolve(MD, RD)
        end = timer()
        print("  |___ TIME: in static solver, solve 'sparse' algebraic equations:", end - start)

        X0 = invA@(np.squeeze(R0) - B@Xb)

        # # rearrange the dofs
        X = np.zeros((2*ugNdof + pgNdof + 1,))
        X[:uTgNdof] = X0[:uTgNdof]
        X[uTgNdof:ugNdof] = Xb[:uFgNdof]
        X[ugNdof:ugNdof+uTgNdof] = X0[uTgNdof:2*uTgNdof]
        X[ugNdof+uTgNdof:2*ugNdof] = Xb[uFgNdof:2*uFgNdof]

        if pTlNdof == 0:
            P = Xb[2*uFgNdof:-1].reshape(-1, 1)  # (PFgNdof,1)
        else:
            P0 = X0[2 * uTgNdof:].reshape(-1, pTlNdof)  # (pTgNdof,pTlNdof)
            Pb = Xb[2 * uFgNdof:-1].reshape(-1, 1)  # (PFgNdof,1)
            P = np.concatenate([Pb, P0], axis=1)

        X[2*ugNdof:-1] = P.flatten()
        return X

    def StokesSolver_1(self):
        vSpace = self.space

        # --- the Poisson (scalar) HHO space
        gdof_scal = self.space.dof.number_of_global_dofs()
        StiffM = self.space.reconstruction_stiff_matrix()  # list, its len is NC, each-term.shape (Cldof,Cldof)
        StabM = self.space.reconstruction_stabilizer_matrix()  # list, its len is NC, each-term.shape (Cldof,Cldof)

        # --- the divergence term
        divM0, divM1 = self.space.cell_divergence_matrix()  # divM0, list, (NC,); divM1, list, (NC,)


from playwright.sync_api import expect

class CommonTools:
    def __init__(self, page):
        self.page = page

    def check_page_is_loaded(self, page_title):
        title = self.page.get_by_text(page_title)
        try:
            expect(title).to_be_visible()
        except:
            self.page.reload()
            expect(title).to_be_visible()


import os
import random
import torch
from torch import nn, optim
from torch.utils import data
from torch.utils.tensorboard import SummaryWriter
import numpy as np
from scipy.io import wavfile
from .model import CatsDogsModel
from .dataset import BarkMeowDB

# Define some constants used throughout the training
AUDIO_DIR = './barkmeow'  # directory that contains 'cat' and 'dog' directories
DATA_OUT_ROOT = './data_out'
# directory where tensorboard logs are stored
LOG_DIR = os.path.join(DATA_OUT_ROOT, 'logs')
MODEL_PATH = os.path.join(DATA_OUT_ROOT, 'model.pt')
DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
LR = 3e-4  # learning rate
EPOCH = 100
SAMPLE_RATE = 16000  # sample rate - used to save audio of mispredicted examples

# fix random seed to make the training deterministic
torch.manual_seed(8912)
np.random.seed(9912)

writer = SummaryWriter(log_dir=LOG_DIR)

model = nn.DataParallel(CatsDogsModel()).to(DEVICE)

# read dataset and create dataloaders
dataset = BarkMeowDB(AUDIO_DIR)
num_data = len(dataset)
num_train = int(num_data * 0.9)

all_indices = list(range(num_data))
random.shuffle(all_indices)

train_dataloader = data.DataLoader(
    dataset,
    batch_size=5,
    sampler=data.sampler.SubsetRandomSampler(all_indices[:num_train]),
    pin_memory=True,
    drop_last=True)
test_dataloader = data.DataLoader(
    dataset,
    batch_size=5,
    sampler=data.sampler.SubsetRandomSampler(all_indices[num_train:]),
    pin_memory=True,
    drop_last=True)

optimizer = optim.Adam(model.parameters(), lr=LR, weight_decay=0.01)

# binary cross entropy loss
loss_func = nn.BCEWithLogitsLoss()

for i in range(EPOCH):
    losses = []
    accs = []
    for targets, samples in train_dataloader:
        targets = targets.type(torch.FloatTensor).to(DEVICE)
        samples = samples.to(DEVICE)

        out = torch.squeeze(model(samples))
        loss = loss_func(out, targets)

        out_bool = torch.sigmoid(out) > 0.5
        target_bool = targets > 0.5
        pred = (out_bool == target_bool).type(torch.FloatTensor)
        accuracy = pred.sum() / pred.numel()

        # accumulate accuracy and loss values for later calculating mean values of each
        accs.append(accuracy.data)
        losses.append(loss.data)

        # back-prop
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    # log values
    avg_loss = sum(losses) / len(losses)
    avg_acc = sum(accs) / len(accs)
    writer.add_scalar('train/loss', avg_loss, global_step=i)
    writer.add_scalar('train/accuracy', avg_acc, global_step=i)
    print(f'epoch: {i}, loss: {avg_loss}, accuracy: {avg_acc}')

    test_losses = []
    test_accs = []
    with torch.no_grad():
        for targets, samples in test_dataloader:
            targets = targets.type(torch.FloatTensor).to(DEVICE)
            samples = samples.to(DEVICE)
            out = torch.squeeze(model(samples))
            loss = loss_func(out, targets)

            out_bool = torch.sigmoid(out) > 0.5
            target_bool = targets > 0.5
            pred = (out_bool == target_bool).type(torch.FloatTensor)
            accuracy = pred.sum() / pred.numel()

            test_accs.append(accuracy.data)
            test_losses.append(loss.data)

            # save any that are mispredicted at the end of training
            if i == EPOCH - 1:
                for target_class, samp, pred_score in zip(targets, samples, out):
                    pred_class = 1 if pred_score > 0 else 0
                    if target_class != pred_class:
                        # misclassified class name
                        as_ = dataset.class_label_name[pred_class]

                        fname = f'mispred_as_{as_}_{pred_score:04f}.wav'
                        mispred_path = os.path.join(DATA_OUT_ROOT, fname)
                        audio_data = samp.cpu().detach().numpy().T

                        # save misclassified audio
                        wavfile.write(mispred_path, SAMPLE_RATE, audio_data)
                        print(f'mispredicted (as: {as_}): {pred_score:04f}')

    avg_test_acc = sum(test_accs) / len(test_accs)
    avg_test_loss = sum(test_losses) / len(test_losses)
    print(f'TEST: {i}, loss:, {avg_test_loss}, accuracy: {avg_test_acc}')
    writer.add_scalar('test/loss', avg_test_loss, global_step=i)
    writer.add_scalar('test/accuracy', avg_test_acc, global_step=i)

# save module
print(f'Saving model: {MODEL_PATH}')
torch.save(model.module.state_dict(), MODEL_PATH)


from app.scrapper import get_page
from app.providers.animeflv.resources import Anime, Episode

class AnimeFLV:
    airing_animes = []
    premiere_animes = []
    latest_episodes = []
    latest_animes = []

    def __init__(self, base_url):
        self.base_url = base_url
        self.page = get_page(base_url)

        self.get_latest_episodes()
        self.get_airing_animes()
        self.get_premiere_animes()

    def _get_anime_id(self, url_path):
        return url_path.split("/")[-1].split(".")[0]

    def get_latest_episodes(self):
        #
        #
        # Episodio 5
        # Hyouken no Majutsushi ga Sekai wo Suberu
        #
        for episode in self.page.find_all("a", class_="fa-play"):
            thumbnail = episode.find("img")["src"]
            anime_id = self._get_anime_id(thumbnail)
            anime_title = episode.find("strong").text

            anime = Anime(id=int(anime_id), title=anime_title)
            self.latest_animes.append(anime)

            link = episode["href"]
            number = episode.find("span", class_="Capi").text.split(" ")[1]
            self.latest_episodes.append(
                Episode(number=int(number), link=link, anime=anime)
            )

    def get_airing_animes(self):
        # Kyuuketsuki Sugu Shinu 2 Anime
        for anime in self.page.find_all("a", class_="fa-play-circle"):
            title = anime.text
            slug = anime["href"].split("/")[-1]
            _type = anime.find("span", class_="Type tv").text
            is_anime = _type.lower() == "anime"

            if not is_anime:
                continue
            self.airing_animes.append(Anime(title=title, slug=slug))

    def get_premiere_animes(self):
        #

        #
        # ESTRENO

        #

        # Anime
        #

        #
Majutsushi Orphen Hagure Tabi: Urbanrama-hen

        #
        #

        #
Majutsushi Orphen Hagure Tabi: Urbanrama-hen

        #
Anime 4.1

        #
Tercera temporada de Majutsushi Orphen Hagure Tabi

        # VER ANIME
        #

        #

        for anime in self.page.find_all("article", class_="Anime alt B"):
            title = anime.find("h3", class_="Title").text
            slug = anime.find("a")["href"].split("/")[-1]
            score = anime.find("span", class_="Vts fa-star").text
            cover = anime.find("img")["src"]
            anime_id = self._get_anime_id(cover)
            _type = anime.find("span", class_="Type tv").text
            is_anime = _type.lower() == "anime"

            if not is_anime:
                continue
            self.premiere_animes.append(
                Anime(id=int(anime_id), title=title, slug=slug, score=float(score))
            )


import requests
from bs4 import BeautifulSoup
url1 = "https://cathay-ds-test.s3-ap-northeast-1.amazonaws.com/user_info?userid=A123456789"
url2="https://cathay-ds-test.s3-ap-northeast-1.amazonaws.com/company_info?companyid=1"
response =requests.get (url1)
soup=BeautifulSoup(response.text, "html.parser")
print("url1",soup.prettify())
response =requests.get (url2)
soup=BeautifulSoup(response.text, "html.parser")
print("url2",soup.prettify())
status1=requests.head(url1)
status2=requests.head(url2)

class TestHtml1:

    def test_case2(self):
        assert status1.status_code==200
class TestHtml2:

    def test_case3(self):
        assert status2.status_code==403


# $Id$

__revision__ = '$Rev$'

def rotate(minval, maxval, ifile):
    for line in ifile:
        words = line.split()
        for x in words:
            print (float(x) - minval) % (maxval - minval) + minval,
        print

def main():
    import sys
    if len(sys.argv) != 4:
        import os.path
        sys.stderr.write('Usage: %s min max datafile|-\n' % os.path.basename(sys.argv[0]))
        sys.exit(1)
    if sys.argv[3] == '-':
        rotate(float(sys.argv[1]), float(sys.argv[2]), sys.stdin)
    else:
        rotate(float(sys.argv[1]), float(sys.argv[2]), file(sys.argv[3]))

if __name__ == '__main__':
    main()


''' Given three integers x,y and z representing the dimensions of a cuboid along with an integer n.
    Print a list of all possible coordinates given by (x,y,z) on a 3D where the sum of i+j+k is not equal to n'''

x = int(input())
y = int(input())
z = int(input())
n = int(input())
templst=[]
lst=[]
for a in range(x+1):
    for b in range(y+1):
       for c in range(z+1):
            if a+b+c!=n:
                templst.append(a)
                templst.append(b)
                templst.append(c)
                lst.append(templst)
            templst=[]
print(lst)


import uuid
from datetime import datetime
from flask import request
from flask_socketio import emit
from pony.orm import db_session
from src.entities.User import User
from src.middlewares.jwt import jwt_decode_user
from src.sockets.notifications import get_user_notifications

def page(socketio):
	@socketio.on_error('')
	def error_handler(e):
		emit('TOKEN_EXPIRED')

	@socketio.on('JOIN_PAGE')
	@jwt_decode_user
	@db_session
	def join_page():
		try:
			print(request.user['name'], 'has joined to page')
			user_id = request.user['id']
			user = User.get(id=uuid.UUID(user_id))
			user.last_logged = datetime.now()
			emit(f'NOTIFICATIONS_{user_id}', get_user_notifications(user_id))
		except Exception as e:
			print(e)

	@socketio.on('disconnect')
	@jwt_decode_user
	@db_session
	def disconnect():
		user_id = request.user['id']
		user = User.get(id=uuid.UUID(user_id))
		user.last_logged = datetime.now()
		print(request.user['name'], 'left the page')


import json
import glob

csvFileName = 'finalData.csv'

### some have missing header columns => causes shifts
header = ['id', 'title', 'link', 'Oferit de', 'Categorie', 'Marca', 'Model', 'Versiune',\
		'Anul fabricatiei', 'Kilometraj', 'Capacitate cilindrica', 'VIN', \
		'Combustibil', 'Putere', 'Cutie de viteze', 'Transmisie', 'Norma de poluare', \
		'Filtru de particule', 'Caroserie', 'Culoare', 'Primul proprietar', \
		'Fara accident in istoric', 'Carte de service', 'Stare']

headerBool = 0

if __name__ == '__main__':
	fileNames = glob.glob('carData*')

	with open(csvFileName, 'w') as f:

		###Header
		headerString = ''
		for h in header:
			headerString += h + ','
		f.write(headerString[:-1] + '\n')

		###Data
		for fileName in fileNames:
			with open(fileName, 'r') as jsonFile:
				res = json.loads(jsonFile.read())
				for entry in res:
					dataString = ''

					keys = entry.keys()
					for head in header:
						if head in keys:
							dataString += str(entry[head]) + ','
						else:
							dataString += 'NODATA,'

					f.write(dataString[:-1] + '\n')


import os, shutil, time
import PySimpleGUI as sg

def automate(path=None):
    files = os.listdir(path)
    if files != []:
        for f in files:
            splitted_file_name = f.split('.')
            extention = splitted_file_name[-1]
            try:
                if not os.path.exists(f"{path}/{extention}"):
                    os.mkdir(f"{path}/{extention}")
                if extention in f:
                    src = f"{path}/{f}"
                    dst = f"{path}/{extention}"
                    shutil.move(src,dst)
            except FileExistsError or FileNotFoundError:
                print("Error")
    else:
        return "empty"

def get_files(path=None):
    files = os.listdir(path)
    if files != []:
        return files
    else:
        return "empty"

layout = [
    [sg.Text("Enter Directory"), sg.Input(key='path')],
    [sg.Button("Manage"), sg.Button("Show"), sg.FileBrowse("Browse")],
    [sg.Text("Press Manage to move",key='ptext'),sg.ProgressBar(100,size=(28,20),key='pbar',bar_color=('lightgreen','snow'))],
    [sg.Listbox([],size=(58,10),key='file-list',no_scrollbar=False),],
]

window = sg.Window("File Manager", layout, finalize=True,)

listbox = window['file-list']
progressbar = window['pbar']
progressbar.hide_row()
listbox.hide_row()

while True:
    event, values = window.read()

    if event in ("Exit", None):
        break

    input_path = values['path']
    if input_path is not None and input_path != " " and input_path != "":
        try:
            total_files = 100 if len(get_files(input_path)) < 100 else len(get_files(input_path))
        except:
            sg.popup_error("Enter a valid directory",title="Error")

        if event == "Show":
            if get_files(input_path) == "empty":
                sg.popup("Enter a valid directory.")
            else:
                listbox.unhide_row()
                listbox.update(get_files(input_path))

        if event == "Manage":
            if get_files(input_path) != "empty":
                progressbar.update(total_files)
                progressbar.unhide_row()
                automate(input_path)
                for i in range(total_files+1):
                    window['pbar'].UpdateBar(i)
                    window['ptext'].update(f"Automated {i}")
                    time.sleep(0.009)
                sg.popup("Files moved successfully.", title="Message")
            else:
                sg.popup_error("Enter a valid directory",title='Error')
    else:
        sg.popup_error("Enter a valid directory",title='Err')

window.close()